/** * @param int $currentTick */ public function mainThreadHeartbeat($currentTick) { $this->currentTick = $currentTick; while ($this->isReady($this->currentTick)) { /** @var TaskHandler $task */ $task = $this->queue->extract(); if ($task->isCancelled()) { unset($this->tasks[$task->getTaskId()]); continue; } else { $task->timings->startTiming(); $task->run($this->currentTick); $task->timings->stopTiming(); } if ($task->isRepeating()) { $task->setNextRun($this->currentTick + $task->getPeriod()); $this->queue->insert($task, $this->currentTick + $task->getPeriod()); } else { $task->remove(); unset($this->tasks[$task->getTaskId()]); } } if ($this->asyncTasks > 0) { //Garbage collector $this->asyncPool->collect([$this, "collectAsyncTask"]); foreach ($this->asyncTaskStorage as $asyncTask) { if ($asyncTask->isFinished() and !$asyncTask->isCompleted()) { $this->collectAsyncTask($asyncTask); } } } }
protected function execute(InputInterface $input, OutputInterface $output) { $pool = new \Pool($input->getOption('threads'), \Worker::class); foreach ($input->getArgument('indexes') as $index) { $pool->submit(new IndexerRunner($input->getOption('indexer'), $input->getOption('config'), $index)); } $pool->shutdown(); $pool->collect(function (IndexerRunner $work) use($output) { $output->writeln($work->return); }); }
public function collectTasks() { Timings::$schedulerAsyncTimer->startTiming(); for ($i = 0; $i < 2; $i++) { if (!$this->pool->collect(function (AsyncTask $task) { if ($task->isGarbage() and !$task->isRunning() and !$task->isCrashed()) { if (!$task->hasCancelledRun()) { $task->onCompletion($this->server); } $this->removeTask($task); } elseif ($task->isTerminated() or $task->isCrashed()) { $this->server->getLogger()->critical("Could not execute asynchronous task " . (new \ReflectionClass($task))->getShortName() . ": Task crashed"); $this->removeTask($task); } return $task->isGarbage(); })) { break; } } Timings::$schedulerAsyncTimer->stopTiming(); }
private function testPool() { $initTime = microtime(true); $pool = new \Pool(8, \Worker::class); for ($i = 0; $i < 10; $i++) { $pool->submit(new Work()); } $initTime = microtime(true) - $initTime; $finishTime = microtime(true); $pool->shutdown(); $pool->collect(function (Work $work) { echo 'Work of ', get_class($work), ' #', $work->i, ' - ', $work->getWork(), PHP_EOL; }); $finishTime = microtime(true) - $finishTime; return [$initTime, $finishTime]; }
public static function Init() { $config = \Rds\Configuration::get(); $boot = \Rds\Bootstrap::getInstace($config); $totalPages = $boot->getTotalPages(); if ($totalPages > 0) { $pool = new \Pool($config["app"]["workers"], \Rds\Worker::class, array("Vendor/autoload.php", new \Rds\StackableConfig($config))); for ($page = 1; $page <= $totalPages; $page++) { $task = new \Rds\Task($page); $pool->submit($task); } $pool->shutdown(); $pool->collect(function ($work) { return $work->isGarbage(); }); } }
/** * @param int $currentTick */ public function mainThreadHeartbeat($currentTick) { $this->currentTick = $currentTick; while ($this->isReady($this->currentTick)) { /** @var TaskHandler $task */ $task = $this->queue->extract(); if ($task->isCancelled()) { unset($this->tasks[$task->getTaskId()]); continue; } else { $task->timings->startTiming(); try { $task->run($this->currentTick); } catch (\Exception $e) { Server::getInstance()->getLogger()->critical("Could not execute task " . $task->getTaskName() . ": " . $e->getMessage()); if (($logger = Server::getInstance()->getLogger()) instanceof MainLogger) { $logger->logException($e); } } $task->timings->stopTiming(); } if ($task->isRepeating()) { $task->setNextRun($this->currentTick + $task->getPeriod()); $this->queue->insert($task, $this->currentTick + $task->getPeriod()); } else { $task->remove(); unset($this->tasks[$task->getTaskId()]); } } if ($this->asyncTasks > 0) { //Garbage collector $this->asyncPool->collect([$this, "collectAsyncTask"]); if ($this->asyncTasks > 0) { foreach ($this->asyncTaskStorage as $asyncTask) { $this->collectAsyncTask($asyncTask); } } } }
public function run() { $mysqli = $this->worker->getConnection(); $result = $mysqli->query($this->sql); if ($result) { while ($row = $result->fetch_assoc()) { $rows[] = $row; } } $this->result = $rows; } public function getResult() { return $this->result; } protected $sql; protected $result; } $pool = new Pool(4, "Connect", ["localhost", "root", "", "mysql"]); $pool->submit(new Query("SHOW PROCESSLIST;")); $pool->submit(new Query("SHOW PROCESSLIST;")); $pool->submit(new Query("SHOW PROCESSLIST;")); $pool->submit(new Query("SHOW PROCESSLIST;")); $pool->submit(new Query("SHOW PROCESSLIST;")); $pool->submit(new Query("SHOW PROCESSLIST;")); $pool->shutdown(); /* ::collect is used here for shorthand to dump query results */ $pool->collect(function ($query) { var_dump($done = $query->getResult()); return count($done); });
} class SafeLog extends Stackable { protected function log($message, $args = []) { $args = func_get_args(); if ($message = array_shift($args)) { echo vsprintf("{$message}\n", $args); } } } $pool = new Pool(8, 'WebWorker', [new SafeLog()]); $pool->submit($w = new WebWork()); $pool->submit(new WebWork()); $pool->submit(new WebWork()); $pool->submit(new WebWork()); $pool->submit(new WebWork()); $pool->submit(new WebWork()); $pool->submit(new WebWork()); $pool->submit(new WebWork()); $pool->submit(new WebWork()); $pool->submit(new WebWork()); $pool->submit(new WebWork()); $pool->submit(new WebWork()); $pool->submit(new WebWork()); $pool->submit(new WebWork()); $pool->shutdown(); $pool->collect(function ($work) { return $work->isGarbage(); }); var_dump($pool);
public function testPoolGc() { $pool = new Pool(1, PoolTestWorker::class, [new stdClass(), new Threaded()]); $work = new PoolTestWork(); $pool->submit($work); while (@$i++ < 2) { $pool->submit(new PoolTestWork()); # nothing to assert, no exceptions please } $pool->submitTo(0, new PoolTestWork()); # nothing to assert, no exceptions please /* synchronize with pool */ $sync = new PoolTestSync(); $pool->submit($sync); $sync->synchronized(function ($sync) { if (!$sync->finished) { $sync->wait(); } }, $sync); $pool->collect(function ($task) { $this->assertTrue($task->isGarbage()); return true; }); $pool->shutdown(); }
$pagesAnalyzed++; $runpagecount++; echo "Submitted {$tpage['title']}, job " . ($tid + 1) . " for analyzing...\n"; $workerQueue->submit(new ThreadedBot($tpage['title'], $tpage['pageid'], $ARCHIVE_ALIVE, $TAG_OVERRIDE, $ARCHIVE_BY_ACCESSDATE, $TOUCH_ARCHIVE, $DEAD_ONLY, $NOTIFY_ERROR_ON_TALK, $NOTIFY_ON_TALK, $TALK_MESSAGE_HEADER, $TALK_MESSAGE, $TALK_ERROR_MESSAGE_HEADER, $TALK_ERROR_MESSAGE, $DEADLINK_TAGS, $CITATION_TAGS, $IGNORE_TAGS, $WAYBACK_TAGS, $WEBCITE_TAGS, $MEMENTO_TAGS, $ARCHIVEIS_TAGS, $ARCHIVE_TAGS, $IC_TAGS, $PAYWALL_TAGS, $VERIFY_DEAD, $LINK_SCAN, $NOTIFY_ON_TALK_ONLY, $MLADDARCHIVE, $MLMODIFYARCHIVE, $MLTAGGED, $MLTAGREMOVED, $MLFIX, $MLDEFAULT, $PLERROR, $MAINEDITSUMMARY, $ERRORTALKEDITSUMMARY, $TALKEDITSUMMARY, $tid)); if (LIMITEDRUN === true && is_int($debugStyle) && $debugStyle === $runpagecount) { break; } } $workerQueue->shutdown(); $workerQueue->collect(function ($thread) { global $pagesModified, $linksAnalyzed, $linksArchived, $linksFixed, $linksTagged; $stats = $thread->result; if ($stats['pagemodified'] === true) { $pagesModified++; } $linksAnalyzed += $stats['linksanalyzed']; $linksArchived += $stats['linksarchived']; $linksFixed += $stats['linksrescued']; $linksTagged += $stats['linkstagged']; $stats = null; unset($stats); return $thread->isGarbage(); }); if (file_exists(IAPROGRESS . WIKIPEDIA . UNIQUEID . "workers/") && ($handle = opendir(IAPROGRESS . WIKIPEDIA . UNIQUEID . "workers"))) { while (false !== ($entry = readdir($handle))) { if ($entry == "." || $entry == "..") { continue; } unlink(IAPROGRESS . WIKIPEDIA . UNIQUEID . "workers/{$entry}"); } } if (file_exists(IAPROGRESS . WIKIPEDIA . UNIQUEID . "workers/")) {
/* * The Workers in the Pool retain references to the WebWork objects submitted * in order to release that memory the Pool::collect method must be invoked in the same * context that created the Pool. * * The Worker::collect method is invoked for every Worker in the Pool, the garbage list * for each Worker is traversed and each Collectable is passed to the provided Closure. * * The Closure must return true if the Collectable can be removed from the garbage list. * * Worker::collect returns the size of the garbage list, Pool::collect returns the sum of the size of * the garbage list of all Workers in the Pool. * * Collecting in a continuous loop will cause the garbage list to be emptied. */ while ($pool->collect(function ($work) { return !$work->isRunning(); })) { continue; } /* * We could submit more stuff here, the Pool is still waiting for Collectables */ $logger->log(function ($pool) { echo '-----------------------'; var_dump($pool); }, $pool); /* * Shutdown Pools at the appropriate time, don't leave it to chance ! */ $pool->shutdown();