/**
* @param int $currentTick
*/
public function mainThreadHeartbeat($currentTick)
{
$this->currentTick = $currentTick;
while ($this->isReady($this->currentTick)) {
/** @var TaskHandler $task */
$task = $this->queue->extract();
if ($task->isCancelled()) {
unset($this->tasks[$task->getTaskId()]);
continue;
} else {
$task->timings->startTiming();
$task->run($this->currentTick);
$task->timings->stopTiming();
}
if ($task->isRepeating()) {
$task->setNextRun($this->currentTick + $task->getPeriod());
$this->queue->insert($task, $this->currentTick + $task->getPeriod());
} else {
$task->remove();
unset($this->tasks[$task->getTaskId()]);
}
}
if ($this->asyncTasks > 0) {
//Garbage collector
$this->asyncPool->collect([$this, "collectAsyncTask"]);
foreach ($this->asyncTaskStorage as $asyncTask) {
if ($asyncTask->isFinished() and !$asyncTask->isCompleted()) {
$this->collectAsyncTask($asyncTask);
}
}
}
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$pool = new \Pool($input->getOption('threads'), \Worker::class);
foreach ($input->getArgument('indexes') as $index) {
$pool->submit(new IndexerRunner($input->getOption('indexer'), $input->getOption('config'), $index));
}
$pool->shutdown();
$pool->collect(function (IndexerRunner $work) use($output) {
$output->writeln($work->return);
});
}
public function collectTasks()
{
Timings::$schedulerAsyncTimer->startTiming();
for ($i = 0; $i < 2; $i++) {
if (!$this->pool->collect(function (AsyncTask $task) {
if ($task->isGarbage() and !$task->isRunning() and !$task->isCrashed()) {
if (!$task->hasCancelledRun()) {
$task->onCompletion($this->server);
}
$this->removeTask($task);
} elseif ($task->isTerminated() or $task->isCrashed()) {
$this->server->getLogger()->critical("Could not execute asynchronous task " . (new \ReflectionClass($task))->getShortName() . ": Task crashed");
$this->removeTask($task);
}
return $task->isGarbage();
})) {
break;
}
}
Timings::$schedulerAsyncTimer->stopTiming();
}
private function testPool()
{
$initTime = microtime(true);
$pool = new \Pool(8, \Worker::class);
for ($i = 0; $i < 10; $i++) {
$pool->submit(new Work());
}
$initTime = microtime(true) - $initTime;
$finishTime = microtime(true);
$pool->shutdown();
$pool->collect(function (Work $work) {
echo 'Work of ', get_class($work), ' #', $work->i, ' - ', $work->getWork(), PHP_EOL;
});
$finishTime = microtime(true) - $finishTime;
return [$initTime, $finishTime];
}
public static function Init()
{
$config = \Rds\Configuration::get();
$boot = \Rds\Bootstrap::getInstace($config);
$totalPages = $boot->getTotalPages();
if ($totalPages > 0) {
$pool = new \Pool($config["app"]["workers"], \Rds\Worker::class, array("Vendor/autoload.php", new \Rds\StackableConfig($config)));
for ($page = 1; $page <= $totalPages; $page++) {
$task = new \Rds\Task($page);
$pool->submit($task);
}
$pool->shutdown();
$pool->collect(function ($work) {
return $work->isGarbage();
});
}
}
/**
* @param int $currentTick
*/
public function mainThreadHeartbeat($currentTick)
{
$this->currentTick = $currentTick;
while ($this->isReady($this->currentTick)) {
/** @var TaskHandler $task */
$task = $this->queue->extract();
if ($task->isCancelled()) {
unset($this->tasks[$task->getTaskId()]);
continue;
} else {
$task->timings->startTiming();
try {
$task->run($this->currentTick);
} catch (\Exception $e) {
Server::getInstance()->getLogger()->critical("Could not execute task " . $task->getTaskName() . ": " . $e->getMessage());
if (($logger = Server::getInstance()->getLogger()) instanceof MainLogger) {
$logger->logException($e);
}
}
$task->timings->stopTiming();
}
if ($task->isRepeating()) {
$task->setNextRun($this->currentTick + $task->getPeriod());
$this->queue->insert($task, $this->currentTick + $task->getPeriod());
} else {
$task->remove();
unset($this->tasks[$task->getTaskId()]);
}
}
if ($this->asyncTasks > 0) {
//Garbage collector
$this->asyncPool->collect([$this, "collectAsyncTask"]);
if ($this->asyncTasks > 0) {
foreach ($this->asyncTaskStorage as $asyncTask) {
$this->collectAsyncTask($asyncTask);
}
}
}
}
public function run()
{
$mysqli = $this->worker->getConnection();
$result = $mysqli->query($this->sql);
if ($result) {
while ($row = $result->fetch_assoc()) {
$rows[] = $row;
}
}
$this->result = $rows;
}
public function getResult()
{
return $this->result;
}
protected $sql;
protected $result;
}
$pool = new Pool(4, "Connect", ["localhost", "root", "", "mysql"]);
$pool->submit(new Query("SHOW PROCESSLIST;"));
$pool->submit(new Query("SHOW PROCESSLIST;"));
$pool->submit(new Query("SHOW PROCESSLIST;"));
$pool->submit(new Query("SHOW PROCESSLIST;"));
$pool->submit(new Query("SHOW PROCESSLIST;"));
$pool->submit(new Query("SHOW PROCESSLIST;"));
$pool->shutdown();
/* ::collect is used here for shorthand to dump query results */
$pool->collect(function ($query) {
var_dump($done = $query->getResult());
return count($done);
});
}
class SafeLog extends Stackable
{
protected function log($message, $args = [])
{
$args = func_get_args();
if ($message = array_shift($args)) {
echo vsprintf("{$message}\n", $args);
}
}
}
$pool = new Pool(8, 'WebWorker', [new SafeLog()]);
$pool->submit($w = new WebWork());
$pool->submit(new WebWork());
$pool->submit(new WebWork());
$pool->submit(new WebWork());
$pool->submit(new WebWork());
$pool->submit(new WebWork());
$pool->submit(new WebWork());
$pool->submit(new WebWork());
$pool->submit(new WebWork());
$pool->submit(new WebWork());
$pool->submit(new WebWork());
$pool->submit(new WebWork());
$pool->submit(new WebWork());
$pool->submit(new WebWork());
$pool->shutdown();
$pool->collect(function ($work) {
return $work->isGarbage();
});
var_dump($pool);
public function testPoolGc()
{
$pool = new Pool(1, PoolTestWorker::class, [new stdClass(), new Threaded()]);
$work = new PoolTestWork();
$pool->submit($work);
while (@$i++ < 2) {
$pool->submit(new PoolTestWork());
# nothing to assert, no exceptions please
}
$pool->submitTo(0, new PoolTestWork());
# nothing to assert, no exceptions please
/* synchronize with pool */
$sync = new PoolTestSync();
$pool->submit($sync);
$sync->synchronized(function ($sync) {
if (!$sync->finished) {
$sync->wait();
}
}, $sync);
$pool->collect(function ($task) {
$this->assertTrue($task->isGarbage());
return true;
});
$pool->shutdown();
}
$pagesAnalyzed++;
$runpagecount++;
echo "Submitted {$tpage['title']}, job " . ($tid + 1) . " for analyzing...\n";
$workerQueue->submit(new ThreadedBot($tpage['title'], $tpage['pageid'], $ARCHIVE_ALIVE, $TAG_OVERRIDE, $ARCHIVE_BY_ACCESSDATE, $TOUCH_ARCHIVE, $DEAD_ONLY, $NOTIFY_ERROR_ON_TALK, $NOTIFY_ON_TALK, $TALK_MESSAGE_HEADER, $TALK_MESSAGE, $TALK_ERROR_MESSAGE_HEADER, $TALK_ERROR_MESSAGE, $DEADLINK_TAGS, $CITATION_TAGS, $IGNORE_TAGS, $WAYBACK_TAGS, $WEBCITE_TAGS, $MEMENTO_TAGS, $ARCHIVEIS_TAGS, $ARCHIVE_TAGS, $IC_TAGS, $PAYWALL_TAGS, $VERIFY_DEAD, $LINK_SCAN, $NOTIFY_ON_TALK_ONLY, $MLADDARCHIVE, $MLMODIFYARCHIVE, $MLTAGGED, $MLTAGREMOVED, $MLFIX, $MLDEFAULT, $PLERROR, $MAINEDITSUMMARY, $ERRORTALKEDITSUMMARY, $TALKEDITSUMMARY, $tid));
if (LIMITEDRUN === true && is_int($debugStyle) && $debugStyle === $runpagecount) {
break;
}
}
$workerQueue->shutdown();
$workerQueue->collect(function ($thread) {
global $pagesModified, $linksAnalyzed, $linksArchived, $linksFixed, $linksTagged;
$stats = $thread->result;
if ($stats['pagemodified'] === true) {
$pagesModified++;
}
$linksAnalyzed += $stats['linksanalyzed'];
$linksArchived += $stats['linksarchived'];
$linksFixed += $stats['linksrescued'];
$linksTagged += $stats['linkstagged'];
$stats = null;
unset($stats);
return $thread->isGarbage();
});
if (file_exists(IAPROGRESS . WIKIPEDIA . UNIQUEID . "workers/") && ($handle = opendir(IAPROGRESS . WIKIPEDIA . UNIQUEID . "workers"))) {
while (false !== ($entry = readdir($handle))) {
if ($entry == "." || $entry == "..") {
continue;
}
unlink(IAPROGRESS . WIKIPEDIA . UNIQUEID . "workers/{$entry}");
}
}
if (file_exists(IAPROGRESS . WIKIPEDIA . UNIQUEID . "workers/")) {
/*
* The Workers in the Pool retain references to the WebWork objects submitted
* in order to release that memory the Pool::collect method must be invoked in the same
* context that created the Pool.
*
* The Worker::collect method is invoked for every Worker in the Pool, the garbage list
* for each Worker is traversed and each Collectable is passed to the provided Closure.
*
* The Closure must return true if the Collectable can be removed from the garbage list.
*
* Worker::collect returns the size of the garbage list, Pool::collect returns the sum of the size of
* the garbage list of all Workers in the Pool.
*
* Collecting in a continuous loop will cause the garbage list to be emptied.
*/
while ($pool->collect(function ($work) {
return !$work->isRunning();
})) {
continue;
}
/*
* We could submit more stuff here, the Pool is still waiting for Collectables
*/
$logger->log(function ($pool) {
echo '-----------------------';
var_dump($pool);
}, $pool);
/*
* Shutdown Pools at the appropriate time, don't leave it to chance !
*/
$pool->shutdown();
