/** * Add an update to the deferred list * @param DeferrableUpdate $update Some object that implements doUpdate() */ public static function addUpdate(DeferrableUpdate $update) { global $wgCommandLineMode; array_push(self::$updates, $update); if (self::$forceDeferral) { return; } // CLI scripts may forget to periodically flush these updates, // so try to handle that rather than OOMing and losing them. // Try to run the updates as soon as there is no local transaction. static $waitingOnTrx = false; // de-duplicate callback if ($wgCommandLineMode && !$waitingOnTrx) { $lb = wfGetLB(); $dbw = $lb->getAnyOpenConnection($lb->getWriterIndex()); // Do the update as soon as there is no transaction if ($dbw && $dbw->trxLevel()) { $waitingOnTrx = true; $dbw->onTransactionIdle(function () use(&$waitingOnTrx) { DeferredUpdates::doUpdates(); $waitingOnTrx = false; }); } else { self::doUpdates(); } } }
protected function autoreview_current(User $user) { $this->output("Auto-reviewing all current page versions...\n"); if (!$user->getID()) { $this->output("Invalid user specified.\n"); return; } elseif (!$user->isAllowed('review')) { $this->output("User specified (id: {$user->getID()}) does not have \"review\" rights.\n"); return; } $db = wfGetDB(DB_MASTER); $this->output("Reviewer username: "******"\n"); $start = $db->selectField('page', 'MIN(page_id)', false, __METHOD__); $end = $db->selectField('page', 'MAX(page_id)', false, __METHOD__); if (is_null($start) || is_null($end)) { $this->output("...page table seems to be empty.\n"); return; } # Do remaining chunk $end += $this->mBatchSize - 1; $blockStart = $start; $blockEnd = $start + $this->mBatchSize - 1; $count = 0; $changed = 0; $flags = FlaggedRevs::quickTags(FR_CHECKED); // Assume basic level while ($blockEnd <= $end) { $this->output("...doing page_id from {$blockStart} to {$blockEnd}\n"); $res = $db->select(array('page', 'revision'), '*', array("page_id BETWEEN {$blockStart} AND {$blockEnd}", 'page_namespace' => FlaggedRevs::getReviewNamespaces(), 'rev_id = page_latest'), __METHOD__); # Go through and autoreview the current version of every page... foreach ($res as $row) { $title = Title::newFromRow($row); $rev = Revision::newFromRow($row); # Is it already reviewed? $frev = FlaggedRevision::newFromTitle($title, $row->page_latest, FR_MASTER); # Rev should exist, but to be safe... if (!$frev && $rev) { $article = new Article($title); $db->begin(); FlaggedRevs::autoReviewEdit($article, $user, $rev, $flags, true); FlaggedRevs::HTMLCacheUpdates($article->getTitle()); $db->commit(); $changed++; } $count++; } $db->freeResult($res); $blockStart += $this->mBatchSize - 1; $blockEnd += $this->mBatchSize - 1; // XXX: Don't let deferred jobs array get absurdly large (bug 24375) DeferredUpdates::doUpdates('commit'); wfWaitForSlaves(5); } $this->output("Auto-reviewing of all pages complete ..." . "{$count} rows [{$changed} changed]\n"); }
public function testDoUpdates() { $updates = array('1' => 'deferred update 1', '2' => 'deferred update 2', '3' => 'deferred update 3', '2-1' => 'deferred update 1 within deferred update 2'); DeferredUpdates::addCallableUpdate(function () use($updates) { echo $updates['1']; }); DeferredUpdates::addCallableUpdate(function () use($updates) { echo $updates['2']; DeferredUpdates::addCallableUpdate(function () use($updates) { echo $updates['2-1']; }); }); DeferredUpdates::addCallableUpdate(function () use($updates) { echo $updates[3]; }); $this->expectOutputString(implode('', $updates)); DeferredUpdates::doUpdates(); }
public function testDoUpdatesCLI() { $this->setMwGlobals('wgCommandLineMode', true); $updates = array('1' => 'deferred update 1', '2' => 'deferred update 2', '2-1' => 'deferred update 1 within deferred update 2', '3' => 'deferred update 3'); DeferredUpdates::addCallableUpdate(function () use($updates) { echo $updates['1']; }); DeferredUpdates::addCallableUpdate(function () use($updates) { echo $updates['2']; DeferredUpdates::addCallableUpdate(function () use($updates) { echo $updates['2-1']; }); }); DeferredUpdates::addCallableUpdate(function () use($updates) { echo $updates[3]; }); $this->expectOutputString(implode('', $updates)); DeferredUpdates::doUpdates(); }
/** * @covers BagOStuff::__construct * @covers BagOStuff::trackDuplicateKeys */ public function testReportDupes() { $logger = $this->getMock('Psr\\Log\\NullLogger'); $logger->expects($this->once())->method('warning')->with('Duplicate get(): "{key}" fetched {count} times', ['key' => 'foo', 'count' => 2]); $cache = new HashBagOStuff(['reportDupes' => true, 'asyncHandler' => 'DeferredUpdates::addCallableUpdate', 'logger' => $logger]); $cache->get('foo'); $cache->get('bar'); $cache->get('foo'); DeferredUpdates::doUpdates(); }
/** * Ends this task peacefully * @param string $mode Use 'fast' to always skip job running */ public function restInPeace($mode = 'fast') { // Assure deferred updates are not in the main transaction wfGetLBFactory()->commitMasterChanges(__METHOD__); // Ignore things like master queries/connections on GET requests // as long as they are in deferred updates (which catch errors). Profiler::instance()->getTransactionProfiler()->resetExpectations(); // Do any deferred jobs DeferredUpdates::doUpdates('enqueue'); // Make sure any lazy jobs are pushed JobQueueGroup::pushLazyJobs(); // Now that everything specific to this request is done, // try to occasionally run jobs (if enabled) from the queues if ($mode === 'normal') { $this->triggerJobs(); } // Log profiling data, e.g. in the database or UDP wfLogProfilingData(); // Commit and close up! $factory = wfGetLBFactory(); $factory->commitMasterChanges(__METHOD__); $factory->shutdown(LBFactory::SHUTDOWN_NO_CHRONPROT); wfDebug("Request ended normally\n"); }
/** * Ends this task peacefully */ public function restInPeace() { // Ignore things like master queries/connections on GET requests // as long as they are in deferred updates (which catch errors). Profiler::instance()->getTransactionProfiler()->resetExpectations(); // Do any deferred jobs DeferredUpdates::doUpdates('commit'); // Make sure any lazy jobs are pushed JobQueueGroup::pushLazyJobs(); // Log profiling data, e.g. in the database or UDP wfLogProfilingData(); // Commit and close up! $factory = wfGetLBFactory(); $factory->commitMasterChanges(); $factory->shutdown(); wfDebug("Request ended normally\n"); }
/** * @param Job $job * @param BufferingStatsdDataFactory $stats * @param float $popTime * @return array Map of status/error/timeMs */ private function executeJob(Job $job, $stats, $popTime) { $jType = $job->getType(); $msg = $job->toString() . " STARTING"; $this->logger->debug($msg); $this->debugCallback($msg); // Run the job... $rssStart = $this->getMaxRssKb(); $jobStartTime = microtime(true); try { $status = $job->run(); $error = $job->getLastError(); $this->commitMasterChanges($job); DeferredUpdates::doUpdates(); $this->commitMasterChanges($job); } catch (Exception $e) { MWExceptionHandler::rollbackMasterChangesAndLog($e); $status = false; $error = get_class($e) . ': ' . $e->getMessage(); MWExceptionHandler::logException($e); } // Commit all outstanding connections that are in a transaction // to get a fresh repeatable read snapshot on every connection. // Note that jobs are still responsible for handling slave lag. wfGetLBFactory()->commitAll(__METHOD__); // Clear out title cache data from prior snapshots LinkCache::singleton()->clear(); $timeMs = intval((microtime(true) - $jobStartTime) * 1000); $rssEnd = $this->getMaxRssKb(); // Record how long jobs wait before getting popped $readyTs = $job->getReadyTimestamp(); if ($readyTs) { $pickupDelay = max(0, $popTime - $readyTs); $stats->timing('jobqueue.pickup_delay.all', 1000 * $pickupDelay); $stats->timing("jobqueue.pickup_delay.{$jType}", 1000 * $pickupDelay); } // Record root job age for jobs being run $root = $job->getRootJobParams(); if ($root['rootJobTimestamp']) { $age = max(0, $popTime - wfTimestamp(TS_UNIX, $root['rootJobTimestamp'])); $stats->timing("jobqueue.pickup_root_age.{$jType}", 1000 * $age); } // Track the execution time for jobs $stats->timing("jobqueue.run.{$jType}", $timeMs); // Track RSS increases for jobs (in case of memory leaks) if ($rssStart && $rssEnd) { $stats->increment("jobqueue.rss_delta.{$jType}", $rssEnd - $rssStart); } if ($status === false) { $msg = $job->toString() . " t={$timeMs} error={$error}"; $this->logger->error($msg); $this->debugCallback($msg); } else { $msg = $job->toString() . " t={$timeMs} good"; $this->logger->info($msg); $this->debugCallback($msg); } return array('status' => $status, 'error' => $error, 'timeMs' => $timeMs); }
public function testDoUpdatesCLI() { $this->setMwGlobals('wgCommandLineMode', true); $updates = ['1' => "deferred update 1;\n", '2' => "deferred update 2;\n", '2-1' => "deferred update 1 within deferred update 2;\n", '2-2' => "deferred update 2 within deferred update 2;\n", '3' => "deferred update 3;\n", '3-1' => "deferred update 1 within deferred update 3;\n", '3-2' => "deferred update 2 within deferred update 3;\n", '3-1-1' => "deferred update 1 within deferred update 1 within deferred update 3;\n", '3-2-1' => "deferred update 1 within deferred update 2 with deferred update 3;\n"]; wfGetLBFactory()->commitMasterChanges(__METHOD__); // clear anything DeferredUpdates::addCallableUpdate(function () use($updates) { echo $updates['1']; }); DeferredUpdates::addCallableUpdate(function () use($updates) { echo $updates['2']; DeferredUpdates::addCallableUpdate(function () use($updates) { echo $updates['2-1']; }); DeferredUpdates::addCallableUpdate(function () use($updates) { echo $updates['2-2']; }); }); DeferredUpdates::addCallableUpdate(function () use($updates) { echo $updates['3']; DeferredUpdates::addCallableUpdate(function () use($updates) { echo $updates['3-1']; DeferredUpdates::addCallableUpdate(function () use($updates) { echo $updates['3-1-1']; }); }); DeferredUpdates::addCallableUpdate(function () use($updates) { echo $updates['3-2']; DeferredUpdates::addCallableUpdate(function () use($updates) { echo $updates['3-2-1']; }); }); }); $this->expectOutputString(implode('', $updates)); DeferredUpdates::doUpdates(); }
/** * @since 2.4 */ public static function executePendingDeferredUpdates() { DeferredCallableUpdate::releasePendingUpdates(); \DeferredUpdates::doUpdates(); }
/** * Do any deferred updates and clear the list * * @deprecated since 1.19 * @see DeferredUpdates::doUpdate() * @param $commit string */ function wfDoUpdates($commit = '') { wfDeprecated(__METHOD__, '1.19'); DeferredUpdates::doUpdates($commit); }
function showReport() { if (!$this->mQuiet) { $delta = microtime(true) - $this->startTime; if ($delta) { $rate = sprintf("%.2f", $this->pageCount / $delta); $revrate = sprintf("%.2f", $this->revCount / $delta); } else { $rate = '-'; $revrate = '-'; } # Logs dumps don't have page tallies if ($this->pageCount) { $this->progress("{$this->pageCount} ({$rate} pages/sec {$revrate} revs/sec)"); } else { $this->progress("{$this->revCount} ({$revrate} revs/sec)"); } } wfWaitForSlaves(); // XXX: Don't let deferred jobs array get absurdly large (bug 24375) DeferredUpdates::doUpdates('commit'); }
$cluster = 'pmtpa'; require "{$IP}/../wmf-config/wgConf.php"; } // Require the configuration (probably LocalSettings.php) require $maintenance->loadSettings(); } if ($maintenance->getDbType() === Maintenance::DB_NONE) { if ($wgLocalisationCacheConf['storeClass'] === false && ($wgLocalisationCacheConf['store'] == 'db' || $wgLocalisationCacheConf['store'] == 'detect' && !$wgCacheDirectory)) { $wgLocalisationCacheConf['storeClass'] = 'LCStoreNull'; } } $maintenance->finalSetup(); // Some last includes require_once "{$IP}/includes/Setup.php"; // Do the work try { $maintenance->execute(); // Potentially debug globals $maintenance->globals(); // Perform deferred updates. DeferredUpdates::doUpdates('commit'); // log profiling info wfLogProfilingData(); // Commit and close up! $factory = wfGetLBFactory(); $factory->commitMasterChanges(); $factory->shutdown(); } catch (MWException $mwe) { echo $mwe->getText(); exit(1); }
private static function push(array &$queue, DeferrableUpdate $update) { global $wgCommandLineMode; if ($update instanceof MergeableUpdate) { $class = get_class($update); // fully-qualified class if (isset($queue[$class])) { /** @var $existingUpdate MergeableUpdate */ $existingUpdate = $queue[$class]; $existingUpdate->merge($update); } else { $queue[$class] = $update; } } else { $queue[] = $update; } // CLI scripts may forget to periodically flush these updates, // so try to handle that rather than OOMing and losing them entirely. // Try to run the updates as soon as there is no current wiki transaction. static $waitingOnTrx = false; // de-duplicate callback if ($wgCommandLineMode && !$waitingOnTrx) { $lb = wfGetLB(); $dbw = $lb->getAnyOpenConnection($lb->getWriterIndex()); // Do the update as soon as there is no transaction if ($dbw && $dbw->trxLevel()) { $waitingOnTrx = true; $dbw->onTransactionIdle(function () use(&$waitingOnTrx) { DeferredUpdates::doUpdates(); $waitingOnTrx = false; }); } else { self::doUpdates(); } } }
public function testAsyncWrites() { $be = TestingAccessWrapper::newFromObject(new FileBackendMultiWrite(array('name' => 'localtesting', 'wikiId' => wfWikiId() . mt_rand(), 'backends' => array(array('name' => 'multitesting0', 'class' => 'MemoryFileBackend', 'isMultiMaster' => false), array('name' => 'multitesting1', 'class' => 'MemoryFileBackend', 'isMultiMaster' => true)), 'replication' => 'async'))); DeferredUpdates::forceDeferral(true); $p = 'container/test-cont/file.txt'; $be->quickCreate(array('dst' => "mwstore://localtesting/{$p}", 'content' => 'cattitude')); $this->assertEquals(false, $be->backends[0]->getFileContents(array('src' => "mwstore://multitesting0/{$p}")), "File not yet written to backend 0"); $this->assertEquals('cattitude', $be->backends[1]->getFileContents(array('src' => "mwstore://multitesting1/{$p}")), "File already written to backend 1"); DeferredUpdates::doUpdates(); DeferredUpdates::forceDeferral(false); $this->assertEquals('cattitude', $be->backends[0]->getFileContents(array('src' => "mwstore://multitesting0/{$p}")), "File now written to backend 0"); }
/** * Cleaning up request by doing deferred updates, DB transaction, and the output */ public function finalCleanup() { wfProfileIn(__METHOD__); // Now commit any transactions, so that unreported errors after // output() don't roll back the whole DB transaction $factory = wfGetLBFactory(); $factory->commitMasterChanges(); // Output everything! $this->context->getOutput()->output(); // Do any deferred jobs DeferredUpdates::doUpdates('commit'); $this->doJobs(); wfProfileOut(__METHOD__); }
/** * Ends this task peacefully * @param string $mode Use 'fast' to always skip job running */ public function restInPeace($mode = 'fast') { $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory(); // Assure deferred updates are not in the main transaction $lbFactory->commitMasterChanges(__METHOD__); // Loosen DB query expectations since the HTTP client is unblocked $trxProfiler = Profiler::instance()->getTransactionProfiler(); $trxProfiler->resetExpectations(); $trxProfiler->setExpectations($this->config->get('TrxProfilerLimits')['PostSend'], __METHOD__); // Do any deferred jobs DeferredUpdates::doUpdates('enqueue'); DeferredUpdates::setImmediateMode(true); // Make sure any lazy jobs are pushed JobQueueGroup::pushLazyJobs(); // Now that everything specific to this request is done, // try to occasionally run jobs (if enabled) from the queues if ($mode === 'normal') { $this->triggerJobs(); } // Log profiling data, e.g. in the database or UDP wfLogProfilingData(); // Commit and close up! $lbFactory->commitMasterChanges(__METHOD__); $lbFactory->shutdown(LBFactory::SHUTDOWN_NO_CHRONPROT); wfDebug("Request ended normally\n"); }
/** * Run jobs of the specified number/type for the specified time * * The response map has a 'job' field that lists status of each job, including: * - type : the job type * - status : ok/failed * - error : any error message string * - time : the job run time in ms * The response map also has: * - backoffs : the (job type => seconds) map of backoff times * - elapsed : the total time spent running tasks in ms * - reached : the reason the script finished, one of (none-ready, job-limit, time-limit) * * This method outputs status information only if a debug handler was set. * Any exceptions are caught and logged, but are not reported as output. * * @param array $options Map of parameters: * - type : the job type (or false for the default types) * - maxJobs : maximum number of jobs to run * - maxTime : maximum time in seconds before stopping * - throttle : whether to respect job backoff configuration * @return array Summary response that can easily be JSON serialized */ public function run(array $options) { global $wgJobClasses, $wgTrxProfilerLimits; $response = array('jobs' => array(), 'reached' => 'none-ready'); $type = isset($options['type']) ? $options['type'] : false; $maxJobs = isset($options['maxJobs']) ? $options['maxJobs'] : false; $maxTime = isset($options['maxTime']) ? $options['maxTime'] : false; $noThrottle = isset($options['throttle']) && !$options['throttle']; if ($type !== false && !isset($wgJobClasses[$type])) { $response['reached'] = 'none-possible'; return $response; } // Bail out if in read-only mode if (wfReadOnly()) { $response['reached'] = 'read-only'; return $response; } // Catch huge single updates that lead to slave lag $trxProfiler = Profiler::instance()->getTransactionProfiler(); $trxProfiler->setLogger(LoggerFactory::getInstance('DBPerformance')); $trxProfiler->setExpectations($wgTrxProfilerLimits['JobRunner'], __METHOD__); // Bail out if there is too much DB lag. // This check should not block as we want to try other wiki queues. $maxAllowedLag = 3; list(, $maxLag) = wfGetLB(wfWikiID())->getMaxLag(); if ($maxLag >= $maxAllowedLag) { $response['reached'] = 'slave-lag-limit'; return $response; } $group = JobQueueGroup::singleton(); // Flush any pending DB writes for sanity wfGetLBFactory()->commitAll(); // Some jobs types should not run until a certain timestamp $backoffs = array(); // map of (type => UNIX expiry) $backoffDeltas = array(); // map of (type => seconds) $wait = 'wait'; // block to read backoffs the first time $stats = RequestContext::getMain()->getStats(); $jobsPopped = 0; $timeMsTotal = 0; $flags = JobQueueGroup::USE_CACHE; $startTime = microtime(true); // time since jobs started running $checkLagPeriod = 1.0; // check slave lag this many seconds $lastCheckTime = 1; // timestamp of last slave check do { // Sync the persistent backoffs with concurrent runners $backoffs = $this->syncBackoffDeltas($backoffs, $backoffDeltas, $wait); $blacklist = $noThrottle ? array() : array_keys($backoffs); $wait = 'nowait'; // less important now if ($type === false) { $job = $group->pop(JobQueueGroup::TYPE_DEFAULT, $flags, $blacklist); } elseif (in_array($type, $blacklist)) { $job = false; // requested queue in backoff state } else { $job = $group->pop($type); // job from a single queue } if ($job) { // found a job $popTime = time(); $jType = $job->getType(); // Back off of certain jobs for a while (for throttling and for errors) $ttw = $this->getBackoffTimeToWait($job); if ($ttw > 0) { // Always add the delta for other runners in case the time running the // job negated the backoff for each individually but not collectively. $backoffDeltas[$jType] = isset($backoffDeltas[$jType]) ? $backoffDeltas[$jType] + $ttw : $ttw; $backoffs = $this->syncBackoffDeltas($backoffs, $backoffDeltas, $wait); } $msg = $job->toString() . " STARTING"; $this->logger->debug($msg); $this->debugCallback($msg); // Run the job... $jobStartTime = microtime(true); try { ++$jobsPopped; $status = $job->run(); $error = $job->getLastError(); $this->commitMasterChanges($job); DeferredUpdates::doUpdates(); $this->commitMasterChanges($job); } catch (Exception $e) { MWExceptionHandler::rollbackMasterChangesAndLog($e); $status = false; $error = get_class($e) . ': ' . $e->getMessage(); MWExceptionHandler::logException($e); } // Commit all outstanding connections that are in a transaction // to get a fresh repeatable read snapshot on every connection. // Note that jobs are still responsible for handling slave lag. wfGetLBFactory()->commitAll(); // Clear out title cache data from prior snapshots LinkCache::singleton()->clear(); $timeMs = intval((microtime(true) - $jobStartTime) * 1000); $timeMsTotal += $timeMs; // Record how long jobs wait before getting popped $readyTs = $job->getReadyTimestamp(); if ($readyTs) { $pickupDelay = $popTime - $readyTs; $stats->timing('jobqueue.pickup_delay.all', 1000 * $pickupDelay); $stats->timing("jobqueue.pickup_delay.{$jType}", 1000 * $pickupDelay); } // Record root job age for jobs being run $root = $job->getRootJobParams(); if ($root['rootJobTimestamp']) { $age = $popTime - wfTimestamp(TS_UNIX, $root['rootJobTimestamp']); $stats->timing("jobqueue.pickup_root_age.{$jType}", 1000 * $age); } // Track the execution time for jobs $stats->timing("jobqueue.run.{$jType}", $timeMs); // Mark the job as done on success or when the job cannot be retried if ($status !== false || !$job->allowRetries()) { $group->ack($job); // done } // Back off of certain jobs for a while (for throttling and for errors) if ($status === false && mt_rand(0, 49) == 0) { $ttw = max($ttw, 30); // too many errors $backoffDeltas[$jType] = isset($backoffDeltas[$jType]) ? $backoffDeltas[$jType] + $ttw : $ttw; } if ($status === false) { $msg = $job->toString() . " t={$timeMs} error={$error}"; $this->logger->error($msg); $this->debugCallback($msg); } else { $msg = $job->toString() . " t={$timeMs} good"; $this->logger->info($msg); $this->debugCallback($msg); } $response['jobs'][] = array('type' => $jType, 'status' => $status === false ? 'failed' : 'ok', 'error' => $error, 'time' => $timeMs); // Break out if we hit the job count or wall time limits... if ($maxJobs && $jobsPopped >= $maxJobs) { $response['reached'] = 'job-limit'; break; } elseif ($maxTime && microtime(true) - $startTime > $maxTime) { $response['reached'] = 'time-limit'; break; } // Don't let any of the main DB slaves get backed up. // This only waits for so long before exiting and letting // other wikis in the farm (on different masters) get a chance. $timePassed = microtime(true) - $lastCheckTime; if ($timePassed >= $checkLagPeriod || $timePassed < 0) { if (!wfWaitForSlaves($lastCheckTime, false, '*', $maxAllowedLag)) { $response['reached'] = 'slave-lag-limit'; break; } $lastCheckTime = microtime(true); } // Don't let any queue slaves/backups fall behind if ($jobsPopped > 0 && $jobsPopped % 100 == 0) { $group->waitForBackups(); } // Bail if near-OOM instead of in a job if (!$this->checkMemoryOK()) { $response['reached'] = 'memory-limit'; break; } } } while ($job); // stop when there are no jobs // Sync the persistent backoffs for the next runJobs.php pass if ($backoffDeltas) { $this->syncBackoffDeltas($backoffs, $backoffDeltas, 'wait'); } $response['backoffs'] = $backoffs; $response['elapsed'] = $timeMsTotal; return $response; }
throw new MWException('ApiBeforMain hook set $processor to a non-ApiMain class'); } } catch (Exception $e) { // Crap. Try to report the exception in API format to be friendly to clients. ApiMain::handleApiBeforeMainException($e); $processor = false; } // Process data & print results if ($processor) { $processor->execute(); } if (function_exists('fastcgi_finish_request')) { fastcgi_finish_request(); } // Execute any deferred updates DeferredUpdates::doUpdates(); // Log what the user did, for book-keeping purposes. $endtime = microtime(true); wfProfileOut('api.php'); wfLogProfilingData(); // Log the request if ($wgAPIRequestLog) { $items = array(wfTimestamp(TS_MW), $endtime - $starttime, $wgRequest->getIP(), $wgRequest->getHeader('User-agent')); $items[] = $wgRequest->wasPosted() ? 'POST' : 'GET'; if ($processor) { try { $manager = $processor->getModuleManager(); $module = $manager->getModule($wgRequest->getVal('action'), 'action'); } catch (Exception $ex) { $module = null; }
/** * @param Job $job * @param LBFactory $lbFactory * @param StatsdDataFactory $stats * @param float $popTime * @return array Map of status/error/timeMs */ private function executeJob(Job $job, LBFactory $lbFactory, $stats, $popTime) { $jType = $job->getType(); $msg = $job->toString() . " STARTING"; $this->logger->debug($msg); $this->debugCallback($msg); // Run the job... $rssStart = $this->getMaxRssKb(); $jobStartTime = microtime(true); try { $fnameTrxOwner = get_class($job) . '::run'; // give run() outer scope $lbFactory->beginMasterChanges($fnameTrxOwner); $status = $job->run(); $error = $job->getLastError(); $this->commitMasterChanges($lbFactory, $job, $fnameTrxOwner); // Run any deferred update tasks; doUpdates() manages transactions itself DeferredUpdates::doUpdates(); } catch (Exception $e) { MWExceptionHandler::rollbackMasterChangesAndLog($e); $status = false; $error = get_class($e) . ': ' . $e->getMessage(); } // Always attempt to call teardown() even if Job throws exception. try { $job->teardown($status); } catch (Exception $e) { MWExceptionHandler::logException($e); } // Commit all outstanding connections that are in a transaction // to get a fresh repeatable read snapshot on every connection. // Note that jobs are still responsible for handling replica DB lag. $lbFactory->flushReplicaSnapshots(__METHOD__); // Clear out title cache data from prior snapshots MediaWikiServices::getInstance()->getLinkCache()->clear(); $timeMs = intval((microtime(true) - $jobStartTime) * 1000); $rssEnd = $this->getMaxRssKb(); // Record how long jobs wait before getting popped $readyTs = $job->getReadyTimestamp(); if ($readyTs) { $pickupDelay = max(0, $popTime - $readyTs); $stats->timing('jobqueue.pickup_delay.all', 1000 * $pickupDelay); $stats->timing("jobqueue.pickup_delay.{$jType}", 1000 * $pickupDelay); } // Record root job age for jobs being run $rootTimestamp = $job->getRootJobParams()['rootJobTimestamp']; if ($rootTimestamp) { $age = max(0, $popTime - wfTimestamp(TS_UNIX, $rootTimestamp)); $stats->timing("jobqueue.pickup_root_age.{$jType}", 1000 * $age); } // Track the execution time for jobs $stats->timing("jobqueue.run.{$jType}", $timeMs); // Track RSS increases for jobs (in case of memory leaks) if ($rssStart && $rssEnd) { $stats->updateCount("jobqueue.rss_delta.{$jType}", $rssEnd - $rssStart); } if ($status === false) { $msg = $job->toString() . " t={$timeMs} error={$error}"; $this->logger->error($msg); $this->debugCallback($msg); } else { $msg = $job->toString() . " t={$timeMs} good"; $this->logger->info($msg); $this->debugCallback($msg); } return ['status' => $status, 'error' => $error, 'timeMs' => $timeMs]; }
/** * Ends this task peacefully */ public function restInPeace() { // Do any deferred jobs DeferredUpdates::doUpdates('commit'); // Execute a job from the queue $this->doJobs(); // Log profiling data, e.g. in the database or UDP wfLogProfilingData(); // Commit and close up! $factory = wfGetLBFactory(); $factory->commitMasterChanges(); $factory->shutdown(); wfDebug("Request ended normally\n"); }
/** * Change an existing article or create a new article. Updates RC and all necessary caches, * optionally via the deferred update array. * * @param $text String: new text * @param $summary String: edit summary * @param $flags Integer bitfield: * EDIT_NEW * Article is known or assumed to be non-existent, create a new one * EDIT_UPDATE * Article is known or assumed to be pre-existing, update it * EDIT_MINOR * Mark this edit minor, if the user is allowed to do so * EDIT_SUPPRESS_RC * Do not log the change in recentchanges * EDIT_FORCE_BOT * Mark the edit a "bot" edit regardless of user rights * EDIT_DEFER_UPDATES * Defer some of the updates until the end of index.php * EDIT_AUTOSUMMARY * Fill in blank summaries with generated text where possible * * If neither EDIT_NEW nor EDIT_UPDATE is specified, the status of the article will be detected. * If EDIT_UPDATE is specified and the article doesn't exist, the function will return an * edit-gone-missing error. If EDIT_NEW is specified and the article does exist, an * edit-already-exists error will be returned. These two conditions are also possible with * auto-detection due to MediaWiki's performance-optimised locking strategy. * * @param $baseRevId the revision ID this edit was based off, if any * @param $user User the user doing the edit * * @return Status object. Possible errors: * edit-hook-aborted: The ArticleSave hook aborted the edit but didn't set the fatal flag of $status * edit-gone-missing: In update mode, but the article didn't exist * edit-conflict: In update mode, the article changed unexpectedly * edit-no-change: Warning that the text was the same as before * edit-already-exists: In creation mode, but the article already exists * * Extensions may define additional errors. * * $return->value will contain an associative array with members as follows: * new: Boolean indicating if the function attempted to create a new article * revision: The revision object for the inserted revision, or null * * Compatibility note: this function previously returned a boolean value indicating success/failure */ public function doEdit($text, $summary, $flags = 0, $baseRevId = false, $user = null) { global $wgUser, $wgDBtransactions, $wgUseAutomaticEditSummaries; # Low-level sanity check if ($this->mTitle->getText() === '') { throw new MWException('Something is trying to edit an article with an empty title'); } wfProfileIn(__METHOD__); $user = is_null($user) ? $wgUser : $user; $status = Status::newGood(array()); # Load $this->mTitle->getArticleID() and $this->mLatest if it's not already $this->loadPageData('fromdbmaster'); $flags = $this->checkFlags($flags); if (!wfRunHooks('ArticleSave', array(&$this, &$user, &$text, &$summary, $flags & EDIT_MINOR, null, null, &$flags, &$status))) { wfDebug(__METHOD__ . ": ArticleSave hook aborted save!\n"); if ($status->isOK()) { $status->fatal('edit-hook-aborted'); } wfProfileOut(__METHOD__); return $status; } # Silently ignore EDIT_MINOR if not allowed $isminor = $flags & EDIT_MINOR && $user->isAllowed('minoredit'); $bot = $flags & EDIT_FORCE_BOT; $oldtext = $this->getRawText(); // current revision $oldsize = strlen($oldtext); $oldid = $this->getLatest(); $oldIsRedirect = $this->isRedirect(); $oldcountable = $this->isCountable(); # Provide autosummaries if one is not provided and autosummaries are enabled. if ($wgUseAutomaticEditSummaries && $flags & EDIT_AUTOSUMMARY && $summary == '') { $summary = self::getAutosummary($oldtext, $text, $flags); } $editInfo = $this->prepareTextForEdit($text, null, $user); $text = $editInfo->pst; $newsize = strlen($text); $dbw = wfGetDB(DB_MASTER); $now = wfTimestampNow(); $this->mTimestamp = $now; if ($flags & EDIT_UPDATE) { # Update article, but only if changed. $status->value['new'] = false; if (!$oldid) { # Article gone missing wfDebug(__METHOD__ . ": EDIT_UPDATE specified but article doesn't exist\n"); $status->fatal('edit-gone-missing'); wfProfileOut(__METHOD__); return $status; } # Make sure the revision is either completely inserted or not inserted at all if (!$wgDBtransactions) { $userAbort = ignore_user_abort(true); } $revision = new Revision(array('page' => $this->getId(), 'comment' => $summary, 'minor_edit' => $isminor, 'text' => $text, 'parent_id' => $oldid, 'user' => $user->getId(), 'user_text' => $user->getName(), 'timestamp' => $now)); $changed = strcmp($text, $oldtext) != 0; if ($changed) { $dbw->begin(); $revisionId = $revision->insertOn($dbw); # Update page # # Note that we use $this->mLatest instead of fetching a value from the master DB # during the course of this function. This makes sure that EditPage can detect # edit conflicts reliably, either by $ok here, or by $article->getTimestamp() # before this function is called. A previous function used a separate query, this # creates a window where concurrent edits can cause an ignored edit conflict. $ok = $this->updateRevisionOn($dbw, $revision, $oldid, $oldIsRedirect); if (!$ok) { /* Belated edit conflict! Run away!! */ $status->fatal('edit-conflict'); # Delete the invalid revision if the DB is not transactional if (!$wgDBtransactions) { $dbw->delete('revision', array('rev_id' => $revisionId), __METHOD__); } $revisionId = 0; $dbw->rollback(); } else { global $wgUseRCPatrol; wfRunHooks('NewRevisionFromEditComplete', array($this, $revision, $baseRevId, $user, false)); # Update recentchanges if (!($flags & EDIT_SUPPRESS_RC)) { # Mark as patrolled if the user can do so $patrolled = $wgUseRCPatrol && !count($this->mTitle->getUserPermissionsErrors('autopatrol', $user)); # Add RC row to the DB $rc = RecentChange::notifyEdit($now, $this->mTitle, $isminor, $user, $summary, $oldid, $this->getTimestamp(), $bot, '', $oldsize, $newsize, $revisionId, $patrolled); # Log auto-patrolled edits if ($patrolled) { PatrolLog::record($rc, true, $user); } } $user->incEditCount(); $dbw->commit(); } } else { // Bug 32948: revision ID must be set to page {{REVISIONID}} and // related variables correctly $revision->setId($this->getLatest()); } if (!$wgDBtransactions) { ignore_user_abort($userAbort); } // Now that ignore_user_abort is restored, we can respond to fatal errors if (!$status->isOK()) { wfProfileOut(__METHOD__); return $status; } # Update links tables, site stats, etc. $this->doEditUpdates($revision, $user, array('changed' => $changed, 'oldcountable' => $oldcountable)); if (!$changed) { $status->warning('edit-no-change'); $revision = null; // Update page_touched, this is usually implicit in the page update // Other cache updates are done in onArticleEdit() $this->mTitle->invalidateCache(); } } else { # Create new article $status->value['new'] = true; $dbw->begin(); # Add the page record; stake our claim on this title! # This will return false if the article already exists $newid = $this->insertOn($dbw); if ($newid === false) { $dbw->rollback(); $status->fatal('edit-already-exists'); wfProfileOut(__METHOD__); return $status; } # Save the revision text... $revision = new Revision(array('page' => $newid, 'comment' => $summary, 'minor_edit' => $isminor, 'text' => $text, 'user' => $user->getId(), 'user_text' => $user->getName(), 'timestamp' => $now)); $revisionId = $revision->insertOn($dbw); # Update the page record with revision data $this->updateRevisionOn($dbw, $revision, 0); wfRunHooks('NewRevisionFromEditComplete', array($this, $revision, false, $user)); # Update recentchanges if (!($flags & EDIT_SUPPRESS_RC)) { global $wgUseRCPatrol, $wgUseNPPatrol; # Mark as patrolled if the user can do so $patrolled = ($wgUseRCPatrol || $wgUseNPPatrol) && !count($this->mTitle->getUserPermissionsErrors('autopatrol', $user)); # Add RC row to the DB $rc = RecentChange::notifyNew($now, $this->mTitle, $isminor, $user, $summary, $bot, '', strlen($text), $revisionId, $patrolled); # Log auto-patrolled edits if ($patrolled) { PatrolLog::record($rc, true, $user); } } $user->incEditCount(); $dbw->commit(); # Update links, etc. $this->doEditUpdates($revision, $user, array('created' => true)); wfRunHooks('ArticleInsertComplete', array(&$this, &$user, $text, $summary, $flags & EDIT_MINOR, null, null, &$flags, $revision)); } # Do updates right now unless deferral was requested if (!($flags & EDIT_DEFER_UPDATES)) { DeferredUpdates::doUpdates(); } // Return the new revision (or null) to the caller $status->value['revision'] = $revision; wfRunHooks('ArticleSaveComplete', array(&$this, &$user, $text, $summary, $flags & EDIT_MINOR, null, null, &$flags, $revision, &$status, $baseRevId)); # Promote user to any groups they meet the criteria for $user->addAutopromoteOnceGroups('onEdit'); wfProfileOut(__METHOD__); return $status; }
/** * Change an existing article or create a new article. Updates RC and all necessary caches, * optionally via the deferred update array. * * @param $content Content: new content * @param string $summary edit summary * @param $flags Integer bitfield: * EDIT_NEW * Article is known or assumed to be non-existent, create a new one * EDIT_UPDATE * Article is known or assumed to be pre-existing, update it * EDIT_MINOR * Mark this edit minor, if the user is allowed to do so * EDIT_SUPPRESS_RC * Do not log the change in recentchanges * EDIT_FORCE_BOT * Mark the edit a "bot" edit regardless of user rights * EDIT_DEFER_UPDATES * Defer some of the updates until the end of index.php * EDIT_AUTOSUMMARY * Fill in blank summaries with generated text where possible * * If neither EDIT_NEW nor EDIT_UPDATE is specified, the status of the article will be detected. * If EDIT_UPDATE is specified and the article doesn't exist, the function will return an * edit-gone-missing error. If EDIT_NEW is specified and the article does exist, an * edit-already-exists error will be returned. These two conditions are also possible with * auto-detection due to MediaWiki's performance-optimised locking strategy. * * @param bool|int $baseRevId the revision ID this edit was based off, if any * @param $user User the user doing the edit * @param $serialisation_format String: format for storing the content in the database * * @throws MWException * @return Status object. Possible errors: * edit-hook-aborted: The ArticleSave hook aborted the edit but didn't set the fatal flag of $status * edit-gone-missing: In update mode, but the article didn't exist * edit-conflict: In update mode, the article changed unexpectedly * edit-no-change: Warning that the text was the same as before * edit-already-exists: In creation mode, but the article already exists * * Extensions may define additional errors. * * $return->value will contain an associative array with members as follows: * new: Boolean indicating if the function attempted to create a new article * revision: The revision object for the inserted revision, or null * * @since 1.21 */ public function doEditContent( Content $content, $summary, $flags = 0, $baseRevId = false, User $user = null, $serialisation_format = null ) { global $wgUser, $wgUseAutomaticEditSummaries, $wgUseRCPatrol, $wgUseNPPatrol; // Low-level sanity check if ( $this->mTitle->getText() === '' ) { throw new MWException( 'Something is trying to edit an article with an empty title' ); } wfProfileIn( __METHOD__ ); if ( !$content->getContentHandler()->canBeUsedOn( $this->getTitle() ) ) { wfProfileOut( __METHOD__ ); return Status::newFatal( 'content-not-allowed-here', ContentHandler::getLocalizedName( $content->getModel() ), $this->getTitle()->getPrefixedText() ); } $user = is_null( $user ) ? $wgUser : $user; $status = Status::newGood( array() ); // Load the data from the master database if needed. // The caller may already loaded it from the master or even loaded it using // SELECT FOR UPDATE, so do not override that using clear(). $this->loadPageData( 'fromdbmaster' ); $flags = $this->checkFlags( $flags ); // handle hook $hook_args = array( &$this, &$user, &$content, &$summary, $flags & EDIT_MINOR, null, null, &$flags, &$status ); if ( !wfRunHooks( 'PageContentSave', $hook_args ) || !ContentHandler::runLegacyHooks( 'ArticleSave', $hook_args ) ) { wfDebug( __METHOD__ . ": ArticleSave or ArticleSaveContent hook aborted save!\n" ); if ( $status->isOK() ) { $status->fatal( 'edit-hook-aborted' ); } wfProfileOut( __METHOD__ ); return $status; } // Silently ignore EDIT_MINOR if not allowed $isminor = ( $flags & EDIT_MINOR ) && $user->isAllowed( 'minoredit' ); $bot = $flags & EDIT_FORCE_BOT; $old_content = $this->getContent( Revision::RAW ); // current revision's content $oldsize = $old_content ? $old_content->getSize() : 0; $oldid = $this->getLatest(); $oldIsRedirect = $this->isRedirect(); $oldcountable = $this->isCountable(); $handler = $content->getContentHandler(); // Provide autosummaries if one is not provided and autosummaries are enabled. if ( $wgUseAutomaticEditSummaries && $flags & EDIT_AUTOSUMMARY && $summary == '' ) { if ( !$old_content ) { $old_content = null; } $summary = $handler->getAutosummary( $old_content, $content, $flags ); } $editInfo = $this->prepareContentForEdit( $content, null, $user, $serialisation_format ); $serialized = $editInfo->pst; /** * @var Content $content */ $content = $editInfo->pstContent; $newsize = $content->getSize(); $dbw = wfGetDB( DB_MASTER ); $now = wfTimestampNow(); $this->mTimestamp = $now; if ( $flags & EDIT_UPDATE ) { // Update article, but only if changed. $status->value['new'] = false; if ( !$oldid ) { // Article gone missing wfDebug( __METHOD__ . ": EDIT_UPDATE specified but article doesn't exist\n" ); $status->fatal( 'edit-gone-missing' ); wfProfileOut( __METHOD__ ); return $status; } elseif ( !$old_content ) { // Sanity check for bug 37225 wfProfileOut( __METHOD__ ); throw new MWException( "Could not find text for current revision {$oldid}." ); } $revision = new Revision( array( 'page' => $this->getId(), 'title' => $this->getTitle(), // for determining the default content model 'comment' => $summary, 'minor_edit' => $isminor, 'text' => $serialized, 'len' => $newsize, 'parent_id' => $oldid, 'user' => $user->getId(), 'user_text' => $user->getName(), 'timestamp' => $now, 'content_model' => $content->getModel(), 'content_format' => $serialisation_format, ) ); // XXX: pass content object?! $changed = !$content->equals( $old_content ); if ( $changed ) { if ( !$content->isValid() ) { wfProfileOut( __METHOD__ ); throw new MWException( "New content failed validity check!" ); } $dbw->begin( __METHOD__ ); $prepStatus = $content->prepareSave( $this, $flags, $baseRevId, $user ); $status->merge( $prepStatus ); if ( !$status->isOK() ) { $dbw->rollback( __METHOD__ ); wfProfileOut( __METHOD__ ); return $status; } $revisionId = $revision->insertOn( $dbw ); // Update page // // Note that we use $this->mLatest instead of fetching a value from the master DB // during the course of this function. This makes sure that EditPage can detect // edit conflicts reliably, either by $ok here, or by $article->getTimestamp() // before this function is called. A previous function used a separate query, this // creates a window where concurrent edits can cause an ignored edit conflict. $ok = $this->updateRevisionOn( $dbw, $revision, $oldid, $oldIsRedirect ); if ( !$ok ) { // Belated edit conflict! Run away!! $status->fatal( 'edit-conflict' ); $dbw->rollback( __METHOD__ ); wfProfileOut( __METHOD__ ); return $status; } wfRunHooks( 'NewRevisionFromEditComplete', array( $this, $revision, $baseRevId, $user ) ); // Update recentchanges if ( !( $flags & EDIT_SUPPRESS_RC ) ) { // Mark as patrolled if the user can do so $patrolled = $wgUseRCPatrol && !count( $this->mTitle->getUserPermissionsErrors( 'autopatrol', $user ) ); // Add RC row to the DB $rc = RecentChange::notifyEdit( $now, $this->mTitle, $isminor, $user, $summary, $oldid, $this->getTimestamp(), $bot, '', $oldsize, $newsize, $revisionId, $patrolled ); // Log auto-patrolled edits if ( $patrolled ) { PatrolLog::record( $rc, true, $user ); } } $user->incEditCount(); $dbw->commit( __METHOD__ ); } else { // Bug 32948: revision ID must be set to page {{REVISIONID}} and // related variables correctly $revision->setId( $this->getLatest() ); } // Update links tables, site stats, etc. $this->doEditUpdates( $revision, $user, array( 'changed' => $changed, 'oldcountable' => $oldcountable ) ); if ( !$changed ) { $status->warning( 'edit-no-change' ); $revision = null; // Update page_touched, this is usually implicit in the page update // Other cache updates are done in onArticleEdit() $this->mTitle->invalidateCache(); } } else { // Create new article $status->value['new'] = true; $dbw->begin( __METHOD__ ); $prepStatus = $content->prepareSave( $this, $flags, $baseRevId, $user ); $status->merge( $prepStatus ); if ( !$status->isOK() ) { $dbw->rollback( __METHOD__ ); wfProfileOut( __METHOD__ ); return $status; } $status->merge( $prepStatus ); // Add the page record; stake our claim on this title! // This will return false if the article already exists $newid = $this->insertOn( $dbw ); if ( $newid === false ) { $dbw->rollback( __METHOD__ ); $status->fatal( 'edit-already-exists' ); wfProfileOut( __METHOD__ ); return $status; } // Save the revision text... $revision = new Revision( array( 'page' => $newid, 'title' => $this->getTitle(), // for determining the default content model 'comment' => $summary, 'minor_edit' => $isminor, 'text' => $serialized, 'len' => $newsize, 'user' => $user->getId(), 'user_text' => $user->getName(), 'timestamp' => $now, 'content_model' => $content->getModel(), 'content_format' => $serialisation_format, ) ); $revisionId = $revision->insertOn( $dbw ); // Bug 37225: use accessor to get the text as Revision may trim it $content = $revision->getContent(); // sanity; get normalized version if ( $content ) { $newsize = $content->getSize(); } // Update the page record with revision data $this->updateRevisionOn( $dbw, $revision, 0 ); wfRunHooks( 'NewRevisionFromEditComplete', array( $this, $revision, false, $user ) ); // Update recentchanges if ( !( $flags & EDIT_SUPPRESS_RC ) ) { // Mark as patrolled if the user can do so $patrolled = ( $wgUseRCPatrol || $wgUseNPPatrol ) && !count( $this->mTitle->getUserPermissionsErrors( 'autopatrol', $user ) ); // Add RC row to the DB $rc = RecentChange::notifyNew( $now, $this->mTitle, $isminor, $user, $summary, $bot, '', $newsize, $revisionId, $patrolled ); // Log auto-patrolled edits if ( $patrolled ) { PatrolLog::record( $rc, true, $user ); } } $user->incEditCount(); $dbw->commit( __METHOD__ ); // Update links, etc. $this->doEditUpdates( $revision, $user, array( 'created' => true ) ); $hook_args = array( &$this, &$user, $content, $summary, $flags & EDIT_MINOR, null, null, &$flags, $revision ); ContentHandler::runLegacyHooks( 'ArticleInsertComplete', $hook_args ); wfRunHooks( 'PageContentInsertComplete', $hook_args ); } // Do updates right now unless deferral was requested if ( !( $flags & EDIT_DEFER_UPDATES ) ) { DeferredUpdates::doUpdates(); } // Return the new revision (or null) to the caller $status->value['revision'] = $revision; $hook_args = array( &$this, &$user, $content, $summary, $flags & EDIT_MINOR, null, null, &$flags, $revision, &$status, $baseRevId ); ContentHandler::runLegacyHooks( 'ArticleSaveComplete', $hook_args ); wfRunHooks( 'PageContentSaveComplete', $hook_args ); // Promote user to any groups they meet the criteria for $user->addAutopromoteOnceGroups( 'onEdit' ); wfProfileOut( __METHOD__ ); return $status; }
/** * Merge page histories * * @param integer $id The page_id * @param Title $newTitle The new title * @return bool */ private function mergePage($row, Title $newTitle) { $id = $row->page_id; // Construct the WikiPage object we will need later, while the // page_id still exists. Note that this cannot use makeTitleSafe(), // we are deliberately constructing an invalid title. $sourceTitle = Title::makeTitle($row->page_namespace, $row->page_title); $sourceTitle->resetArticleID($id); $wikiPage = new WikiPage($sourceTitle); $wikiPage->loadPageData('fromdbmaster'); $destId = $newTitle->getArticleID(); $this->beginTransaction($this->db, __METHOD__); $this->db->update('revision', ['rev_page' => $destId], ['rev_page' => $id], __METHOD__); $this->db->delete('page', ['page_id' => $id], __METHOD__); $this->commitTransaction($this->db, __METHOD__); /* Call LinksDeletionUpdate to delete outgoing links from the old title, * and update category counts. * * Calling external code with a fake broken Title is a fairly dubious * idea. It's necessary because it's quite a lot of code to duplicate, * but that also makes it fragile since it would be easy for someone to * accidentally introduce an assumption of title validity to the code we * are calling. */ DeferredUpdates::addUpdate(new LinksDeletionUpdate($wikiPage)); DeferredUpdates::doUpdates(); return true; }