/** * Opportunistically enqueue link update jobs given fresh parser output if useful * * @param ParserOutput $parserOutput Current version page output * @since 1.25 */ public function triggerOpportunisticLinksUpdate(ParserOutput $parserOutput) { if (wfReadOnly()) { return; } if (!Hooks::run('OpportunisticLinksUpdate', [$this, $this->mTitle, $parserOutput])) { return; } $config = RequestContext::getMain()->getConfig(); $params = ['isOpportunistic' => true, 'rootJobTimestamp' => $parserOutput->getCacheTime()]; if ($this->mTitle->areRestrictionsCascading()) { // If the page is cascade protecting, the links should really be up-to-date JobQueueGroup::singleton()->lazyPush(RefreshLinksJob::newPrioritized($this->mTitle, $params)); } elseif (!$config->get('MiserMode') && $parserOutput->hasDynamicContent()) { // Assume the output contains "dynamic" time/random based magic words. // Only update pages that expired due to dynamic content and NOT due to edits // to referenced templates/files. When the cache expires due to dynamic content, // page_touched is unchanged. We want to avoid triggering redundant jobs due to // views of pages that were just purged via HTMLCacheUpdateJob. In that case, the // template/file edit already triggered recursive RefreshLinksJob jobs. if ($this->getLinksTimestamp() > $this->getTouched()) { // If a page is uncacheable, do not keep spamming a job for it. // Although it would be de-duplicated, it would still waste I/O. $cache = ObjectCache::getLocalClusterInstance(); $key = $cache->makeKey('dynamic-linksupdate', 'last', $this->getId()); $ttl = max($parserOutput->getCacheExpiry(), 3600); if ($cache->add($key, time(), $ttl)) { JobQueueGroup::singleton()->lazyPush(RefreshLinksJob::newDynamic($this->mTitle, $params)); } } } }
/** * Convert wikitext to HTML * Do not call this function recursively. * * @param string $text Text we want to parse * @param Title $title * @param ParserOptions $options * @param bool $linestart * @param bool $clearState * @param int $revid Number to pass in {{REVISIONID}} * @return ParserOutput A ParserOutput */ public function parse($text, Title $title, ParserOptions $options, $linestart = true, $clearState = true, $revid = null) { /** * First pass--just handle <nowiki> sections, pass the rest off * to internalParse() which does all the real work. */ global $wgShowHostnames; if ($clearState) { // We use U+007F DELETE to construct strip markers, so we have to make // sure that this character does not occur in the input text. $text = strtr($text, "", "?"); $magicScopeVariable = $this->lock(); } $this->startParse($title, $options, self::OT_HTML, $clearState); $this->currentRevisionCache = null; $this->mInputSize = strlen($text); if ($this->mOptions->getEnableLimitReport()) { $this->mOutput->resetParseStartTime(); } $oldRevisionId = $this->mRevisionId; $oldRevisionObject = $this->mRevisionObject; $oldRevisionTimestamp = $this->mRevisionTimestamp; $oldRevisionUser = $this->mRevisionUser; $oldRevisionSize = $this->mRevisionSize; if ($revid !== null) { $this->mRevisionId = $revid; $this->mRevisionObject = null; $this->mRevisionTimestamp = null; $this->mRevisionUser = null; $this->mRevisionSize = null; } Hooks::run('ParserBeforeStrip', array(&$this, &$text, &$this->mStripState)); # No more strip! Hooks::run('ParserAfterStrip', array(&$this, &$text, &$this->mStripState)); $text = $this->internalParse($text); Hooks::run('ParserAfterParse', array(&$this, &$text, &$this->mStripState)); $text = $this->internalParseHalfParsed($text, true, $linestart); /** * A converted title will be provided in the output object if title and * content conversion are enabled, the article text does not contain * a conversion-suppressing double-underscore tag, and no * {{DISPLAYTITLE:...}} is present. DISPLAYTITLE takes precedence over * automatic link conversion. */ if (!($options->getDisableTitleConversion() || isset($this->mDoubleUnderscores['nocontentconvert']) || isset($this->mDoubleUnderscores['notitleconvert']) || $this->mOutput->getDisplayTitle() !== false)) { $convruletitle = $this->getConverterLanguage()->getConvRuleTitle(); if ($convruletitle) { $this->mOutput->setTitleText($convruletitle); } else { $titleText = $this->getConverterLanguage()->convertTitle($title); $this->mOutput->setTitleText($titleText); } } if ($this->mExpensiveFunctionCount > $this->mOptions->getExpensiveParserFunctionLimit()) { $this->limitationWarn('expensive-parserfunction', $this->mExpensiveFunctionCount, $this->mOptions->getExpensiveParserFunctionLimit()); } # Information on include size limits, for the benefit of users who try to skirt them if ($this->mOptions->getEnableLimitReport()) { $max = $this->mOptions->getMaxIncludeSize(); $cpuTime = $this->mOutput->getTimeSinceStart('cpu'); if ($cpuTime !== null) { $this->mOutput->setLimitReportData('limitreport-cputime', sprintf("%.3f", $cpuTime)); } $wallTime = $this->mOutput->getTimeSinceStart('wall'); $this->mOutput->setLimitReportData('limitreport-walltime', sprintf("%.3f", $wallTime)); $this->mOutput->setLimitReportData('limitreport-ppvisitednodes', array($this->mPPNodeCount, $this->mOptions->getMaxPPNodeCount())); $this->mOutput->setLimitReportData('limitreport-ppgeneratednodes', array($this->mGeneratedPPNodeCount, $this->mOptions->getMaxGeneratedPPNodeCount())); $this->mOutput->setLimitReportData('limitreport-postexpandincludesize', array($this->mIncludeSizes['post-expand'], $max)); $this->mOutput->setLimitReportData('limitreport-templateargumentsize', array($this->mIncludeSizes['arg'], $max)); $this->mOutput->setLimitReportData('limitreport-expansiondepth', array($this->mHighestExpansionDepth, $this->mOptions->getMaxPPExpandDepth())); $this->mOutput->setLimitReportData('limitreport-expensivefunctioncount', array($this->mExpensiveFunctionCount, $this->mOptions->getExpensiveParserFunctionLimit())); Hooks::run('ParserLimitReportPrepare', array($this, $this->mOutput)); $limitReport = "NewPP limit report\n"; if ($wgShowHostnames) { $limitReport .= 'Parsed by ' . wfHostname() . "\n"; } $limitReport .= 'Cached time: ' . $this->mOutput->getCacheTime() . "\n"; $limitReport .= 'Cache expiry: ' . $this->mOutput->getCacheExpiry() . "\n"; $limitReport .= 'Dynamic content: ' . ($this->mOutput->hasDynamicContent() ? 'true' : 'false') . "\n"; foreach ($this->mOutput->getLimitReportData() as $key => $value) { if (Hooks::run('ParserLimitReportFormat', array($key, &$value, &$limitReport, false, false))) { $keyMsg = wfMessage($key)->inLanguage('en')->useDatabase(false); $valueMsg = wfMessage(array("{$key}-value-text", "{$key}-value"))->inLanguage('en')->useDatabase(false); if (!$valueMsg->exists()) { $valueMsg = new RawMessage('$1'); } if (!$keyMsg->isDisabled() && !$valueMsg->isDisabled()) { $valueMsg->params($value); $limitReport .= "{$keyMsg->text()}: {$valueMsg->text()}\n"; } } } // Since we're not really outputting HTML, decode the entities and // then re-encode the things that need hiding inside HTML comments. $limitReport = htmlspecialchars_decode($limitReport); Hooks::run('ParserLimitReport', array($this, &$limitReport)); // Sanitize for comment. Note '‐' in the replacement is U+2010, // which looks much like the problematic '-'. $limitReport = str_replace(array('-', '&'), array('‐', '&'), $limitReport); $text .= "\n<!-- \n{$limitReport}-->\n"; // Add on template profiling data $dataByFunc = $this->mProfiler->getFunctionStats(); uasort($dataByFunc, function ($a, $b) { return $a['real'] < $b['real']; // descending order }); $profileReport = "Transclusion expansion time report (%,ms,calls,template)\n"; foreach (array_slice($dataByFunc, 0, 10) as $item) { $profileReport .= sprintf("%6.2f%% %8.3f %6d - %s\n", $item['%real'], $item['real'], $item['calls'], htmlspecialchars($item['name'])); } $text .= "\n<!-- \n{$profileReport}-->\n"; if ($this->mGeneratedPPNodeCount > $this->mOptions->getMaxGeneratedPPNodeCount() / 10) { wfDebugLog('generated-pp-node-count', $this->mGeneratedPPNodeCount . ' ' . $this->mTitle->getPrefixedDBkey()); } } $this->mOutput->setText($text); $this->mRevisionId = $oldRevisionId; $this->mRevisionObject = $oldRevisionObject; $this->mRevisionTimestamp = $oldRevisionTimestamp; $this->mRevisionUser = $oldRevisionUser; $this->mRevisionSize = $oldRevisionSize; $this->mInputSize = false; $this->currentRevisionCache = null; return $this->mOutput; }
/** * Build a value to store in memcached based on the PST content and parser output * * This makes a simple version of WikiPage::prepareContentForEdit() as stash info * * @param Content $pstContent * @param ParserOutput $parserOutput * @param string $timestamp TS_MW * @return array (stash info array, TTL in seconds) or (null, 0) */ protected static function buildStashValue(Content $pstContent, ParserOutput $parserOutput, $timestamp) { // If an item is renewed, mind the cache TTL determined by config and parser functions $since = time() - wfTimestamp(TS_UNIX, $parserOutput->getTimestamp()); $ttl = min($parserOutput->getCacheExpiry() - $since, 5 * 60); if ($ttl > 0 && !$parserOutput->getFlag('vary-revision')) { // Only store what is actually needed $stashInfo = (object) array('pstContent' => $pstContent, 'output' => $parserOutput, 'timestamp' => $timestamp); return array($stashInfo, $ttl); } return array(null, 0); }
/** * @param ParserOutput $parserOutput * @param WikiPage $page * @param ParserOptions $popts * @param string $cacheTime Time when the cache was generated * @param int $revId Revision ID that was parsed */ public function save($parserOutput, $page, $popts, $cacheTime = null, $revId = null) { $expire = $parserOutput->getCacheExpiry(); if ($expire > 0) { $cacheTime = $cacheTime ?: wfTimestampNow(); if (!$revId) { $revision = $page->getRevision(); $revId = $revision ? $revision->getId() : null; } $optionsKey = new CacheTime(); $optionsKey->mUsedOptions = $parserOutput->getUsedOptions(); $optionsKey->updateCacheExpiry($expire); $optionsKey->setCacheTime($cacheTime); $parserOutput->setCacheTime($cacheTime); $optionsKey->setCacheRevisionId($revId); $parserOutput->setCacheRevisionId($revId); $parserOutputKey = $this->getParserOutputKey($page, $popts->optionsHash($optionsKey->mUsedOptions, $page->getTitle())); // Save the timestamp so that we don't have to load the revision row on view $parserOutput->setTimestamp($page->getTimestamp()); $msg = "Saved in parser cache with key {$parserOutputKey}" . " and timestamp {$cacheTime}" . " and revision id {$revId}" . "\n"; $parserOutput->mText .= "\n<!-- {$msg} -->\n"; wfDebug($msg); // Save the parser output $this->mMemc->set($parserOutputKey, $parserOutput, $expire); // ...and its pointer $this->mMemc->set($this->getOptionsKey($page), $optionsKey, $expire); Hooks::run('ParserCacheSaveComplete', array($this, $parserOutput, $page->getTitle(), $popts, $revId)); } else { wfDebug("Parser output was marked as uncacheable and has not been saved.\n"); } }
/** * Convert wikitext to HTML * Do not call this function recursively. * * @param string $text Text we want to parse * @param Title $title * @param ParserOptions $options * @param bool $linestart * @param bool $clearState * @param int $revid Number to pass in {{REVISIONID}} * @return ParserOutput A ParserOutput */ public function parse($text, Title $title, ParserOptions $options, $linestart = true, $clearState = true, $revid = null) { /** * First pass--just handle <nowiki> sections, pass the rest off * to internalParse() which does all the real work. */ global $wgShowHostnames; if ($clearState) { // We use U+007F DELETE to construct strip markers, so we have to make // sure that this character does not occur in the input text. $text = strtr($text, "", "?"); $magicScopeVariable = $this->lock(); } $this->startParse($title, $options, self::OT_HTML, $clearState); $this->currentRevisionCache = null; $this->mInputSize = strlen($text); if ($this->mOptions->getEnableLimitReport()) { $this->mOutput->resetParseStartTime(); } $oldRevisionId = $this->mRevisionId; $oldRevisionObject = $this->mRevisionObject; $oldRevisionTimestamp = $this->mRevisionTimestamp; $oldRevisionUser = $this->mRevisionUser; $oldRevisionSize = $this->mRevisionSize; if ($revid !== null) { $this->mRevisionId = $revid; $this->mRevisionObject = null; $this->mRevisionTimestamp = null; $this->mRevisionUser = null; $this->mRevisionSize = null; } Hooks::run('ParserBeforeStrip', [&$this, &$text, &$this->mStripState]); # No more strip! Hooks::run('ParserAfterStrip', [&$this, &$text, &$this->mStripState]); $text = $this->internalParse($text); Hooks::run('ParserAfterParse', [&$this, &$text, &$this->mStripState]); $text = $this->internalParseHalfParsed($text, true, $linestart); /** * A converted title will be provided in the output object if title and * content conversion are enabled, the article text does not contain * a conversion-suppressing double-underscore tag, and no * {{DISPLAYTITLE:...}} is present. DISPLAYTITLE takes precedence over * automatic link conversion. */ if (!($options->getDisableTitleConversion() || isset($this->mDoubleUnderscores['nocontentconvert']) || isset($this->mDoubleUnderscores['notitleconvert']) || $this->mOutput->getDisplayTitle() !== false)) { $convruletitle = $this->getConverterLanguage()->getConvRuleTitle(); if ($convruletitle) { $this->mOutput->setTitleText($convruletitle); } else { $titleText = $this->getConverterLanguage()->convertTitle($title); $this->mOutput->setTitleText($titleText); } } # Done parsing! Compute runtime adaptive expiry if set $this->mOutput->finalizeAdaptiveCacheExpiry(); # Warn if too many heavyweight parser functions were used if ($this->mExpensiveFunctionCount > $this->mOptions->getExpensiveParserFunctionLimit()) { $this->limitationWarn('expensive-parserfunction', $this->mExpensiveFunctionCount, $this->mOptions->getExpensiveParserFunctionLimit()); } # Information on include size limits, for the benefit of users who try to skirt them if ($this->mOptions->getEnableLimitReport()) { $max = $this->mOptions->getMaxIncludeSize(); $cpuTime = $this->mOutput->getTimeSinceStart('cpu'); if ($cpuTime !== null) { $this->mOutput->setLimitReportData('limitreport-cputime', sprintf("%.3f", $cpuTime)); } $wallTime = $this->mOutput->getTimeSinceStart('wall'); $this->mOutput->setLimitReportData('limitreport-walltime', sprintf("%.3f", $wallTime)); $this->mOutput->setLimitReportData('limitreport-ppvisitednodes', [$this->mPPNodeCount, $this->mOptions->getMaxPPNodeCount()]); $this->mOutput->setLimitReportData('limitreport-ppgeneratednodes', [$this->mGeneratedPPNodeCount, $this->mOptions->getMaxGeneratedPPNodeCount()]); $this->mOutput->setLimitReportData('limitreport-postexpandincludesize', [$this->mIncludeSizes['post-expand'], $max]); $this->mOutput->setLimitReportData('limitreport-templateargumentsize', [$this->mIncludeSizes['arg'], $max]); $this->mOutput->setLimitReportData('limitreport-expansiondepth', [$this->mHighestExpansionDepth, $this->mOptions->getMaxPPExpandDepth()]); $this->mOutput->setLimitReportData('limitreport-expensivefunctioncount', [$this->mExpensiveFunctionCount, $this->mOptions->getExpensiveParserFunctionLimit()]); Hooks::run('ParserLimitReportPrepare', [$this, $this->mOutput]); $limitReport = ''; Hooks::run('ParserLimitReport', [$this, &$limitReport]); if ($limitReport != '') { // Sanitize for comment. Note '‐' in the replacement is U+2010, // which looks much like the problematic '-'. $limitReport = str_replace(['-', '&'], ['‐', '&'], $limitReport); $text .= "\n<!-- \nNewPP limit report\n{$limitReport}-->\n"; } // Add on template profiling data in human/machine readable way $dataByFunc = $this->mProfiler->getFunctionStats(); uasort($dataByFunc, function ($a, $b) { return $a['real'] < $b['real']; // descending order }); $profileReport = []; foreach (array_slice($dataByFunc, 0, 10) as $item) { $profileReport[] = sprintf("%6.2f%% %8.3f %6d %s", $item['%real'], $item['real'], $item['calls'], $item['name']); } $this->mOutput->setLimitReportData('limitreport-timingprofile', $profileReport); // Add other cache related metadata if ($wgShowHostnames) { $this->mOutput->setLimitReportData('cachereport-origin', wfHostname()); } $this->mOutput->setLimitReportData('cachereport-timestamp', $this->mOutput->getCacheTime()); $this->mOutput->setLimitReportData('cachereport-ttl', $this->mOutput->getCacheExpiry()); $this->mOutput->setLimitReportData('cachereport-transientcontent', $this->mOutput->hasDynamicContent()); if ($this->mGeneratedPPNodeCount > $this->mOptions->getMaxGeneratedPPNodeCount() / 10) { wfDebugLog('generated-pp-node-count', $this->mGeneratedPPNodeCount . ' ' . $this->mTitle->getPrefixedDBkey()); } } $this->mOutput->setText($text); $this->mRevisionId = $oldRevisionId; $this->mRevisionObject = $oldRevisionObject; $this->mRevisionTimestamp = $oldRevisionTimestamp; $this->mRevisionUser = $oldRevisionUser; $this->mRevisionSize = $oldRevisionSize; $this->mInputSize = false; $this->currentRevisionCache = null; return $this->mOutput; }
/** * @param $parserOutput ParserOutput * @param $article Page * @param $popts ParserOptions */ public function save(ParserOutput $parserOutput, Page $article, ParserOptions $popts) { wfRunHooks('BeforeParserCacheSave', [$parserOutput, $article]); $expire = $parserOutput->getCacheExpiry(); if ($expire > 0) { $now = wfTimestampNow(); $optionsKey = new CacheTime(); $optionsKey->mUsedOptions = $parserOutput->getUsedOptions(); $optionsKey->updateCacheExpiry($expire); $optionsKey->setCacheTime($now); $parserOutput->setCacheTime($now); $optionsKey->setContainsOldMagic($parserOutput->containsOldMagic()); $parserOutputKey = $this->getParserOutputKey($article, $popts->optionsHash($optionsKey->mUsedOptions, $article->getTitle())); // Save the timestamp so that we don't have to load the revision row on view $parserOutput->setTimestamp($article->getTimestamp()); // Wikia change - begin // @author macbre - BAC-1172 #$info = "Saved in parser cache with key $parserOutputKey and timestamp $now"; global $wgArticleAsJson; if (!$wgArticleAsJson) { $info = "Saved in parser cache with key {$parserOutputKey}"; $parserOutput->mText .= "\n<!-- {$info} -->\n"; } wfDebug("{$info}\n"); // Wikia change - end // Save the parser output $this->mMemc->set($parserOutputKey, $parserOutput, $expire); // ...and its pointer $this->mMemc->set($this->getOptionsKey($article), $optionsKey, $expire); } else { wfDebug("Parser output was marked as uncacheable and has not been saved.\n"); } }
/** * Set template and image versions from parsing a revision * @param Title $title * @param int $revId * @param ParserOutput $rev * @return void */ public static function setRevIncludes(Title $title, $revId, ParserOutput $pOut) { global $wgMemc; $key = self::getCacheKey($title, $revId); # Get the template/file versions used... $versions = array($pOut->getTemplateIds(), $pOut->getFileSearchOptions()); # Save to cache (check cache expiry for dynamic elements)... $data = FlaggedRevs::makeMemcObj($versions); $wgMemc->set($key, $data, $pOut->getCacheExpiry()); }
/** * Build a value to store in memcached based on the PST content and parser output * * This makes a simple version of WikiPage::prepareContentForEdit() as stash info * * @param Content $pstContent Pre-Save transformed content * @param ParserOutput $parserOutput * @param string $timestamp TS_MW * @param User $user * @return array (stash info array, TTL in seconds, info code) or (null, 0, info code) */ private static function buildStashValue(Content $pstContent, ParserOutput $parserOutput, $timestamp, User $user) { // If an item is renewed, mind the cache TTL determined by config and parser functions. // Put an upper limit on the TTL for sanity to avoid extreme template/file staleness. $since = time() - wfTimestamp(TS_UNIX, $parserOutput->getTimestamp()); $ttl = min($parserOutput->getCacheExpiry() - $since, self::MAX_CACHE_TTL); if ($ttl <= 0) { return [null, 0, 'no_ttl']; } // Only store what is actually needed $stashInfo = (object) ['pstContent' => $pstContent, 'output' => $parserOutput, 'timestamp' => $timestamp, 'edits' => $user->getEditCount()]; return [$stashInfo, $ttl, 'ok']; }
/** * @param ParserOutput $parserOutput * @param WikiPage $page * @param ParserOptions $popts * @param string $cacheTime Time when the cache was generated */ public function save($parserOutput, $page, $popts, $cacheTime = null) { $expire = $parserOutput->getCacheExpiry(); if ($expire > 0) { $cacheTime = $cacheTime ?: wfTimestampNow(); $optionsKey = new CacheTime(); $optionsKey->mUsedOptions = $parserOutput->getUsedOptions(); $optionsKey->updateCacheExpiry($expire); $optionsKey->setCacheTime($cacheTime); $parserOutput->setCacheTime($cacheTime); $optionsKey->setContainsOldMagic($parserOutput->containsOldMagic()); $parserOutputKey = $this->getParserOutputKey($page, $popts->optionsHash($optionsKey->mUsedOptions, $page->getTitle())); // Save the timestamp so that we don't have to load the revision row on view $parserOutput->setTimestamp($page->getTimestamp()); $parserOutput->mText .= "\n<!-- Saved in parser cache with key {$parserOutputKey} and timestamp {$cacheTime}\n -->\n"; wfDebug("Saved in parser cache with key {$parserOutputKey} and timestamp {$cacheTime}\n"); // Save the parser output $this->mMemc->set($parserOutputKey, $parserOutput, $expire); // ...and its pointer $this->mMemc->set($this->getOptionsKey($page), $optionsKey, $expire); } else { wfDebug("Parser output was marked as uncacheable and has not been saved.\n"); } }