/**
* Opportunistically enqueue link update jobs given fresh parser output if useful
*
* @param ParserOutput $parserOutput Current version page output
* @since 1.25
*/
public function triggerOpportunisticLinksUpdate(ParserOutput $parserOutput)
{
if (wfReadOnly()) {
return;
}
if (!Hooks::run('OpportunisticLinksUpdate', [$this, $this->mTitle, $parserOutput])) {
return;
}
$config = RequestContext::getMain()->getConfig();
$params = ['isOpportunistic' => true, 'rootJobTimestamp' => $parserOutput->getCacheTime()];
if ($this->mTitle->areRestrictionsCascading()) {
// If the page is cascade protecting, the links should really be up-to-date
JobQueueGroup::singleton()->lazyPush(RefreshLinksJob::newPrioritized($this->mTitle, $params));
} elseif (!$config->get('MiserMode') && $parserOutput->hasDynamicContent()) {
// Assume the output contains "dynamic" time/random based magic words.
// Only update pages that expired due to dynamic content and NOT due to edits
// to referenced templates/files. When the cache expires due to dynamic content,
// page_touched is unchanged. We want to avoid triggering redundant jobs due to
// views of pages that were just purged via HTMLCacheUpdateJob. In that case, the
// template/file edit already triggered recursive RefreshLinksJob jobs.
if ($this->getLinksTimestamp() > $this->getTouched()) {
// If a page is uncacheable, do not keep spamming a job for it.
// Although it would be de-duplicated, it would still waste I/O.
$cache = ObjectCache::getLocalClusterInstance();
$key = $cache->makeKey('dynamic-linksupdate', 'last', $this->getId());
$ttl = max($parserOutput->getCacheExpiry(), 3600);
if ($cache->add($key, time(), $ttl)) {
JobQueueGroup::singleton()->lazyPush(RefreshLinksJob::newDynamic($this->mTitle, $params));
}
}
}
}
/**
* Convert wikitext to HTML
* Do not call this function recursively.
*
* @param string $text Text we want to parse
* @param Title $title
* @param ParserOptions $options
* @param bool $linestart
* @param bool $clearState
* @param int $revid Number to pass in {{REVISIONID}}
* @return ParserOutput A ParserOutput
*/
public function parse($text, Title $title, ParserOptions $options, $linestart = true, $clearState = true, $revid = null)
{
/**
* First pass--just handle <nowiki> sections, pass the rest off
* to internalParse() which does all the real work.
*/
global $wgShowHostnames;
if ($clearState) {
// We use U+007F DELETE to construct strip markers, so we have to make
// sure that this character does not occur in the input text.
$text = strtr($text, "", "?");
$magicScopeVariable = $this->lock();
}
$this->startParse($title, $options, self::OT_HTML, $clearState);
$this->currentRevisionCache = null;
$this->mInputSize = strlen($text);
if ($this->mOptions->getEnableLimitReport()) {
$this->mOutput->resetParseStartTime();
}
$oldRevisionId = $this->mRevisionId;
$oldRevisionObject = $this->mRevisionObject;
$oldRevisionTimestamp = $this->mRevisionTimestamp;
$oldRevisionUser = $this->mRevisionUser;
$oldRevisionSize = $this->mRevisionSize;
if ($revid !== null) {
$this->mRevisionId = $revid;
$this->mRevisionObject = null;
$this->mRevisionTimestamp = null;
$this->mRevisionUser = null;
$this->mRevisionSize = null;
}
Hooks::run('ParserBeforeStrip', array(&$this, &$text, &$this->mStripState));
# No more strip!
Hooks::run('ParserAfterStrip', array(&$this, &$text, &$this->mStripState));
$text = $this->internalParse($text);
Hooks::run('ParserAfterParse', array(&$this, &$text, &$this->mStripState));
$text = $this->internalParseHalfParsed($text, true, $linestart);
/**
* A converted title will be provided in the output object if title and
* content conversion are enabled, the article text does not contain
* a conversion-suppressing double-underscore tag, and no
* {{DISPLAYTITLE:...}} is present. DISPLAYTITLE takes precedence over
* automatic link conversion.
*/
if (!($options->getDisableTitleConversion() || isset($this->mDoubleUnderscores['nocontentconvert']) || isset($this->mDoubleUnderscores['notitleconvert']) || $this->mOutput->getDisplayTitle() !== false)) {
$convruletitle = $this->getConverterLanguage()->getConvRuleTitle();
if ($convruletitle) {
$this->mOutput->setTitleText($convruletitle);
} else {
$titleText = $this->getConverterLanguage()->convertTitle($title);
$this->mOutput->setTitleText($titleText);
}
}
if ($this->mExpensiveFunctionCount > $this->mOptions->getExpensiveParserFunctionLimit()) {
$this->limitationWarn('expensive-parserfunction', $this->mExpensiveFunctionCount, $this->mOptions->getExpensiveParserFunctionLimit());
}
# Information on include size limits, for the benefit of users who try to skirt them
if ($this->mOptions->getEnableLimitReport()) {
$max = $this->mOptions->getMaxIncludeSize();
$cpuTime = $this->mOutput->getTimeSinceStart('cpu');
if ($cpuTime !== null) {
$this->mOutput->setLimitReportData('limitreport-cputime', sprintf("%.3f", $cpuTime));
}
$wallTime = $this->mOutput->getTimeSinceStart('wall');
$this->mOutput->setLimitReportData('limitreport-walltime', sprintf("%.3f", $wallTime));
$this->mOutput->setLimitReportData('limitreport-ppvisitednodes', array($this->mPPNodeCount, $this->mOptions->getMaxPPNodeCount()));
$this->mOutput->setLimitReportData('limitreport-ppgeneratednodes', array($this->mGeneratedPPNodeCount, $this->mOptions->getMaxGeneratedPPNodeCount()));
$this->mOutput->setLimitReportData('limitreport-postexpandincludesize', array($this->mIncludeSizes['post-expand'], $max));
$this->mOutput->setLimitReportData('limitreport-templateargumentsize', array($this->mIncludeSizes['arg'], $max));
$this->mOutput->setLimitReportData('limitreport-expansiondepth', array($this->mHighestExpansionDepth, $this->mOptions->getMaxPPExpandDepth()));
$this->mOutput->setLimitReportData('limitreport-expensivefunctioncount', array($this->mExpensiveFunctionCount, $this->mOptions->getExpensiveParserFunctionLimit()));
Hooks::run('ParserLimitReportPrepare', array($this, $this->mOutput));
$limitReport = "NewPP limit report\n";
if ($wgShowHostnames) {
$limitReport .= 'Parsed by ' . wfHostname() . "\n";
}
$limitReport .= 'Cached time: ' . $this->mOutput->getCacheTime() . "\n";
$limitReport .= 'Cache expiry: ' . $this->mOutput->getCacheExpiry() . "\n";
$limitReport .= 'Dynamic content: ' . ($this->mOutput->hasDynamicContent() ? 'true' : 'false') . "\n";
foreach ($this->mOutput->getLimitReportData() as $key => $value) {
if (Hooks::run('ParserLimitReportFormat', array($key, &$value, &$limitReport, false, false))) {
$keyMsg = wfMessage($key)->inLanguage('en')->useDatabase(false);
$valueMsg = wfMessage(array("{$key}-value-text", "{$key}-value"))->inLanguage('en')->useDatabase(false);
if (!$valueMsg->exists()) {
$valueMsg = new RawMessage('$1');
}
if (!$keyMsg->isDisabled() && !$valueMsg->isDisabled()) {
$valueMsg->params($value);
$limitReport .= "{$keyMsg->text()}: {$valueMsg->text()}\n";
}
}
}
// Since we're not really outputting HTML, decode the entities and
// then re-encode the things that need hiding inside HTML comments.
$limitReport = htmlspecialchars_decode($limitReport);
Hooks::run('ParserLimitReport', array($this, &$limitReport));
// Sanitize for comment. Note '‐' in the replacement is U+2010,
// which looks much like the problematic '-'.
$limitReport = str_replace(array('-', '&'), array('‐', '&'), $limitReport);
$text .= "\n<!-- \n{$limitReport}-->\n";
// Add on template profiling data
$dataByFunc = $this->mProfiler->getFunctionStats();
uasort($dataByFunc, function ($a, $b) {
return $a['real'] < $b['real'];
// descending order
});
$profileReport = "Transclusion expansion time report (%,ms,calls,template)\n";
foreach (array_slice($dataByFunc, 0, 10) as $item) {
$profileReport .= sprintf("%6.2f%% %8.3f %6d - %s\n", $item['%real'], $item['real'], $item['calls'], htmlspecialchars($item['name']));
}
$text .= "\n<!-- \n{$profileReport}-->\n";
if ($this->mGeneratedPPNodeCount > $this->mOptions->getMaxGeneratedPPNodeCount() / 10) {
wfDebugLog('generated-pp-node-count', $this->mGeneratedPPNodeCount . ' ' . $this->mTitle->getPrefixedDBkey());
}
}
$this->mOutput->setText($text);
$this->mRevisionId = $oldRevisionId;
$this->mRevisionObject = $oldRevisionObject;
$this->mRevisionTimestamp = $oldRevisionTimestamp;
$this->mRevisionUser = $oldRevisionUser;
$this->mRevisionSize = $oldRevisionSize;
$this->mInputSize = false;
$this->currentRevisionCache = null;
return $this->mOutput;
}
/**
* Build a value to store in memcached based on the PST content and parser output
*
* This makes a simple version of WikiPage::prepareContentForEdit() as stash info
*
* @param Content $pstContent
* @param ParserOutput $parserOutput
* @param string $timestamp TS_MW
* @return array (stash info array, TTL in seconds) or (null, 0)
*/
protected static function buildStashValue(Content $pstContent, ParserOutput $parserOutput, $timestamp)
{
// If an item is renewed, mind the cache TTL determined by config and parser functions
$since = time() - wfTimestamp(TS_UNIX, $parserOutput->getTimestamp());
$ttl = min($parserOutput->getCacheExpiry() - $since, 5 * 60);
if ($ttl > 0 && !$parserOutput->getFlag('vary-revision')) {
// Only store what is actually needed
$stashInfo = (object) array('pstContent' => $pstContent, 'output' => $parserOutput, 'timestamp' => $timestamp);
return array($stashInfo, $ttl);
}
return array(null, 0);
}
/**
* @param ParserOutput $parserOutput
* @param WikiPage $page
* @param ParserOptions $popts
* @param string $cacheTime Time when the cache was generated
* @param int $revId Revision ID that was parsed
*/
public function save($parserOutput, $page, $popts, $cacheTime = null, $revId = null)
{
$expire = $parserOutput->getCacheExpiry();
if ($expire > 0) {
$cacheTime = $cacheTime ?: wfTimestampNow();
if (!$revId) {
$revision = $page->getRevision();
$revId = $revision ? $revision->getId() : null;
}
$optionsKey = new CacheTime();
$optionsKey->mUsedOptions = $parserOutput->getUsedOptions();
$optionsKey->updateCacheExpiry($expire);
$optionsKey->setCacheTime($cacheTime);
$parserOutput->setCacheTime($cacheTime);
$optionsKey->setCacheRevisionId($revId);
$parserOutput->setCacheRevisionId($revId);
$parserOutputKey = $this->getParserOutputKey($page, $popts->optionsHash($optionsKey->mUsedOptions, $page->getTitle()));
// Save the timestamp so that we don't have to load the revision row on view
$parserOutput->setTimestamp($page->getTimestamp());
$msg = "Saved in parser cache with key {$parserOutputKey}" . " and timestamp {$cacheTime}" . " and revision id {$revId}" . "\n";
$parserOutput->mText .= "\n<!-- {$msg} -->\n";
wfDebug($msg);
// Save the parser output
$this->mMemc->set($parserOutputKey, $parserOutput, $expire);
// ...and its pointer
$this->mMemc->set($this->getOptionsKey($page), $optionsKey, $expire);
Hooks::run('ParserCacheSaveComplete', array($this, $parserOutput, $page->getTitle(), $popts, $revId));
} else {
wfDebug("Parser output was marked as uncacheable and has not been saved.\n");
}
}
/**
* Convert wikitext to HTML
* Do not call this function recursively.
*
* @param string $text Text we want to parse
* @param Title $title
* @param ParserOptions $options
* @param bool $linestart
* @param bool $clearState
* @param int $revid Number to pass in {{REVISIONID}}
* @return ParserOutput A ParserOutput
*/
public function parse($text, Title $title, ParserOptions $options, $linestart = true, $clearState = true, $revid = null)
{
/**
* First pass--just handle <nowiki> sections, pass the rest off
* to internalParse() which does all the real work.
*/
global $wgShowHostnames;
if ($clearState) {
// We use U+007F DELETE to construct strip markers, so we have to make
// sure that this character does not occur in the input text.
$text = strtr($text, "", "?");
$magicScopeVariable = $this->lock();
}
$this->startParse($title, $options, self::OT_HTML, $clearState);
$this->currentRevisionCache = null;
$this->mInputSize = strlen($text);
if ($this->mOptions->getEnableLimitReport()) {
$this->mOutput->resetParseStartTime();
}
$oldRevisionId = $this->mRevisionId;
$oldRevisionObject = $this->mRevisionObject;
$oldRevisionTimestamp = $this->mRevisionTimestamp;
$oldRevisionUser = $this->mRevisionUser;
$oldRevisionSize = $this->mRevisionSize;
if ($revid !== null) {
$this->mRevisionId = $revid;
$this->mRevisionObject = null;
$this->mRevisionTimestamp = null;
$this->mRevisionUser = null;
$this->mRevisionSize = null;
}
Hooks::run('ParserBeforeStrip', [&$this, &$text, &$this->mStripState]);
# No more strip!
Hooks::run('ParserAfterStrip', [&$this, &$text, &$this->mStripState]);
$text = $this->internalParse($text);
Hooks::run('ParserAfterParse', [&$this, &$text, &$this->mStripState]);
$text = $this->internalParseHalfParsed($text, true, $linestart);
/**
* A converted title will be provided in the output object if title and
* content conversion are enabled, the article text does not contain
* a conversion-suppressing double-underscore tag, and no
* {{DISPLAYTITLE:...}} is present. DISPLAYTITLE takes precedence over
* automatic link conversion.
*/
if (!($options->getDisableTitleConversion() || isset($this->mDoubleUnderscores['nocontentconvert']) || isset($this->mDoubleUnderscores['notitleconvert']) || $this->mOutput->getDisplayTitle() !== false)) {
$convruletitle = $this->getConverterLanguage()->getConvRuleTitle();
if ($convruletitle) {
$this->mOutput->setTitleText($convruletitle);
} else {
$titleText = $this->getConverterLanguage()->convertTitle($title);
$this->mOutput->setTitleText($titleText);
}
}
# Done parsing! Compute runtime adaptive expiry if set
$this->mOutput->finalizeAdaptiveCacheExpiry();
# Warn if too many heavyweight parser functions were used
if ($this->mExpensiveFunctionCount > $this->mOptions->getExpensiveParserFunctionLimit()) {
$this->limitationWarn('expensive-parserfunction', $this->mExpensiveFunctionCount, $this->mOptions->getExpensiveParserFunctionLimit());
}
# Information on include size limits, for the benefit of users who try to skirt them
if ($this->mOptions->getEnableLimitReport()) {
$max = $this->mOptions->getMaxIncludeSize();
$cpuTime = $this->mOutput->getTimeSinceStart('cpu');
if ($cpuTime !== null) {
$this->mOutput->setLimitReportData('limitreport-cputime', sprintf("%.3f", $cpuTime));
}
$wallTime = $this->mOutput->getTimeSinceStart('wall');
$this->mOutput->setLimitReportData('limitreport-walltime', sprintf("%.3f", $wallTime));
$this->mOutput->setLimitReportData('limitreport-ppvisitednodes', [$this->mPPNodeCount, $this->mOptions->getMaxPPNodeCount()]);
$this->mOutput->setLimitReportData('limitreport-ppgeneratednodes', [$this->mGeneratedPPNodeCount, $this->mOptions->getMaxGeneratedPPNodeCount()]);
$this->mOutput->setLimitReportData('limitreport-postexpandincludesize', [$this->mIncludeSizes['post-expand'], $max]);
$this->mOutput->setLimitReportData('limitreport-templateargumentsize', [$this->mIncludeSizes['arg'], $max]);
$this->mOutput->setLimitReportData('limitreport-expansiondepth', [$this->mHighestExpansionDepth, $this->mOptions->getMaxPPExpandDepth()]);
$this->mOutput->setLimitReportData('limitreport-expensivefunctioncount', [$this->mExpensiveFunctionCount, $this->mOptions->getExpensiveParserFunctionLimit()]);
Hooks::run('ParserLimitReportPrepare', [$this, $this->mOutput]);
$limitReport = '';
Hooks::run('ParserLimitReport', [$this, &$limitReport]);
if ($limitReport != '') {
// Sanitize for comment. Note '‐' in the replacement is U+2010,
// which looks much like the problematic '-'.
$limitReport = str_replace(['-', '&'], ['‐', '&'], $limitReport);
$text .= "\n<!-- \nNewPP limit report\n{$limitReport}-->\n";
}
// Add on template profiling data in human/machine readable way
$dataByFunc = $this->mProfiler->getFunctionStats();
uasort($dataByFunc, function ($a, $b) {
return $a['real'] < $b['real'];
// descending order
});
$profileReport = [];
foreach (array_slice($dataByFunc, 0, 10) as $item) {
$profileReport[] = sprintf("%6.2f%% %8.3f %6d %s", $item['%real'], $item['real'], $item['calls'], $item['name']);
}
$this->mOutput->setLimitReportData('limitreport-timingprofile', $profileReport);
// Add other cache related metadata
if ($wgShowHostnames) {
$this->mOutput->setLimitReportData('cachereport-origin', wfHostname());
}
$this->mOutput->setLimitReportData('cachereport-timestamp', $this->mOutput->getCacheTime());
$this->mOutput->setLimitReportData('cachereport-ttl', $this->mOutput->getCacheExpiry());
$this->mOutput->setLimitReportData('cachereport-transientcontent', $this->mOutput->hasDynamicContent());
if ($this->mGeneratedPPNodeCount > $this->mOptions->getMaxGeneratedPPNodeCount() / 10) {
wfDebugLog('generated-pp-node-count', $this->mGeneratedPPNodeCount . ' ' . $this->mTitle->getPrefixedDBkey());
}
}
$this->mOutput->setText($text);
$this->mRevisionId = $oldRevisionId;
$this->mRevisionObject = $oldRevisionObject;
$this->mRevisionTimestamp = $oldRevisionTimestamp;
$this->mRevisionUser = $oldRevisionUser;
$this->mRevisionSize = $oldRevisionSize;
$this->mInputSize = false;
$this->currentRevisionCache = null;
return $this->mOutput;
}
/**
* @param $parserOutput ParserOutput
* @param $article Page
* @param $popts ParserOptions
*/
public function save(ParserOutput $parserOutput, Page $article, ParserOptions $popts)
{
wfRunHooks('BeforeParserCacheSave', [$parserOutput, $article]);
$expire = $parserOutput->getCacheExpiry();
if ($expire > 0) {
$now = wfTimestampNow();
$optionsKey = new CacheTime();
$optionsKey->mUsedOptions = $parserOutput->getUsedOptions();
$optionsKey->updateCacheExpiry($expire);
$optionsKey->setCacheTime($now);
$parserOutput->setCacheTime($now);
$optionsKey->setContainsOldMagic($parserOutput->containsOldMagic());
$parserOutputKey = $this->getParserOutputKey($article, $popts->optionsHash($optionsKey->mUsedOptions, $article->getTitle()));
// Save the timestamp so that we don't have to load the revision row on view
$parserOutput->setTimestamp($article->getTimestamp());
// Wikia change - begin
// @author macbre - BAC-1172
#$info = "Saved in parser cache with key $parserOutputKey and timestamp $now";
global $wgArticleAsJson;
if (!$wgArticleAsJson) {
$info = "Saved in parser cache with key {$parserOutputKey}";
$parserOutput->mText .= "\n<!-- {$info} -->\n";
}
wfDebug("{$info}\n");
// Wikia change - end
// Save the parser output
$this->mMemc->set($parserOutputKey, $parserOutput, $expire);
// ...and its pointer
$this->mMemc->set($this->getOptionsKey($article), $optionsKey, $expire);
} else {
wfDebug("Parser output was marked as uncacheable and has not been saved.\n");
}
}
/**
* Set template and image versions from parsing a revision
* @param Title $title
* @param int $revId
* @param ParserOutput $rev
* @return void
*/
public static function setRevIncludes(Title $title, $revId, ParserOutput $pOut)
{
global $wgMemc;
$key = self::getCacheKey($title, $revId);
# Get the template/file versions used...
$versions = array($pOut->getTemplateIds(), $pOut->getFileSearchOptions());
# Save to cache (check cache expiry for dynamic elements)...
$data = FlaggedRevs::makeMemcObj($versions);
$wgMemc->set($key, $data, $pOut->getCacheExpiry());
}
/**
* Build a value to store in memcached based on the PST content and parser output
*
* This makes a simple version of WikiPage::prepareContentForEdit() as stash info
*
* @param Content $pstContent Pre-Save transformed content
* @param ParserOutput $parserOutput
* @param string $timestamp TS_MW
* @param User $user
* @return array (stash info array, TTL in seconds, info code) or (null, 0, info code)
*/
private static function buildStashValue(Content $pstContent, ParserOutput $parserOutput, $timestamp, User $user)
{
// If an item is renewed, mind the cache TTL determined by config and parser functions.
// Put an upper limit on the TTL for sanity to avoid extreme template/file staleness.
$since = time() - wfTimestamp(TS_UNIX, $parserOutput->getTimestamp());
$ttl = min($parserOutput->getCacheExpiry() - $since, self::MAX_CACHE_TTL);
if ($ttl <= 0) {
return [null, 0, 'no_ttl'];
}
// Only store what is actually needed
$stashInfo = (object) ['pstContent' => $pstContent, 'output' => $parserOutput, 'timestamp' => $timestamp, 'edits' => $user->getEditCount()];
return [$stashInfo, $ttl, 'ok'];
}
/**
* @param ParserOutput $parserOutput
* @param WikiPage $page
* @param ParserOptions $popts
* @param string $cacheTime Time when the cache was generated
*/
public function save($parserOutput, $page, $popts, $cacheTime = null)
{
$expire = $parserOutput->getCacheExpiry();
if ($expire > 0) {
$cacheTime = $cacheTime ?: wfTimestampNow();
$optionsKey = new CacheTime();
$optionsKey->mUsedOptions = $parserOutput->getUsedOptions();
$optionsKey->updateCacheExpiry($expire);
$optionsKey->setCacheTime($cacheTime);
$parserOutput->setCacheTime($cacheTime);
$optionsKey->setContainsOldMagic($parserOutput->containsOldMagic());
$parserOutputKey = $this->getParserOutputKey($page, $popts->optionsHash($optionsKey->mUsedOptions, $page->getTitle()));
// Save the timestamp so that we don't have to load the revision row on view
$parserOutput->setTimestamp($page->getTimestamp());
$parserOutput->mText .= "\n<!-- Saved in parser cache with key {$parserOutputKey} and timestamp {$cacheTime}\n -->\n";
wfDebug("Saved in parser cache with key {$parserOutputKey} and timestamp {$cacheTime}\n");
// Save the parser output
$this->mMemc->set($parserOutputKey, $parserOutput, $expire);
// ...and its pointer
$this->mMemc->set($this->getOptionsKey($page), $optionsKey, $expire);
} else {
wfDebug("Parser output was marked as uncacheable and has not been saved.\n");
}
}
