/** * @param array $params * @param Config $mainConfig * @return array */ public static function applyDefaultParameters(array $params, Config $mainConfig) { $logger = LoggerFactory::getInstance('Mime'); $params += ['typeFile' => $mainConfig->get('MimeTypeFile'), 'infoFile' => $mainConfig->get('MimeInfoFile'), 'xmlTypes' => $mainConfig->get('XMLMimeTypes'), 'guessCallback' => function ($mimeAnalyzer, &$head, &$tail, $file, &$mime) use($logger) { // Also test DjVu $deja = new DjVuImage($file); if ($deja->isValid()) { $logger->info(__METHOD__ . ": detected {$file} as image/vnd.djvu\n"); $mime = 'image/vnd.djvu'; return; } // Some strings by reference for performance - assuming well-behaved hooks Hooks::run('MimeMagicGuessFromContent', [$mimeAnalyzer, &$head, &$tail, $file, &$mime]); }, 'extCallback' => function ($mimeAnalyzer, $ext, &$mime) { // Media handling extensions can improve the MIME detected Hooks::run('MimeMagicImproveFromExtension', [$mimeAnalyzer, $ext, &$mime]); }, 'initCallback' => function ($mimeAnalyzer) { // Allow media handling extensions adding MIME-types and MIME-info Hooks::run('MimeMagicInit', [$mimeAnalyzer]); }, 'logger' => $logger]; if ($params['infoFile'] === 'includes/mime.info') { $params['infoFile'] = __DIR__ . "/libs/mime/mime.info"; } if ($params['typeFile'] === 'includes/mime.types') { $params['typeFile'] = __DIR__ . "/libs/mime/mime.types"; } $detectorCmd = $mainConfig->get('MimeDetectorCommand'); if ($detectorCmd) { $params['detectCallback'] = function ($file) use($detectorCmd) { return wfShellExec("{$detectorCmd} " . wfEscapeShellArg($file)); }; } return $params; }
public function execute() { global $wgCommandLineMode; if ($this->hasOption('procs')) { $procs = intval($this->getOption('procs')); if ($procs < 1 || $procs > 1000) { $this->error("Invalid argument to --procs", true); } elseif ($procs != 1) { $fc = new ForkController($procs); if ($fc->start() != 'child') { exit(0); } } } $outputJSON = $this->getOption('result') === 'json'; // Enable DBO_TRX for atomicity; JobRunner manages transactions // and works well in web server mode already (@TODO: this is a hack) $wgCommandLineMode = false; $runner = new JobRunner(LoggerFactory::getInstance('runJobs')); if (!$outputJSON) { $runner->setDebugHandler(array($this, 'debugInternal')); } $response = $runner->run(array('type' => $this->getOption('type', false), 'maxJobs' => $this->getOption('maxjobs', false), 'maxTime' => $this->getOption('maxtime', false), 'throttle' => $this->hasOption('nothrottle') ? false : true)); if ($outputJSON) { $this->output(FormatJson::encode($response, true)); } $wgCommandLineMode = true; }
/** * @param LoggerInterface $logger */ public function __construct(LoggerInterface $logger = null) { if ($logger === null) { $logger = LoggerFactory::getInstance('runJobs'); } $this->setLogger($logger); }
protected static function getMonthViews(Title $title) { global $wgMemc; $key = wfMemcKey('pvi', 'month', md5($title->getPrefixedText())); $data = $wgMemc->get($key); if ($data) { return $data; } $today = date('Ymd'); $lastMonth = date('Ymd', time() - 60 * 60 * 24 * 30); $url = self::buildApiUrl($title, $lastMonth, $today); $req = MWHttpRequest::factory($url, ['timeout' => 10], __METHOD__); $status = $req->execute(); if (!$status->isOK()) { LoggerFactory::getInstance('PageViewInfo')->error("Failed fetching {$url}: {$status->getWikiText()}", ['url' => $url, 'title' => $title->getPrefixedText()]); return false; } $data = FormatJson::decode($req->getContent(), true); // Add our start/end periods $data['start'] = $lastMonth; $data['end'] = $today; // Cache for an hour $wgMemc->set($key, $data, 60 * 60); return $data; }
public function execute() { if (wfReadOnly()) { $this->error("Unable to run jobs; the wiki is in read-only mode.", 1); // die } if ($this->hasOption('procs')) { $procs = intval($this->getOption('procs')); if ($procs < 1 || $procs > 1000) { $this->error("Invalid argument to --procs", true); } elseif ($procs != 1) { $fc = new ForkController($procs); if ($fc->start() != 'child') { exit(0); } } } $json = $this->getOption('result') === 'json'; $runner = new JobRunner(LoggerFactory::getInstance('runJobs')); if (!$json) { $runner->setDebugHandler(array($this, 'debugInternal')); } $response = $runner->run(array('type' => $this->getOption('type', false), 'maxJobs' => $this->getOption('maxjobs', false), 'maxTime' => $this->getOption('maxtime', false), 'throttle' => $this->hasOption('nothrottle') ? false : true)); if ($json) { $this->output(FormatJson::encode($response, true)); } }
/** * @param array $params Possible keys: * - redisConfig : An array of parameters to RedisConnectionPool::__construct(). * - redisServers : Array of server entries, the first being the primary and the * others being fallback servers. Each entry is either a hostname/port * combination or the absolute path of a UNIX socket. * If a hostname is specified but no port, the standard port number * 6379 will be used. Required. */ public function __construct(array $params) { parent::__construct($params); $this->servers = isset($params['redisServers']) ? $params['redisServers'] : [$params['redisServer']]; // b/c $params['redisConfig']['serializer'] = 'none'; $this->redisPool = RedisConnectionPool::singleton($params['redisConfig']); $this->logger = \MediaWiki\Logger\LoggerFactory::getInstance('redis'); }
/** * Construct a factory based on a configuration array (typically from $wgLBFactoryConf) * @param array $conf */ public function __construct(array $conf) { if (isset($conf['readOnlyReason']) && is_string($conf['readOnlyReason'])) { $this->readOnlyReason = $conf['readOnlyReason']; } $this->chronProt = $this->newChronologyProtector(); $this->trxProfiler = Profiler::instance()->getTransactionProfiler(); $this->logger = LoggerFactory::getInstance('DBTransaction'); }
public function execute($par = '') { $this->getOutput()->disable(); if (wfReadOnly()) { // HTTP 423 Locked HttpStatus::header(423); print 'Wiki is in read-only mode'; return; } elseif (!$this->getRequest()->wasPosted()) { HttpStatus::header(400); print 'Request must be POSTed'; return; } $optional = array('maxjobs' => 0, 'maxtime' => 30, 'type' => false, 'async' => true); $required = array_flip(array('title', 'tasks', 'signature', 'sigexpiry')); $params = array_intersect_key($this->getRequest()->getValues(), $required + $optional); $missing = array_diff_key($required, $params); if (count($missing)) { HttpStatus::header(400); print 'Missing parameters: ' . implode(', ', array_keys($missing)); return; } $squery = $params; unset($squery['signature']); $correctSignature = self::getQuerySignature($squery, $this->getConfig()->get('SecretKey')); $providedSignature = $params['signature']; $verified = is_string($providedSignature) && hash_equals($correctSignature, $providedSignature); if (!$verified || $params['sigexpiry'] < time()) { HttpStatus::header(400); print 'Invalid or stale signature provided'; return; } // Apply any default parameter values $params += $optional; if ($params['async']) { // Client will usually disconnect before checking the response, // but it needs to know when it is safe to disconnect. Until this // reaches ignore_user_abort(), it is not safe as the jobs won't run. ignore_user_abort(true); // jobs may take a bit of time // HTTP 202 Accepted HttpStatus::header(202); ob_flush(); flush(); // Once the client receives this response, it can disconnect } // Do all of the specified tasks... if (in_array('jobs', explode('|', $params['tasks']))) { $runner = new JobRunner(LoggerFactory::getInstance('runJobs')); $response = $runner->run(array('type' => $params['type'], 'maxJobs' => $params['maxjobs'] ? $params['maxjobs'] : 1, 'maxTime' => $params['maxtime'] ? $params['maxjobs'] : 30)); if (!$params['async']) { print FormatJson::encode($response, true); } } }
/** * @param array $lbConf Config for LBFactory::__construct() * @param Config $mainConfig Main config object from MediaWikiServices * @return array */ public static function applyDefaultConfig(array $lbConf, Config $mainConfig) { global $wgCommandLineMode; $lbConf += ['localDomain' => new DatabaseDomain($mainConfig->get('DBname'), null, $mainConfig->get('DBprefix')), 'profiler' => Profiler::instance(), 'trxProfiler' => Profiler::instance()->getTransactionProfiler(), 'replLogger' => LoggerFactory::getInstance('DBReplication'), 'queryLogger' => LoggerFactory::getInstance('DBQuery'), 'connLogger' => LoggerFactory::getInstance('DBConnection'), 'perfLogger' => LoggerFactory::getInstance('DBPerformance'), 'errorLogger' => [MWExceptionHandler::class, 'logException'], 'cliMode' => $wgCommandLineMode, 'hostname' => wfHostname(), 'readOnlyReason' => wfConfiguredReadOnlyReason()]; if ($lbConf['class'] === 'LBFactorySimple') { if (isset($lbConf['servers'])) { // Server array is already explicitly configured; leave alone } elseif (is_array($mainConfig->get('DBservers'))) { foreach ($mainConfig->get('DBservers') as $i => $server) { if ($server['type'] === 'sqlite') { $server += ['dbDirectory' => $mainConfig->get('SQLiteDataDir')]; } elseif ($server['type'] === 'postgres') { $server += ['port' => $mainConfig->get('DBport')]; } $lbConf['servers'][$i] = $server + ['schema' => $mainConfig->get('DBmwschema'), 'tablePrefix' => $mainConfig->get('DBprefix'), 'flags' => DBO_DEFAULT, 'sqlMode' => $mainConfig->get('SQLMode'), 'utf8Mode' => $mainConfig->get('DBmysql5')]; } } else { $flags = DBO_DEFAULT; $flags |= $mainConfig->get('DebugDumpSql') ? DBO_DEBUG : 0; $flags |= $mainConfig->get('DBssl') ? DBO_SSL : 0; $flags |= $mainConfig->get('DBcompress') ? DBO_COMPRESS : 0; $server = ['host' => $mainConfig->get('DBserver'), 'user' => $mainConfig->get('DBuser'), 'password' => $mainConfig->get('DBpassword'), 'dbname' => $mainConfig->get('DBname'), 'schema' => $mainConfig->get('DBmwschema'), 'tablePrefix' => $mainConfig->get('DBprefix'), 'type' => $mainConfig->get('DBtype'), 'load' => 1, 'flags' => $flags, 'sqlMode' => $mainConfig->get('SQLMode'), 'utf8Mode' => $mainConfig->get('DBmysql5')]; if ($server['type'] === 'sqlite') { $server['dbDirectory'] = $mainConfig->get('SQLiteDataDir'); } elseif ($server['type'] === 'postgres') { $server['port'] = $mainConfig->get('DBport'); } $lbConf['servers'] = [$server]; } if (!isset($lbConf['externalClusters'])) { $lbConf['externalClusters'] = $mainConfig->get('ExternalServers'); } } elseif ($lbConf['class'] === 'LBFactoryMulti') { if (isset($lbConf['serverTemplate'])) { $lbConf['serverTemplate']['schema'] = $mainConfig->get('DBmwschema'); $lbConf['serverTemplate']['sqlMode'] = $mainConfig->get('SQLMode'); $lbConf['serverTemplate']['utf8Mode'] = $mainConfig->get('DBmysql5'); } } // Use APC/memcached style caching, but avoids loops with CACHE_DB (T141804) $sCache = MediaWikiServices::getInstance()->getLocalServerObjectCache(); if ($sCache->getQoS($sCache::ATTR_EMULATION) > $sCache::QOS_EMULATION_SQL) { $lbConf['srvCache'] = $sCache; } $cCache = ObjectCache::getLocalClusterInstance(); if ($cCache->getQoS($cCache::ATTR_EMULATION) > $cCache::QOS_EMULATION_SQL) { $lbConf['memCache'] = $cCache; } $wCache = MediaWikiServices::getInstance()->getMainWANObjectCache(); if ($wCache->getQoS($wCache::ATTR_EMULATION) > $wCache::QOS_EMULATION_SQL) { $lbConf['wanCache'] = $wCache; } return $lbConf; }
/** * Logs a content-security-policy violation report from web browser. */ public function execute() { $reportOnly = $this->getParameter('reportonly'); $logname = $reportOnly ? 'csp-report-only' : 'csp'; $this->log = LoggerFactory::getInstance($logname); $userAgent = $this->getRequest()->getHeader('user-agent'); $this->verifyPostBodyOk(); $report = $this->getReport(); $flags = $this->getFlags($report); $warningText = $this->generateLogLine($flags, $report); $this->logReport($flags, $warningText, ['csp-report' => $report, 'method' => __METHOD__, 'user' => $this->getUser()->getName(), 'user-agent' => $userAgent, 'source' => $this->getParameter('source')]); $this->getResult()->addValue(null, $this->getModuleName(), 'success'); }
private function doLog() { $logger = LoggerFactory::getInstance('HttpError'); $content = $this->content; if ($content instanceof Message) { $content = $content->text(); } $context = ['file' => $this->getFile(), 'line' => $this->getLine(), 'http_code' => $this->httpCode]; $logMsg = "{$content} ({http_code}) from {file}:{line}"; if ($this->getStatusCode() < 500) { $logger->info($logMsg, $context); } else { $logger->error($logMsg, $context); } }
protected function doJob() { // Reload pages from pageIds to throw into the updater $pageData = array(); foreach ($this->params['pageDBKeys'] as $pageDBKey) { $title = Title::newFromDBKey($pageDBKey); // Skip any titles with broken keys. We can't do anything with them. if (!$title) { LoggerFactory::getInstance('CirrusSearch')->warning("Skipping invalid DBKey: {pageDBKey}", array('pageDBKey' => $pageDBKey)); continue; } $pageData[] = WikiPage::factory($title); } // Now invoke the updater! $updater = $this->createUpdater(); $count = $updater->updatePages($pageData, null, null, $this->params['updateFlags']); return $count >= 0; }
/** * @param array $conditions An array of arrays describing throttling conditions. * Defaults to $wgPasswordAttemptThrottle. See documentation of that variable for format. * @param array $params Parameters (all optional): * - type: throttle type, used as a namespace for counters, * - cache: a BagOStuff object where throttle counters are stored. * - warningLimit: the log level will be raised to warning when rejecting an attempt after * no less than this many failures. */ public function __construct(array $conditions = null, array $params = []) { $invalidParams = array_diff_key($params, array_fill_keys(['type', 'cache', 'warningLimit'], true)); if ($invalidParams) { throw new \InvalidArgumentException('unrecognized parameters: ' . implode(', ', array_keys($invalidParams))); } if ($conditions === null) { $config = \ConfigFactory::getDefaultInstance()->makeConfig('main'); $conditions = $config->get('PasswordAttemptThrottle'); $params += ['type' => 'password', 'cache' => \ObjectCache::getLocalClusterInstance(), 'warningLimit' => 50]; } else { $params += ['type' => 'custom', 'cache' => \ObjectCache::getLocalClusterInstance(), 'warningLimit' => INF]; } $this->type = $params['type']; $this->conditions = static::normalizeThrottleConditions($conditions); $this->cache = $params['cache']; $this->warningLimit = $params['warningLimit']; $this->setLogger(LoggerFactory::getInstance('throttler')); }
/** * Pick the best near match if possible. * * @return Title|null title if there is a near match and null otherwise */ public function pickBest() { if (!$this->titles) { return null; } if (!$this->term) { return null; } if (count($this->titles) === 1) { if (isset($this->titles[0]['titleMatch'])) { return $this->titles[0]['titleMatch']; } if (isset($this->titles[0]['redirectMatches'][0])) { return $this->titles[0]['redirectMatches'][0]; } LoggerFactory::getInstance('CirrusSearch')->info('NearMatchPicker built with busted matches. Assuming no near match'); return null; } $transformers = array(function ($term) { return $term; }, array($this->language, 'lc'), array($this->language, 'ucwords')); foreach ($transformers as $transformer) { $transformedTerm = call_user_func($transformer, $this->term); $found = null; foreach ($this->titles as $title) { $match = $this->checkAllMatches($transformer, $transformedTerm, $title); if ($match) { if (!$found) { $found = $match; } else { // Found more than one result so we try another transformer $found = null; break; } } } if ($found) { return $found; } } // Didn't find anything return null; }
public function execute() { global $wgCommandLineMode; if ($this->hasOption('procs')) { $procs = intval($this->getOption('procs')); if ($procs < 1 || $procs > 1000) { $this->error("Invalid argument to --procs", true); } elseif ($procs != 1) { $fc = new ForkController($procs); if ($fc->start() != 'child') { exit(0); } } } $outputJSON = $this->getOption('result') === 'json'; $wait = $this->hasOption('wait'); // Enable DBO_TRX for atomicity; JobRunner manages transactions // and works well in web server mode already (@TODO: this is a hack) $wgCommandLineMode = false; $runner = new JobRunner(LoggerFactory::getInstance('runJobs')); if (!$outputJSON) { $runner->setDebugHandler([$this, 'debugInternal']); } $type = $this->getOption('type', false); $maxJobs = $this->getOption('maxjobs', false); $maxTime = $this->getOption('maxtime', false); $throttle = !$this->hasOption('nothrottle'); while (true) { $response = $runner->run(['type' => $type, 'maxJobs' => $maxJobs, 'maxTime' => $maxTime, 'throttle' => $throttle]); if ($outputJSON) { $this->output(FormatJson::encode($response, true)); } if (!$wait || $response['reached'] === 'time-limit' || $response['reached'] === 'job-limit' || $response['reached'] === 'memory-limit') { break; } if ($maxJobs !== false) { $maxJobs -= count($response['jobs']); } sleep(1); } $wgCommandLineMode = true; }
/** * Get text to index from a ParserOutput assuming the content was wikitext. * * @param ParserOutput $parserOutput The parsed wikitext's parser output * @return array who's first entry is text and second is opening text, and third is an * array of auxiliary text */ private function formatWikitext(ParserOutput $parserOutput) { global $wgCirrusSearchBoostOpening; $parserOutput->setEditSectionTokens(false); $parserOutput->setTOCEnabled(false); $text = $parserOutput->getText(); $opening = null; switch ($wgCirrusSearchBoostOpening) { case 'first_heading': $opening = $this->extractHeadingBeforeFirstHeading($text); case 'none': break; default: LoggerFactory::getInstance('CirrusSearch')->warning("Invalid value for \$wgCirrusSearchBoostOpening: {wgCirrusSearchBoostOpening}", array('wgCirrusSearchBoostOpening' => $wgCirrusSearchBoostOpening)); } // Add extra spacing around break tags so text crammed together like<br>this doesn't make one word. $text = str_replace('<br', "\n<br", $text); $formatter = new HtmlFormatter($text); // Strip elements from the page that we never want in the search text. $formatter->remove($this->excludedElementSelectors); $filterResult = $formatter->filterContent(); if ($filterResult === null) { // We're running against Mediawiki < 1.24wm10 which won't support auxiliary text // because it can't extract it using the HtmlFormatter. We'll just set text to // all the text. $allText = trim(Sanitizer::stripAllTags($formatter->getText())); $auxiliary = array(); } else { // Strip elements from the page that are auxiliary text. These will still be // searched but matches will be ranked lower and non-auxiliary matches will be // prefered in highlighting. $formatter->remove($this->auxiliaryElementSelectors); $auxiliaryElements = $formatter->filterContent(); $allText = trim(Sanitizer::stripAllTags($formatter->getText())); $auxiliary = array(); foreach ($auxiliaryElements as $auxiliaryElement) { $auxiliary[] = trim(Sanitizer::stripAllTags($formatter->getText($auxiliaryElement))); } } return array($allText, $opening, $auxiliary); }
/** * Attempt to load a precomputed document tree for some given wikitext * from the cache. * * @param string $text * @param int $flags * @return PPNode_Hash_Tree|bool */ protected function cacheGetTree($text, $flags) { $config = RequestContext::getMain()->getConfig(); $length = strlen($text); $threshold = $config->get('PreprocessorCacheThreshold'); if ($threshold === false || $length < $threshold || $length > 1000000.0) { return false; } $cache = ObjectCache::getInstance($config->get('MainCacheType')); $key = wfMemcKey(defined('static::CACHE_PREFIX') ? static::CACHE_PREFIX : get_called_class(), md5($text), $flags); $value = $cache->get($key); if (!$value) { return false; } $version = intval(substr($value, 0, 8)); if ($version !== static::CACHE_VERSION) { return false; } LoggerFactory::getInstance('Preprocessor')->info("Loaded preprocessor output from cache (key: {$key})"); return substr($value, 8); }
/** * Perform an HTTP request * * @param string $method HTTP method. Usually GET/POST * @param string $url Full URL to act on. If protocol-relative, will be expanded to an http:// URL * @param array $options Options to pass to MWHttpRequest object. * Possible keys for the array: * - timeout Timeout length in seconds * - connectTimeout Timeout for connection, in seconds (curl only) * - postData An array of key-value pairs or a url-encoded form data * - proxy The proxy to use. * Otherwise it will use $wgHTTPProxy (if set) * Otherwise it will use the environment variable "http_proxy" (if set) * - noProxy Don't use any proxy at all. Takes precedence over proxy value(s). * - sslVerifyHost Verify hostname against certificate * - sslVerifyCert Verify SSL certificate * - caInfo Provide CA information * - maxRedirects Maximum number of redirects to follow (defaults to 5) * - followRedirects Whether to follow redirects (defaults to false). * Note: this should only be used when the target URL is trusted, * to avoid attacks on intranet services accessible by HTTP. * - userAgent A user agent, if you want to override the default * MediaWiki/$wgVersion * @param string $caller The method making this request, for profiling * @return string|bool (bool)false on failure or a string on success */ public static function request($method, $url, $options = array(), $caller = __METHOD__) { wfDebug("HTTP: {$method}: {$url}\n"); $options['method'] = strtoupper($method); if (!isset($options['timeout'])) { $options['timeout'] = 'default'; } if (!isset($options['connectTimeout'])) { $options['connectTimeout'] = 'default'; } $req = MWHttpRequest::factory($url, $options, $caller); $status = $req->execute(); if ($status->isOK()) { return $req->getContent(); } else { $errors = $status->getErrorsByType('error'); $logger = LoggerFactory::getInstance('http'); $logger->warning($status->getWikiText(), array('caller' => $caller)); return false; } }
public static function onBeforePageDisplay(OutputPage &$out, Skin &$skin) { // Enable only if the user has turned it on in Beta Preferences, or BetaFeatures is not installed. // Will only be loaded if PageImages & TextExtracts extensions are installed. $registry = ExtensionRegistry::getInstance(); if (!$registry->isLoaded('TextExtracts') || !class_exists('ApiQueryPageImages')) { $logger = LoggerFactory::getInstance('popups'); $logger->error('Popups requires the PageImages and TextExtracts extensions.'); return true; } if (self::getConfig()->get('PopupsBetaFeature') === true) { if (!class_exists('BetaFeatures')) { $logger = LoggerFactory::getInstance('popups'); $logger->error('PopupsMode cannot be used as a beta feature unless ' . 'the BetaFeatures extension is present.'); return true; } if (!BetaFeatures::isFeatureEnabled($skin->getUser(), 'popups')) { return true; } } $out->addModules(array('ext.popups', 'schema.Popups')); return true; }
public function execute() { if ($this->hasOption('procs')) { $procs = intval($this->getOption('procs')); if ($procs < 1 || $procs > 1000) { $this->error("Invalid argument to --procs", true); } elseif ($procs != 1) { $fc = new ForkController($procs); if ($fc->start() != 'child') { exit(0); } } } $outputJSON = $this->getOption('result') === 'json'; $wait = $this->hasOption('wait'); $runner = new JobRunner(LoggerFactory::getInstance('runJobs')); if (!$outputJSON) { $runner->setDebugHandler([$this, 'debugInternal']); } $type = $this->getOption('type', false); $maxJobs = $this->getOption('maxjobs', false); $maxTime = $this->getOption('maxtime', false); $throttle = !$this->hasOption('nothrottle'); while (true) { $response = $runner->run(['type' => $type, 'maxJobs' => $maxJobs, 'maxTime' => $maxTime, 'throttle' => $throttle]); if ($outputJSON) { $this->output(FormatJson::encode($response, true)); } if (!$wait || $response['reached'] === 'time-limit' || $response['reached'] === 'job-limit' || $response['reached'] === 'memory-limit') { break; } if ($maxJobs !== false) { $maxJobs -= count($response['jobs']); } sleep(1); } }
/** * @param array $options * - config: Config to fetch configuration from. Defaults to the default 'main' config. * - logger: LoggerInterface to use for logging. Defaults to the 'session' channel. * - store: BagOStuff to store session data in. */ public function __construct($options = array()) { if (isset($options['config'])) { $this->config = $options['config']; if (!$this->config instanceof Config) { throw new \InvalidArgumentException('$options[\'config\'] must be an instance of Config'); } } else { $this->config = \ConfigFactory::getDefaultInstance()->makeConfig('main'); } if (isset($options['logger'])) { if (!$options['logger'] instanceof LoggerInterface) { throw new \InvalidArgumentException('$options[\'logger\'] must be an instance of LoggerInterface'); } $this->setLogger($options['logger']); } else { $this->setLogger(\MediaWiki\Logger\LoggerFactory::getInstance('session')); } if (isset($options['store'])) { if (!$options['store'] instanceof BagOStuff) { throw new \InvalidArgumentException('$options[\'store\'] must be an instance of BagOStuff'); } $this->store = $options['store']; } else { $this->store = \ObjectCache::getInstance($this->config->get('SessionCacheType')); $this->store->setLogger($this->logger); } register_shutdown_function(array($this, 'shutdown')); }
/** * Get a ResourceLoader object associated with this OutputPage * * @return ResourceLoader */ public function getResourceLoader() { if (is_null($this->mResourceLoader)) { $this->mResourceLoader = new ResourceLoader($this->getConfig(), LoggerFactory::getInstance('resourceloader')); } return $this->mResourceLoader; }
/** * Run jobs of the specified number/type for the specified time * * The response map has a 'job' field that lists status of each job, including: * - type : the job type * - status : ok/failed * - error : any error message string * - time : the job run time in ms * The response map also has: * - backoffs : the (job type => seconds) map of backoff times * - elapsed : the total time spent running tasks in ms * - reached : the reason the script finished, one of (none-ready, job-limit, time-limit) * * This method outputs status information only if a debug handler was set. * Any exceptions are caught and logged, but are not reported as output. * * @param array $options Map of parameters: * - type : the job type (or false for the default types) * - maxJobs : maximum number of jobs to run * - maxTime : maximum time in seconds before stopping * - throttle : whether to respect job backoff configuration * @return array Summary response that can easily be JSON serialized */ public function run(array $options) { global $wgJobClasses, $wgTrxProfilerLimits; $response = array('jobs' => array(), 'reached' => 'none-ready'); $type = isset($options['type']) ? $options['type'] : false; $maxJobs = isset($options['maxJobs']) ? $options['maxJobs'] : false; $maxTime = isset($options['maxTime']) ? $options['maxTime'] : false; $noThrottle = isset($options['throttle']) && !$options['throttle']; if ($type !== false && !isset($wgJobClasses[$type])) { $response['reached'] = 'none-possible'; return $response; } // Bail out if in read-only mode if (wfReadOnly()) { $response['reached'] = 'read-only'; return $response; } // Catch huge single updates that lead to slave lag $trxProfiler = Profiler::instance()->getTransactionProfiler(); $trxProfiler->setLogger(LoggerFactory::getInstance('DBPerformance')); $trxProfiler->setExpectations($wgTrxProfilerLimits['JobRunner'], __METHOD__); // Bail out if there is too much DB lag. // This check should not block as we want to try other wiki queues. $maxAllowedLag = 3; list(, $maxLag) = wfGetLB(wfWikiID())->getMaxLag(); if ($maxLag >= $maxAllowedLag) { $response['reached'] = 'slave-lag-limit'; return $response; } $group = JobQueueGroup::singleton(); // Flush any pending DB writes for sanity wfGetLBFactory()->commitAll(); // Some jobs types should not run until a certain timestamp $backoffs = array(); // map of (type => UNIX expiry) $backoffDeltas = array(); // map of (type => seconds) $wait = 'wait'; // block to read backoffs the first time $stats = RequestContext::getMain()->getStats(); $jobsPopped = 0; $timeMsTotal = 0; $flags = JobQueueGroup::USE_CACHE; $startTime = microtime(true); // time since jobs started running $checkLagPeriod = 1.0; // check slave lag this many seconds $lastCheckTime = 1; // timestamp of last slave check do { // Sync the persistent backoffs with concurrent runners $backoffs = $this->syncBackoffDeltas($backoffs, $backoffDeltas, $wait); $blacklist = $noThrottle ? array() : array_keys($backoffs); $wait = 'nowait'; // less important now if ($type === false) { $job = $group->pop(JobQueueGroup::TYPE_DEFAULT, $flags, $blacklist); } elseif (in_array($type, $blacklist)) { $job = false; // requested queue in backoff state } else { $job = $group->pop($type); // job from a single queue } if ($job) { // found a job $popTime = time(); $jType = $job->getType(); // Back off of certain jobs for a while (for throttling and for errors) $ttw = $this->getBackoffTimeToWait($job); if ($ttw > 0) { // Always add the delta for other runners in case the time running the // job negated the backoff for each individually but not collectively. $backoffDeltas[$jType] = isset($backoffDeltas[$jType]) ? $backoffDeltas[$jType] + $ttw : $ttw; $backoffs = $this->syncBackoffDeltas($backoffs, $backoffDeltas, $wait); } $msg = $job->toString() . " STARTING"; $this->logger->debug($msg); $this->debugCallback($msg); // Run the job... $jobStartTime = microtime(true); try { ++$jobsPopped; $status = $job->run(); $error = $job->getLastError(); $this->commitMasterChanges($job); DeferredUpdates::doUpdates(); $this->commitMasterChanges($job); } catch (Exception $e) { MWExceptionHandler::rollbackMasterChangesAndLog($e); $status = false; $error = get_class($e) . ': ' . $e->getMessage(); MWExceptionHandler::logException($e); } // Commit all outstanding connections that are in a transaction // to get a fresh repeatable read snapshot on every connection. // Note that jobs are still responsible for handling slave lag. wfGetLBFactory()->commitAll(); // Clear out title cache data from prior snapshots LinkCache::singleton()->clear(); $timeMs = intval((microtime(true) - $jobStartTime) * 1000); $timeMsTotal += $timeMs; // Record how long jobs wait before getting popped $readyTs = $job->getReadyTimestamp(); if ($readyTs) { $pickupDelay = $popTime - $readyTs; $stats->timing('jobqueue.pickup_delay.all', 1000 * $pickupDelay); $stats->timing("jobqueue.pickup_delay.{$jType}", 1000 * $pickupDelay); } // Record root job age for jobs being run $root = $job->getRootJobParams(); if ($root['rootJobTimestamp']) { $age = $popTime - wfTimestamp(TS_UNIX, $root['rootJobTimestamp']); $stats->timing("jobqueue.pickup_root_age.{$jType}", 1000 * $age); } // Track the execution time for jobs $stats->timing("jobqueue.run.{$jType}", $timeMs); // Mark the job as done on success or when the job cannot be retried if ($status !== false || !$job->allowRetries()) { $group->ack($job); // done } // Back off of certain jobs for a while (for throttling and for errors) if ($status === false && mt_rand(0, 49) == 0) { $ttw = max($ttw, 30); // too many errors $backoffDeltas[$jType] = isset($backoffDeltas[$jType]) ? $backoffDeltas[$jType] + $ttw : $ttw; } if ($status === false) { $msg = $job->toString() . " t={$timeMs} error={$error}"; $this->logger->error($msg); $this->debugCallback($msg); } else { $msg = $job->toString() . " t={$timeMs} good"; $this->logger->info($msg); $this->debugCallback($msg); } $response['jobs'][] = array('type' => $jType, 'status' => $status === false ? 'failed' : 'ok', 'error' => $error, 'time' => $timeMs); // Break out if we hit the job count or wall time limits... if ($maxJobs && $jobsPopped >= $maxJobs) { $response['reached'] = 'job-limit'; break; } elseif ($maxTime && microtime(true) - $startTime > $maxTime) { $response['reached'] = 'time-limit'; break; } // Don't let any of the main DB slaves get backed up. // This only waits for so long before exiting and letting // other wikis in the farm (on different masters) get a chance. $timePassed = microtime(true) - $lastCheckTime; if ($timePassed >= $checkLagPeriod || $timePassed < 0) { if (!wfWaitForSlaves($lastCheckTime, false, '*', $maxAllowedLag)) { $response['reached'] = 'slave-lag-limit'; break; } $lastCheckTime = microtime(true); } // Don't let any queue slaves/backups fall behind if ($jobsPopped > 0 && $jobsPopped % 100 == 0) { $group->waitForBackups(); } // Bail if near-OOM instead of in a job if (!$this->checkMemoryOK()) { $response['reached'] = 'memory-limit'; break; } } } while ($job); // stop when there are no jobs // Sync the persistent backoffs for the next runJobs.php pass if ($backoffDeltas) { $this->syncBackoffDeltas($backoffs, $backoffDeltas, 'wait'); } $response['backoffs'] = $backoffs; $response['elapsed'] = $timeMsTotal; return $response; }
/** * Returns an associative array, ID => param value, for all items that match * Removes the matched items from the input string (passed by reference) * * @param string $text * * @return array */ public function matchAndRemove(&$text) { $found = array(); $regexes = $this->getRegex(); foreach ($regexes as $regex) { if ($regex === '') { continue; } $matches = array(); $res = preg_match_all($regex, $text, $matches, PREG_SET_ORDER); if ($res === false) { LoggerFactory::getInstance('parser')->warning('preg_match_all returned false', array('code' => preg_last_error(), 'regex' => $regex, 'text' => $text)); } elseif ($res) { foreach ($matches as $m) { list($name, $param) = $this->parseMatch($m); $found[$name] = $param; } } $res = preg_replace($regex, '', $text); if ($res === null) { LoggerFactory::getInstance('parser')->warning('preg_replace returned null', array('code' => preg_last_error(), 'regex' => $regex, 'text' => $text)); } $text = $res; } return $found; }
/** * @param array $options * $options include: * - connectTimeout : The timeout for new connections, in seconds. * Optional, default is 1 second. * - readTimeout : The timeout for operation reads, in seconds. * Commands like BLPOP can fail if told to wait longer than this. * Optional, default is 1 second. * - persistent : Set this to true to allow connections to persist across * multiple web requests. False by default. * - password : The authentication password, will be sent to Redis in clear text. * Optional, if it is unspecified, no AUTH command will be sent. * - serializer : Set to "php", "igbinary", or "none". Default is "php". * @return RedisConnectionPool */ public static function singleton(array $options) { $options = self::applyDefaultConfig($options); // Map the options to a unique hash... ksort($options); // normalize to avoid pool fragmentation $id = sha1(serialize($options)); // Initialize the object at the hash as needed... if (!isset(self::$instances[$id])) { self::$instances[$id] = new self($options); LoggerFactory::getInstance('redis')->debug("Creating a new " . __CLASS__ . " instance with id {$id}."); } return self::$instances[$id]; }
/** * @var Connection */ public function __construct(Connection $conn) { parent::__construct($conn, null, null); $this->log = LoggerFactory::getInstance('CirrusSearch'); $this->failedLog = LoggerFactory::getInstance('CirrusSearchChangeFailed'); }
/** * Log an exception that wasn't thrown but made to wrap an error. * * @since 1.25 * @param ErrorException $e * @param string $channel */ protected static function logError(ErrorException $e, $channel) { // The set_error_handler callback is independent from error_reporting. // Filter out unwanted errors manually (e.g. when // MediaWiki\suppressWarnings is active). $suppressed = (error_reporting() & $e->getSeverity()) === 0; if (!$suppressed) { $logger = LoggerFactory::getInstance($channel); $logger->error(self::getLogMessage($e), self::getLogContext($e)); } // Include all errors in the json log (surpressed errors will be flagged) $json = self::jsonSerializeException($e, false, FormatJson::ALL_OK); if ($json !== false) { $logger = LoggerFactory::getInstance("{$channel}-json"); $logger->error($json, ['private' => true]); } Hooks::run('LogException', [$e, $suppressed]); }
/** * Do standard deferred updates after page edit. * Update links tables, site stats, search index and message cache. * Purges pages that include this page if the text was changed here. * Every 100th edit, prune the recent changes table. * * @param Revision $revision * @param User $user User object that did the revision * @param array $options Array of options, following indexes are used: * - changed: boolean, whether the revision changed the content (default true) * - created: boolean, whether the revision created the page (default false) * - moved: boolean, whether the page was moved (default false) * - restored: boolean, whether the page was undeleted (default false) * - oldrevision: Revision object for the pre-update revision (default null) * - oldcountable: boolean, null, or string 'no-change' (default null): * - boolean: whether the page was counted as an article before that * revision, only used in changed is true and created is false * - null: if created is false, don't update the article count; if created * is true, do update the article count * - 'no-change': don't update the article count, ever */ public function doEditUpdates(Revision $revision, User $user, array $options = []) { global $wgRCWatchCategoryMembership, $wgContLang; $options += ['changed' => true, 'created' => false, 'moved' => false, 'restored' => false, 'oldrevision' => null, 'oldcountable' => null]; $content = $revision->getContent(); $logger = LoggerFactory::getInstance('SaveParse'); // See if the parser output before $revision was inserted is still valid $editInfo = false; if (!$this->mPreparedEdit) { $logger->debug(__METHOD__ . ": No prepared edit...\n"); } elseif ($this->mPreparedEdit->output->getFlag('vary-revision')) { $logger->info(__METHOD__ . ": Prepared edit has vary-revision...\n"); } elseif ($this->mPreparedEdit->output->getFlag('vary-revision-id') && $this->mPreparedEdit->output->getSpeculativeRevIdUsed() !== $revision->getId()) { $logger->info(__METHOD__ . ": Prepared edit has vary-revision-id with wrong ID...\n"); } elseif ($this->mPreparedEdit->output->getFlag('vary-user') && !$options['changed']) { $logger->info(__METHOD__ . ": Prepared edit has vary-user and is null...\n"); } else { wfDebug(__METHOD__ . ": Using prepared edit...\n"); $editInfo = $this->mPreparedEdit; } if (!$editInfo) { // Parse the text again if needed. Be careful not to do pre-save transform twice: // $text is usually already pre-save transformed once. Avoid using the edit stash // as any prepared content from there or in doEditContent() was already rejected. $editInfo = $this->prepareContentForEdit($content, $revision, $user, null, false); } // Save it to the parser cache. // Make sure the cache time matches page_touched to avoid double parsing. ParserCache::singleton()->save($editInfo->output, $this, $editInfo->popts, $revision->getTimestamp(), $editInfo->revid); // Update the links tables and other secondary data if ($content) { $recursive = $options['changed']; // bug 50785 $updates = $content->getSecondaryDataUpdates($this->getTitle(), null, $recursive, $editInfo->output); foreach ($updates as $update) { if ($update instanceof LinksUpdate) { $update->setRevision($revision); $update->setTriggeringUser($user); } DeferredUpdates::addUpdate($update); } if ($wgRCWatchCategoryMembership && $this->getContentHandler()->supportsCategories() === true && ($options['changed'] || $options['created']) && !$options['restored']) { // Note: jobs are pushed after deferred updates, so the job should be able to see // the recent change entry (also done via deferred updates) and carry over any // bot/deletion/IP flags, ect. JobQueueGroup::singleton()->lazyPush(new CategoryMembershipChangeJob($this->getTitle(), ['pageId' => $this->getId(), 'revTimestamp' => $revision->getTimestamp()])); } } Hooks::run('ArticleEditUpdates', [&$this, &$editInfo, $options['changed']]); if (Hooks::run('ArticleEditUpdatesDeleteFromRecentchanges', [&$this])) { // Flush old entries from the `recentchanges` table if (mt_rand(0, 9) == 0) { JobQueueGroup::singleton()->lazyPush(RecentChangesUpdateJob::newPurgeJob()); } } if (!$this->exists()) { return; } $id = $this->getId(); $title = $this->mTitle->getPrefixedDBkey(); $shortTitle = $this->mTitle->getDBkey(); if ($options['oldcountable'] === 'no-change' || !$options['changed'] && !$options['moved']) { $good = 0; } elseif ($options['created']) { $good = (int) $this->isCountable($editInfo); } elseif ($options['oldcountable'] !== null) { $good = (int) $this->isCountable($editInfo) - (int) $options['oldcountable']; } else { $good = 0; } $edits = $options['changed'] ? 1 : 0; $total = $options['created'] ? 1 : 0; DeferredUpdates::addUpdate(new SiteStatsUpdate(0, $edits, $good, $total)); DeferredUpdates::addUpdate(new SearchUpdate($id, $title, $content)); // If this is another user's talk page, update newtalk. // Don't do this if $options['changed'] = false (null-edits) nor if // it's a minor edit and the user doesn't want notifications for those. if ($options['changed'] && $this->mTitle->getNamespace() == NS_USER_TALK && $shortTitle != $user->getTitleKey() && !($revision->isMinor() && $user->isAllowed('nominornewtalk'))) { $recipient = User::newFromName($shortTitle, false); if (!$recipient) { wfDebug(__METHOD__ . ": invalid username\n"); } else { // Allow extensions to prevent user notification // when a new message is added to their talk page if (Hooks::run('ArticleEditUpdateNewTalk', [&$this, $recipient])) { if (User::isIP($shortTitle)) { // An anonymous user $recipient->setNewtalk(true, $revision); } elseif ($recipient->isLoggedIn()) { $recipient->setNewtalk(true, $revision); } else { wfDebug(__METHOD__ . ": don't need to notify a nonexistent user\n"); } } } } if ($this->mTitle->getNamespace() == NS_MEDIAWIKI) { // XXX: could skip pseudo-messages like js/css here, based on content model. $msgtext = $content ? $content->getWikitextForTransclusion() : null; if ($msgtext === false || $msgtext === null) { $msgtext = ''; } MessageCache::singleton()->replace($shortTitle, $msgtext); if ($wgContLang->hasVariants()) { $wgContLang->updateConversionTable($this->mTitle); } } if ($options['created']) { self::onArticleCreate($this->mTitle); } elseif ($options['changed']) { // bug 50785 self::onArticleEdit($this->mTitle, $revision); } ResourceLoaderWikiModule::invalidateModuleCache($this->mTitle, $options['oldrevision'], $revision, wfWikiID()); }
/** * @todo document */ function wfLogProfilingData() { global $wgDebugLogGroups, $wgDebugRawPage; $context = RequestContext::getMain(); $request = $context->getRequest(); $profiler = Profiler::instance(); $profiler->setContext($context); $profiler->logData(); $config = $context->getConfig(); if ($config->get('StatsdServer')) { $statsdServer = explode(':', $config->get('StatsdServer')); $statsdHost = $statsdServer[0]; $statsdPort = isset($statsdServer[1]) ? $statsdServer[1] : 8125; $statsdSender = new SocketSender($statsdHost, $statsdPort); $statsdClient = new StatsdClient($statsdSender); $statsdClient->send($context->getStats()->getBuffer()); } # Profiling must actually be enabled... if ($profiler instanceof ProfilerStub) { return; } if (isset($wgDebugLogGroups['profileoutput']) && $wgDebugLogGroups['profileoutput'] === false) { // Explicitly disabled return; } if (!$wgDebugRawPage && wfIsDebugRawPage()) { return; } $ctx = array('elapsed' => $request->getElapsedTime()); if (!empty($_SERVER['HTTP_X_FORWARDED_FOR'])) { $ctx['forwarded_for'] = $_SERVER['HTTP_X_FORWARDED_FOR']; } if (!empty($_SERVER['HTTP_CLIENT_IP'])) { $ctx['client_ip'] = $_SERVER['HTTP_CLIENT_IP']; } if (!empty($_SERVER['HTTP_FROM'])) { $ctx['from'] = $_SERVER['HTTP_FROM']; } if (isset($ctx['forwarded_for']) || isset($ctx['client_ip']) || isset($ctx['from'])) { $ctx['proxy'] = $_SERVER['REMOTE_ADDR']; } // Don't load $wgUser at this late stage just for statistics purposes // @todo FIXME: We can detect some anons even if it is not loaded. // See User::getId() $user = $context->getUser(); $ctx['anon'] = $user->isItemLoaded('id') && $user->isAnon(); // Command line script uses a FauxRequest object which does not have // any knowledge about an URL and throw an exception instead. try { $ctx['url'] = urldecode($request->getRequestURL()); } catch (Exception $ignored) { // no-op } $ctx['output'] = $profiler->getOutput(); $log = LoggerFactory::getInstance('profileoutput'); $log->info("Elapsed: {elapsed}; URL: <{url}>\n{output}", $ctx); }
$profName = $fname . '-extensions-' . get_class($func[0]) . '::' . $func[1]; } else { $profName = $fname . '-extensions-' . implode('::', $func); } } else { $profName = $fname . '-extensions-' . strval($func); } $ps_ext_func = Profiler::instance()->scopedProfileIn($profName); call_user_func($func); Profiler::instance()->scopedProfileOut($ps_ext_func); } // If the session user has a 0 id but a valid name, that means we need to // autocreate it. if (!defined('MW_NO_SESSION') && !$wgCommandLineMode) { $sessionUser = MediaWiki\Session\SessionManager::getGlobalSession()->getUser(); if ($sessionUser->getId() === 0 && User::isValidUserName($sessionUser->getName())) { $ps_autocreate = Profiler::instance()->scopedProfileIn($fname . '-autocreate'); $res = MediaWiki\Auth\AuthManager::singleton()->autoCreateUser($sessionUser, MediaWiki\Auth\AuthManager::AUTOCREATE_SOURCE_SESSION, true); Profiler::instance()->scopedProfileOut($ps_autocreate); \MediaWiki\Logger\LoggerFactory::getInstance('authevents')->info('Autocreation attempt', ['event' => 'autocreate', 'status' => $res]); unset($res); } unset($sessionUser); } if (!$wgCommandLineMode) { Pingback::schedulePingback(); } wfDebug("Fully initialised\n"); $wgFullyInitialised = true; Profiler::instance()->scopedProfileOut($ps_extensions); Profiler::instance()->scopedProfileOut($ps_setup);