/** * Load in previous master positions for the client */ protected function initPositions() { if ($this->initialized) { return; } $this->initialized = true; if ($this->wait) { // If there is an expectation to see master positions with a certain min // timestamp, then block until they appear, or until a timeout is reached. if ($this->waitForPosTime > 0.0) { $data = null; $loop = new WaitConditionLoop(function () use(&$data) { $data = $this->store->get($this->key); return self::minPosTime($data) >= $this->waitForPosTime ? WaitConditionLoop::CONDITION_REACHED : WaitConditionLoop::CONDITION_CONTINUE; }, $this->waitForPosTimeout); $result = $loop->invoke(); $waitedMs = $loop->getLastWaitTime() * 1000.0; if ($result == $loop::CONDITION_REACHED) { $msg = "expected and found pos time {$this->waitForPosTime} ({$waitedMs}ms)"; $this->logger->debug($msg); } else { $msg = "expected but missed pos time {$this->waitForPosTime} ({$waitedMs}ms)"; $this->logger->info($msg); } } else { $data = $this->store->get($this->key); } $this->startupPositions = $data ? $data['positions'] : []; $this->logger->info(__METHOD__ . ": key is {$this->key} (read)\n"); } else { $this->startupPositions = []; $this->logger->info(__METHOD__ . ": key is {$this->key} (unread)\n"); } }
/** * @see JobQueueAggregator::doAllGetReadyWikiQueues() */ protected function doGetAllReadyWikiQueues() { $key = $this->getReadyQueueCacheKey(); // If the cache entry wasn't present, is stale, or in .1% of cases otherwise, // regenerate the cache. Use any available stale cache if another process is // currently regenerating the pending DB information. $pendingDbInfo = $this->cache->get($key); if (!is_array($pendingDbInfo) || time() - $pendingDbInfo['timestamp'] > $this->cacheTTL || mt_rand(0, 999) == 0) { if ($this->cache->add("{$key}:rebuild", 1, 1800)) { // lock $pendingDbInfo = array('pendingDBs' => $this->findPendingWikiQueues(), 'timestamp' => time()); for ($attempts = 1; $attempts <= 25; ++$attempts) { if ($this->cache->add("{$key}:lock", 1, 60)) { // lock $this->cache->set($key, $pendingDbInfo); $this->cache->delete("{$key}:lock"); // unlock break; } } $this->cache->delete("{$key}:rebuild"); // unlock } } return is_array($pendingDbInfo) ? $pendingDbInfo['pendingDBs'] : array(); // cache is both empty and locked }
protected function doPop() { $key = $this->getCacheKey( 'empty' ); $isEmpty = $this->cache->get( $key ); if ( $isEmpty === 'true' ) { return false; } $partitionsTry = $this->partitionMap; // (partition => weight) while ( count( $partitionsTry ) ) { $partition = ArrayUtils::pickRandom( $partitionsTry ); if ( $partition === false ) { break; // all partitions at 0 weight } $queue = $this->partitionQueues[$partition]; try { $job = $queue->pop(); } catch ( JobQueueError $e ) { $job = false; MWExceptionHandler::logException( $e ); } if ( $job ) { $job->metadata['QueuePartition'] = $partition; return $job; } else { unset( $partitionsTry[$partition] ); // blacklist partition } } $this->cache->set( $key, 'true', JobQueueDB::CACHE_TTL_LONG ); return false; }
/** * @covers BagOStuff::incr */ public function testIncr() { $key = wfMemcKey('test'); $this->cache->add($key, 0); $this->cache->incr($key); $expectedValue = 1; $actualValue = $this->cache->get($key); $this->assertEquals($expectedValue, $actualValue, 'Value should be 1 after incrementing'); }
/** * Reserve a row with a single UPDATE without holding row locks over RTTs... * * @param string $uuid 32 char hex string * @param $rand integer Random unsigned integer (31 bits) * @param bool $gte Search for job_random >= $random (otherwise job_random <= $random) * @return Row|false */ protected function claimRandom($uuid, $rand, $gte) { list($dbw, $scope) = $this->getMasterDB(); // Check cache to see if the queue has <= OFFSET items $tinyQueue = $this->cache->get($this->getCacheKey('small')); $row = false; // the row acquired $invertedDirection = false; // whether one job_random direction was already scanned // This uses a replication safe method for acquiring jobs. One could use UPDATE+LIMIT // instead, but that either uses ORDER BY (in which case it deadlocks in MySQL) or is // not replication safe. Due to http://bugs.mysql.com/bug.php?id=6980, subqueries cannot // be used here with MySQL. do { if ($tinyQueue) { // queue has <= MAX_OFFSET rows // For small queues, using OFFSET will overshoot and return no rows more often. // Instead, this uses job_random to pick a row (possibly checking both directions). $ineq = $gte ? '>=' : '<='; $dir = $gte ? 'ASC' : 'DESC'; $row = $dbw->selectRow('job', '*', array('job_cmd' => $this->type, 'job_token' => '', "job_random {$ineq} {$dbw->addQuotes($rand)}"), __METHOD__, array('ORDER BY' => "job_random {$dir}")); if (!$row && !$invertedDirection) { $gte = !$gte; $invertedDirection = true; continue; // try the other direction } } else { // table *may* have >= MAX_OFFSET rows // Bug 42614: "ORDER BY job_random" with a job_random inequality causes high CPU // in MySQL if there are many rows for some reason. This uses a small OFFSET // instead of job_random for reducing excess claim retries. $row = $dbw->selectRow('job', '*', array('job_cmd' => $this->type, 'job_token' => ''), __METHOD__, array('OFFSET' => mt_rand(0, self::MAX_OFFSET))); if (!$row) { $tinyQueue = true; // we know the queue must have <= MAX_OFFSET rows $this->cache->set($this->getCacheKey('small'), 1, 30); continue; // use job_random } } if ($row) { // claim the job $dbw->update('job', array('job_token' => $uuid, 'job_token_timestamp' => $dbw->timestamp(), 'job_attempts = job_attempts+1'), array('job_cmd' => $this->type, 'job_id' => $row->job_id, 'job_token' => ''), __METHOD__); // This might get raced out by another runner when claiming the previously // selected row. The use of job_random should minimize this problem, however. if (!$dbw->affectedRows()) { $row = false; // raced out } } else { break; // nothing to do } } while (!$row); return $row; }
/** * Get a hash of a file's contents, either by retrieving a previously- * computed hash from the cache, or by computing a hash from the file. * * @private * @param string $filePath Full path to the file. * @param string $algo Name of selected hashing algorithm. * @return string|bool Hash of file contents, or false if the file could not be read. */ public function getFileContentsHashInternal($filePath, $algo = 'md4') { $mtime = MediaWiki\quietCall('filemtime', $filePath); if ($mtime === false) { return false; } $cacheKey = wfGlobalCacheKey(__CLASS__, $filePath, $mtime, $algo); $hash = $this->cache->get($cacheKey); if ($hash) { return $hash; } $contents = MediaWiki\quietCall('file_get_contents', $filePath); if ($contents === false) { return false; } $hash = hash($algo, $contents); $this->cache->set($cacheKey, $hash, 60 * 60 * 24); // 24h return $hash; }
/** * @see SiteStore::getSites * * @since 1.25 * * @return SiteList */ public function getSites() { if ($this->sites === null) { $this->sites = $this->cache->get($this->getCacheKey()); if (!is_object($this->sites)) { $this->sites = $this->siteStore->getSites(); $this->cache->set($this->getCacheKey(), $this->sites, $this->cacheTimeout); } } return $this->sites; }
/** * Get a hash of a file's contents, either by retrieving a previously- * computed hash from the cache, or by computing a hash from the file. * * @private * @param string $filePath Full path to the file. * @param string $algo Name of selected hashing algorithm. * @return string|bool Hash of file contents, or false if the file could not be read. */ public function getFileContentsHashInternal($filePath, $algo = 'md4') { $mtime = filemtime($filePath); if ($mtime === false) { return false; } $cacheKey = $this->cache->makeGlobalKey(__CLASS__, $filePath, $mtime, $algo); $hash = $this->cache->get($cacheKey); if ($hash) { return $hash; } $contents = file_get_contents($filePath); if ($contents === false) { return false; } $hash = hash($algo, $contents); $this->cache->set($cacheKey, $hash, 60 * 60 * 24); // 24h return $hash; }
/** * @see JobQueue::isRootJobOldDuplicate() * @param Job $job * @return bool */ protected function doIsRootJobOldDuplicate(Job $job) { if (!$job->hasRootJobParams()) { return false; // job has no de-deplication info } $params = $job->getRootJobParams(); $key = $this->getRootJobCacheKey($params['rootJobSignature']); // Get the last time this root job was enqueued $timestamp = $this->dupCache->get($key); // Check if a new root job was started at the location after this one's... return $timestamp && $timestamp > $params['rootJobTimestamp']; }
/** * Log a lock request failure to the cache * * @param $lockDb string * @return bool Success */ protected function cacheRecordFailure($lockDb) { if ($this->statusCache && $this->safeDelay > 0) { $path = $this->getMissKey($lockDb); $misses = $this->statusCache->get($path); if ($misses) { return $this->statusCache->incr($path); } else { return $this->statusCache->add($path, 1, $this->safeDelay); } } return true; }
/** * @param string $code * @param array $where List of wfDebug() comments * @param integer $mode Use MessageCache::FOR_UPDATE to use DB_MASTER * @return bool|string True on success or one of ("cantacquire", "disabled") */ protected function loadFromDBWithLock($code, array &$where, $mode = null) { global $wgUseLocalMessageCache; # If cache updates on all levels fail, give up on message overrides. # This is to avoid easy site outages; see $saveSuccess comments below. $statusKey = wfMemcKey('messages', $code, 'status'); $status = $this->mMemc->get($statusKey); if ($status === 'error') { $where[] = "could not load; method is still globally disabled"; return 'disabled'; } # Now let's regenerate $where[] = 'loading from database'; # Lock the cache to prevent conflicting writes. # This lock is non-blocking so stale cache can quickly be used. # Note that load() will call a blocking getReentrantScopedLock() # after this if it really need to wait for any current thread. $cacheKey = wfMemcKey('messages', $code); $scopedLock = $this->getReentrantScopedLock($cacheKey, 0); if (!$scopedLock) { $where[] = 'could not acquire main lock'; return 'cantacquire'; } $cache = $this->loadFromDB($code, $mode); $this->mCache[$code] = $cache; $saveSuccess = $this->saveToCaches($cache, 'all', $code); if (!$saveSuccess) { # Cache save has failed. # There are two main scenarios where this could be a problem: # # - The cache is more than the maximum size (typically # 1MB compressed). # # - Memcached has no space remaining in the relevant slab # class. This is unlikely with recent versions of # memcached. # # Either way, if there is a local cache, nothing bad will # happen. If there is no local cache, disabling the message # cache for all requests avoids incurring a loadFromDB() # overhead on every request, and thus saves the wiki from # complete downtime under moderate traffic conditions. if (!$wgUseLocalMessageCache) { $this->mMemc->set($statusKey, 'error', 60 * 5); $where[] = 'could not save cache, disabled globally for 5 minutes'; } else { $where[] = "could not save global cache"; } } return true; }
/** * Get an authenticated connection handle to the Swift proxy * * @throws CloudFilesException * @throws CloudFilesException|Exception * @return CF_Connection|bool False on failure */ protected function getConnection() { if ($this->connException instanceof CloudFilesException) { if (time() - $this->connErrorTime < 60) { throw $this->connException; // failed last attempt; don't bother } else { // actually retry this time $this->connException = null; $this->connErrorTime = 0; } } // Session keys expire after a while, so we renew them periodically $reAuth = time() - $this->sessionStarted > $this->authTTL; // Authenticate with proxy and get a session key... if (!$this->conn || $reAuth) { $this->sessionStarted = 0; $this->connContainerCache->clear(); $cacheKey = $this->getCredsCacheKey($this->auth->username); $creds = $this->srvCache->get($cacheKey); // credentials if (is_array($creds)) { // cache hit $this->auth->load_cached_credentials($creds['auth_token'], $creds['storage_url'], $creds['cdnm_url']); $this->sessionStarted = time() - ceil($this->authTTL / 2); // skew for worst case } else { // cache miss try { $this->auth->authenticate(); $creds = $this->auth->export_credentials(); $this->srvCache->add($cacheKey, $creds, ceil($this->authTTL / 2)); // cache $this->sessionStarted = time(); } catch (CloudFilesException $e) { $this->connException = $e; // don't keep re-trying $this->connErrorTime = time(); throw $e; // throw it back } } if ($this->conn) { // re-authorizing? $this->conn->close(); // close active cURL handles in CF_Http object } $this->conn = new CF_Connection($this->auth); } return $this->conn; }
/** * Wait for a given slave to catch up to the master pos stored in $this * @param int $index Server index * @param bool $open Check the server even if a new connection has to be made * @param int $timeout Max seconds to wait; default is mWaitTimeout * @return bool */ protected function doWait($index, $open = false, $timeout = null) { $close = false; // close the connection afterwards // Check if we already know that the DB has reached this point $server = $this->getServerName($index); $key = $this->srvCache->makeGlobalKey(__CLASS__, 'last-known-pos', $server); /** @var DBMasterPos $knownReachedPos */ $knownReachedPos = $this->srvCache->get($key); if ($knownReachedPos && $knownReachedPos->hasReached($this->mWaitForPos)) { wfDebugLog('replication', __METHOD__ . ": slave {$server} known to be caught up (pos >= {$knownReachedPos}).\n"); return true; } // Find a connection to wait on, creating one if needed and allowed $conn = $this->getAnyOpenConnection($index); if (!$conn) { if (!$open) { wfDebugLog('replication', __METHOD__ . ": no connection open for {$server}\n"); return false; } else { $conn = $this->openConnection($index, ''); if (!$conn) { wfDebugLog('replication', __METHOD__ . ": failed to connect to {$server}\n"); return false; } // Avoid connection spam in waitForAll() when connections // are made just for the sake of doing this lag check. $close = true; } } wfDebugLog('replication', __METHOD__ . ": Waiting for slave {$server} to catch up...\n"); $timeout = $timeout ?: $this->mWaitTimeout; $result = $conn->masterPosWait($this->mWaitForPos, $timeout); if ($result == -1 || is_null($result)) { // Timed out waiting for slave, use master instead $msg = __METHOD__ . ": Timed out waiting on {$server} pos {$this->mWaitForPos}"; wfDebugLog('replication', "{$msg}\n"); wfDebugLog('DBPerformance', "{$msg}:\n" . wfBacktrace(true)); $ok = false; } else { wfDebugLog('replication', __METHOD__ . ": Done\n"); $ok = true; // Remember that the DB reached this point $this->srvCache->set($key, $this->mWaitForPos, BagOStuff::TTL_DAY); } if ($close) { $this->closeConnection($conn); } return $ok; }
/** * @return array|null Credential map */ protected function getAuthentication() { if ($this->authErrorTimestamp !== null) { if (time() - $this->authErrorTimestamp < 60) { return null; // failed last attempt; don't bother } else { // actually retry this time $this->authErrorTimestamp = null; } } // Session keys expire after a while, so we renew them periodically $reAuth = time() - $this->authSessionTimestamp > $this->authTTL; // Authenticate with proxy and get a session key... if (!$this->authCreds || $reAuth) { $this->authSessionTimestamp = 0; $cacheKey = $this->getCredsCacheKey($this->swiftUser); $creds = $this->srvCache->get($cacheKey); // credentials // Try to use the credential cache if (isset($creds['auth_token']) && isset($creds['storage_url'])) { $this->authCreds = $creds; // Skew the timestamp for worst case to avoid using stale credentials $this->authSessionTimestamp = time() - ceil($this->authTTL / 2); } else { // cache miss list($rcode, $rdesc, $rhdrs, $rbody, $rerr) = $this->http->run(array('method' => 'GET', 'url' => "{$this->swiftAuthUrl}/v1.0", 'headers' => array('x-auth-user' => $this->swiftUser, 'x-auth-key' => $this->swiftKey))); if ($rcode >= 200 && $rcode <= 299) { // OK $this->authCreds = array('auth_token' => $rhdrs['x-auth-token'], 'storage_url' => $rhdrs['x-storage-url']); $this->srvCache->set($cacheKey, $this->authCreds, ceil($this->authTTL / 2)); $this->authSessionTimestamp = time(); } elseif ($rcode === 401) { $this->onError(null, __METHOD__, array(), "Authentication failed.", $rcode); $this->authErrorTimestamp = time(); return null; } else { $this->onError(null, __METHOD__, array(), "HTTP return code: {$rcode}", $rcode); $this->authErrorTimestamp = time(); return null; } } // Ceph RGW does not use <account> in URLs (OpenStack Swift uses "/v1/<account>") if (substr($this->authCreds['storage_url'], -3) === '/v1') { $this->isRGW = true; // take advantage of strong consistency in Ceph } } return $this->authCreds; }
/** * Load in previous master positions for the client */ protected function initPositions() { if ($this->initialized) { return; } $this->initialized = true; if ($this->wait) { $data = $this->store->get($this->key); $this->startupPositions = $data ? $data['positions'] : array(); wfDebugLog('replication', __METHOD__ . ": key is {$this->key} (read)\n"); } else { $this->startupPositions = array(); wfDebugLog('replication', __METHOD__ . ": key is {$this->key} (unread)\n"); } }
/** * Retrieves number of unread notifications that a user has, would return * $wgEchoMaxNotificationCount + 1 at most * * @param $cached bool Set to false to bypass the cache. * @param $dbSource int use master or slave database to pull count * @return integer: Number of unread notifications. */ public function getNotificationCount($cached = true, $dbSource = DB_SLAVE) { global $wgEchoConfig; //XXCHANGEDXX - allow anons [sc|rs] // if ( $this->mUser->isAnon() ) { // return 0; // } $memcKey = wfMemcKey('echo-notification-count', $this->mUser->getId(), $wgEchoConfig['version']); if ($cached && $this->cache->get($memcKey) !== false) { return (int) $this->cache->get($memcKey); } $count = $this->storage->getNotificationCount($this->mUser, $dbSource); $this->cache->set($memcKey, $count, 86400); return (int) $count; }
/** * Retrieve the ParserOutput from ParserCache. * false if not found or outdated. * * @param WikiPage|Article $article * @param ParserOptions $popts * @param bool $useOutdated (default false) * * @return ParserOutput|bool False on failure */ public function get($article, $popts, $useOutdated = false) { global $wgCacheEpoch; $canCache = $article->checkTouched(); if (!$canCache) { // It's a redirect now return false; } $touched = $article->getTouched(); $parserOutputKey = $this->getKey($article, $popts, $useOutdated); if ($parserOutputKey === false) { wfIncrStats('pcache.miss.absent'); return false; } $casToken = null; /** @var ParserOutput $value */ $value = $this->mMemc->get($parserOutputKey, $casToken, BagOStuff::READ_VERIFIED); if (!$value) { wfDebug("ParserOutput cache miss.\n"); wfIncrStats("pcache.miss.absent"); return false; } wfDebug("ParserOutput cache found.\n"); // The edit section preference may not be the appropiate one in // the ParserOutput, as we are not storing it in the parsercache // key. Force it here. See bug 31445. $value->setEditSectionTokens($popts->getEditSection()); $wikiPage = method_exists($article, 'getPage') ? $article->getPage() : $article; if (!$useOutdated && $value->expired($touched)) { wfIncrStats("pcache.miss.expired"); $cacheTime = $value->getCacheTime(); wfDebugLog("ParserCache", "ParserOutput key expired, touched {$touched}, " . "epoch {$wgCacheEpoch}, cached {$cacheTime}\n"); $value = false; } elseif (!$useOutdated && $value->isDifferentRevision($article->getLatest())) { wfIncrStats("pcache.miss.revid"); $revId = $article->getLatest(); $cachedRevId = $value->getCacheRevisionId(); wfDebugLog("ParserCache", "ParserOutput key is for an old revision, latest {$revId}, cached {$cachedRevId}\n"); $value = false; } elseif (Hooks::run('RejectParserCacheValue', [$value, $wikiPage, $popts]) === false) { wfIncrStats('pcache.miss.rejected'); wfDebugLog("ParserCache", "ParserOutput key valid, but rejected by RejectParserCacheValue hook handler.\n"); $value = false; } else { wfIncrStats("pcache.hit"); } return $value; }
/** * Get a connection to the Swift proxy * * @return CF_Connection|false * @throws InvalidResponseException */ protected function getConnection() { if ($this->conn === false) { throw new InvalidResponseException(); // failed last attempt } // Session keys expire after a while, so we renew them periodically if ($this->conn && time() - $this->connStarted > $this->authTTL) { $this->closeConnection(); } // Authenticate with proxy and get a session key... if ($this->conn === null) { $cacheKey = $this->getCredsCacheKey($this->auth->username); $creds = $this->srvCache->get($cacheKey); // credentials if (is_array($creds)) { // cache hit $this->auth->load_cached_credentials($creds['auth_token'], $creds['storage_url'], $creds['cdnm_url']); $this->connStarted = time() - ceil($this->authTTL / 2); // skew for worst case } else { // cache miss try { $this->auth->authenticate(); $creds = $this->auth->export_credentials(); $this->srvCache->set($cacheKey, $creds, ceil($this->authTTL / 2)); // cache $this->connStarted = time(); } catch (AuthenticationException $e) { $this->conn = false; // don't keep re-trying $this->logException($e, __METHOD__, $creds); } catch (InvalidResponseException $e) { $this->conn = false; // don't keep re-trying $this->logException($e, __METHOD__, $creds); } } $this->conn = new CF_Connection($this->auth); } if (!$this->conn) { throw new InvalidResponseException(); // auth/connection problem } return $this->conn; }
/** * @param string $type * @param string $method * @return int */ protected function getCrossPartitionSum($type, $method) { $key = $this->getCacheKey($type); $count = $this->cache->get($key); if ($count !== false) { return $count; } $failed = 0; foreach ($this->partitionQueues as $queue) { try { $count += $queue->{$method}(); } catch (JobQueueError $e) { ++$failed; MWExceptionHandler::logException($e); } } $this->throwErrorIfAllPartitionsDown($failed); $this->cache->set($key, $count, self::CACHE_TTL_SHORT); return $count; }
public function loadFromQueue() { if (!$this->queued) { return; } // See if this queue is in APC $key = wfMemcKey('registration', md5(json_encode($this->queued)), self::CACHE_VERSION); $data = $this->cache->get($key); if ($data) { $this->exportExtractedData($data); } else { $data = $this->readFromQueue($this->queued); $this->exportExtractedData($data); // Do this late since we don't want to extract it since we already // did that, but it should be cached $data['globals']['wgAutoloadClasses'] += $data['autoload']; unset($data['autoload']); $this->cache->set($key, $data, 60 * 60 * 24); } $this->queued = array(); }
public function loadFromQueue() { global $wgVersion; if (!$this->queued) { return; } // A few more things to vary the cache on $versions = ['registration' => self::CACHE_VERSION, 'mediawiki' => $wgVersion]; // See if this queue is in APC $key = wfMemcKey('registration', md5(json_encode($this->queued + $versions))); $data = $this->cache->get($key); if ($data) { $this->exportExtractedData($data); } else { $data = $this->readFromQueue($this->queued); $this->exportExtractedData($data); // Do this late since we don't want to extract it since we already // did that, but it should be cached $data['globals']['wgAutoloadClasses'] += $data['autoload']; unset($data['autoload']); $this->cache->set($key, $data, 60 * 60 * 24); } $this->queued = []; }
/** * Attempt to get a value from the cache. If the value is expired or missing, * it will be generated with the callback function (if present), and the newly * calculated value will be stored to the cache in a wrapper. * * @param BagOStuff $cache A cache object * @param string $key The cache key * @param int $expiry The expiry timestamp or interval in seconds * @param bool|callable $callback The callback for generating the value, or false * @param array $callbackParams The function parameters for the callback * @param array $deps The dependencies to store on a cache miss. Note: these * are not the dependencies used on a cache hit! Cache hits use the stored * dependency array. * * @return mixed The value, or null if it was not present in the cache and no * callback was defined. */ static function getValueFromCache($cache, $key, $expiry = 0, $callback = false, $callbackParams = array(), $deps = array()) { $obj = $cache->get($key); if (is_object($obj) && $obj instanceof DependencyWrapper && !$obj->isExpired()) { $value = $obj->value; } elseif ($callback) { $value = call_user_func_array($callback, $callbackParams); # Cache the newly-generated value $wrapper = new DependencyWrapper($value, $deps); $wrapper->storeToCache($cache, $key, $expiry); } else { $value = null; } return $value; }
/** * Loads messages from caches or from database in this order: * (1) local message cache (if $wgUseLocalMessageCache is enabled) * (2) memcached * (3) from the database. * * When succesfully loading from (2) or (3), all higher level caches are * updated for the newest version. * * Nothing is loaded if member variable mDisable is true, either manually * set by calling code or if message loading fails (is this possible?). * * Returns true if cache is already populated or it was succesfully populated, * or false if populating empty cache fails. Also returns true if MessageCache * is disabled. * * @param bool|string $code Language to which load messages * @param integer $mode Use MessageCache::FOR_UPDATE to skip process cache * @throws MWException * @return bool */ function load($code = false, $mode = null) { if (!is_string($code)) { # This isn't really nice, so at least make a note about it and try to # fall back wfDebug(__METHOD__ . " called without providing a language code\n"); $code = 'en'; } # Don't do double loading... if (isset($this->mLoadedLanguages[$code]) && $mode != self::FOR_UPDATE) { return true; } # 8 lines of code just to say (once) that message cache is disabled if ($this->mDisable) { static $shownDisabled = false; if (!$shownDisabled) { wfDebug(__METHOD__ . ": disabled\n"); $shownDisabled = true; } return true; } # Loading code starts $success = false; # Keep track of success $staleCache = false; # a cache array with expired data, or false if none has been loaded $where = array(); # Debug info, delayed to avoid spamming debug log too much # Hash of the contents is stored in memcache, to detect if data-center cache # or local cache goes out of date (e.g. due to replace() on some other server) list($hash, $hashVolatile) = $this->getValidationHash($code); # Try the local cache and check against the cluster hash key... $cache = $this->getLocalCache($code); if (!$cache) { $where[] = 'local cache is empty'; } elseif (!isset($cache['HASH']) || $cache['HASH'] !== $hash) { $where[] = 'local cache has the wrong hash'; $staleCache = $cache; } elseif ($this->isCacheExpired($cache)) { $where[] = 'local cache is expired'; $staleCache = $cache; } elseif ($hashVolatile) { $where[] = 'local cache validation key is expired/volatile'; $staleCache = $cache; } else { $where[] = 'got from local cache'; $success = true; $this->mCache[$code] = $cache; } if (!$success) { $cacheKey = wfMemcKey('messages', $code); # Key in memc for messages # Try the global cache. If it is empty, try to acquire a lock. If # the lock can't be acquired, wait for the other thread to finish # and then try the global cache a second time. for ($failedAttempts = 0; $failedAttempts < 2; $failedAttempts++) { if ($hashVolatile && $staleCache) { # Do not bother fetching the whole cache blob to avoid I/O. # Instead, just try to get the non-blocking $statusKey lock # below, and use the local stale value if it was not acquired. $where[] = 'global cache is presumed expired'; } else { $cache = $this->mMemc->get($cacheKey); if (!$cache) { $where[] = 'global cache is empty'; } elseif ($this->isCacheExpired($cache)) { $where[] = 'global cache is expired'; $staleCache = $cache; } elseif ($hashVolatile) { # DB results are slave lag prone until the holdoff TTL passes. # By then, updates should be reflected in loadFromDBWithLock(). # One thread renerates the cache while others use old values. $where[] = 'global cache is expired/volatile'; $staleCache = $cache; } else { $where[] = 'got from global cache'; $this->mCache[$code] = $cache; $this->saveToCaches($cache, 'local-only', $code); $success = true; } } if ($success) { # Done, no need to retry break; } # We need to call loadFromDB. Limit the concurrency to one process. # This prevents the site from going down when the cache expires. # Note that the slam-protection lock here is non-blocking. if ($this->loadFromDBWithLock($code, $where)) { $success = true; break; } elseif ($staleCache) { # Use the stale cache while some other thread constructs the new one $where[] = 'using stale cache'; $this->mCache[$code] = $staleCache; $success = true; break; } elseif ($failedAttempts > 0) { # Already retried once, still failed, so don't do another lock/unlock cycle # This case will typically be hit if memcached is down, or if # loadFromDB() takes longer than MSG_WAIT_TIMEOUT $where[] = "could not acquire status key."; break; } else { $statusKey = wfMemcKey('messages', $code, 'status'); $status = $this->mMemc->get($statusKey); if ($status === 'error') { # Disable cache break; } else { # Wait for the other thread to finish, then retry. Normally, # the memcached get() will then yeild the other thread's result. $where[] = 'waited for other thread to complete'; $this->getReentrantScopedLock($cacheKey); } } } } if (!$success) { $where[] = 'loading FAILED - cache is disabled'; $this->mDisable = true; $this->mCache = false; # This used to throw an exception, but that led to nasty side effects like # the whole wiki being instantly down if the memcached server died } else { # All good, just record the success $this->mLoadedLanguages[$code] = true; } $info = implode(', ', $where); wfDebugLog('MessageCache', __METHOD__ . ": Loading {$code}... {$info}\n"); return $success; }
/** * Get a message from the MediaWiki namespace, with caching. The key must * first be converted to two-part lang/msg form if necessary. * * Unlike self::get(), this function doesn't resolve fallback chains, and * some callers require this behavior. LanguageConverter::parseCachedTable() * and self::get() are some examples in core. * * @param string $title Message cache key with initial uppercase letter. * @param string $code Code denoting the language to try. * @return string|bool The message, or false if it does not exist or on error */ function getMsgFromNamespace($title, $code) { $this->load($code); if (isset($this->mCache[$code][$title])) { $entry = $this->mCache[$code][$title]; if (substr($entry, 0, 1) === ' ') { // The message exists, so make sure a string // is returned. return (string) substr($entry, 1); } elseif ($entry === '!NONEXISTENT') { return false; } elseif ($entry === '!TOO BIG') { // Fall through and try invididual message cache below } } else { // XXX: This is not cached in process cache, should it? $message = false; Hooks::run('MessagesPreLoad', array($title, &$message)); if ($message !== false) { return $message; } return false; } # Try the individual message cache $titleKey = wfMemcKey('messages', 'individual', $title); $entry = $this->mMemc->get($titleKey); if ($entry) { if (substr($entry, 0, 1) === ' ') { $this->mCache[$code][$title] = $entry; // The message exists, so make sure a string // is returned. return (string) substr($entry, 1); } elseif ($entry === '!NONEXISTENT') { $this->mCache[$code][$title] = '!NONEXISTENT'; return false; } else { # Corrupt/obsolete entry, delete it $this->mMemc->delete($titleKey); } } # Try loading it from the database $revision = Revision::newFromTitle(Title::makeTitle(NS_MEDIAWIKI, $title), false, Revision::READ_LATEST); if ($revision) { $content = $revision->getContent(); if (!$content) { // A possibly temporary loading failure. wfDebugLog('MessageCache', __METHOD__ . ": failed to load message page text for {$title} ({$code})"); $message = null; // no negative caching } else { // XXX: Is this the right way to turn a Content object into a message? // NOTE: $content is typically either WikitextContent, JavaScriptContent or // CssContent. MessageContent is *not* used for storing messages, it's // only used for wrapping them when needed. $message = $content->getWikitextForTransclusion(); if ($message === false || $message === null) { wfDebugLog('MessageCache', __METHOD__ . ": message content doesn't provide wikitext " . "(content model: " . $content->getContentHandler() . ")"); $message = false; // negative caching } else { $this->mCache[$code][$title] = ' ' . $message; $this->mMemc->set($titleKey, ' ' . $message, $this->mExpiry); } } } else { $message = false; // negative caching } if ($message === false) { // negative caching $this->mCache[$code][$title] = '!NONEXISTENT'; $this->mMemc->set($titleKey, '!NONEXISTENT', $this->mExpiry); } return $message; }
protected function doGet($key, $flags = 0) { return $flags & self::READ_LATEST ? $this->writeStore->get($key, $flags) : $this->readStore->get($key, $flags); }
protected function getServerStates(array $serverIndexes, $domain) { $writerIndex = $this->parent->getWriterIndex(); if (count($serverIndexes) == 1 && reset($serverIndexes) == $writerIndex) { # Single server only, just return zero without caching return ['lagTimes' => [$writerIndex => 0], 'weightScales' => [$writerIndex => 1.0]]; } $key = $this->getCacheKey($serverIndexes); # Randomize TTLs to reduce stampedes (4.0 - 5.0 sec) $ttl = mt_rand(4000000.0, 5000000.0) / 1000000.0; # Keep keys around longer as fallbacks $staleTTL = 60; # (a) Check the local APC cache $value = $this->srvCache->get($key); if ($value && $value['timestamp'] > microtime(true) - $ttl) { $this->replLogger->debug(__METHOD__ . ": got lag times ({$key}) from local cache"); return $value; // cache hit } $staleValue = $value ?: false; # (b) Check the shared cache and backfill APC $value = $this->mainCache->get($key); if ($value && $value['timestamp'] > microtime(true) - $ttl) { $this->srvCache->set($key, $value, $staleTTL); $this->replLogger->debug(__METHOD__ . ": got lag times ({$key}) from main cache"); return $value; // cache hit } $staleValue = $value ?: $staleValue; # (c) Cache key missing or expired; regenerate and backfill if ($this->mainCache->lock($key, 0, 10)) { # Let this process alone update the cache value $cache = $this->mainCache; /** @noinspection PhpUnusedLocalVariableInspection */ $unlocker = new ScopedCallback(function () use($cache, $key) { $cache->unlock($key); }); } elseif ($staleValue) { # Could not acquire lock but an old cache exists, so use it return $staleValue; } $lagTimes = []; $weightScales = []; $movAveRatio = $this->movingAveRatio; foreach ($serverIndexes as $i) { if ($i == $this->parent->getWriterIndex()) { $lagTimes[$i] = 0; // master always has no lag $weightScales[$i] = 1.0; // nominal weight continue; } $conn = $this->parent->getAnyOpenConnection($i); if ($conn) { $close = false; // already open } else { $conn = $this->parent->openConnection($i, $domain); $close = true; // new connection } $lastWeight = isset($staleValue['weightScales'][$i]) ? $staleValue['weightScales'][$i] : 1.0; $coefficient = $this->getWeightScale($i, $conn ?: null); $newWeight = $movAveRatio * $coefficient + (1 - $movAveRatio) * $lastWeight; // Scale from 10% to 100% of nominal weight $weightScales[$i] = max($newWeight, 0.1); if (!$conn) { $lagTimes[$i] = false; $host = $this->parent->getServerName($i); $this->replLogger->error(__METHOD__ . ": host {$host} is unreachable"); continue; } if ($conn->getLBInfo('is static')) { $lagTimes[$i] = 0; } else { $lagTimes[$i] = $conn->getLag(); if ($lagTimes[$i] === false) { $host = $this->parent->getServerName($i); $this->replLogger->error(__METHOD__ . ": host {$host} is not replicating?"); } } if ($close) { # Close the connection to avoid sleeper connections piling up. # Note that the caller will pick one of these DBs and reconnect, # which is slightly inefficient, but this only matters for the lag # time cache miss cache, which is far less common that cache hits. $this->parent->closeConnection($conn); } } # Add a timestamp key so we know when it was cached $value = ['lagTimes' => $lagTimes, 'weightScales' => $weightScales, 'timestamp' => microtime(true)]; $this->mainCache->set($key, $value, $staleTTL); $this->srvCache->set($key, $value, $staleTTL); $this->replLogger->info(__METHOD__ . ": re-calculated lag times ({$key})"); return $value; }
public function get($key, &$casToken = null, $flags = 0) { return $flags & self::READ_LATEST ? $this->writeStore->get($key, $casToken, $flags) : $this->readStore->get($key, $casToken, $flags); }
public function get($key, &$casToken = null) { return $this->readStore->get($key, $casToken); }
/** * Checks if the DB has not recently had connection/query errors. * This just avoids wasting time on doomed connection attempts. * * @param string $lockDb * @return bool */ protected function cacheCheckFailures($lockDb) { return $this->statusCache && $this->safeDelay > 0 ? !$this->statusCache->get($this->getMissKey($lockDb)) : true; }
public function getLagTimes($serverIndexes, $wiki) { if (count($serverIndexes) == 1 && reset($serverIndexes) == 0) { # Single server only, just return zero without caching return array(0 => 0); } $key = $this->getLagTimeCacheKey(); # Randomize TTLs to reduce stampedes (4.0 - 5.0 sec) $ttl = mt_rand(4000000.0, 5000000.0) / 1000000.0; # Keep keys around longer as fallbacks $staleTTL = 60; # (a) Check the local APC cache $value = $this->srvCache->get($key); if ($value && $value['timestamp'] > microtime(true) - $ttl) { wfDebugLog('replication', __METHOD__ . ": got lag times ({$key}) from local cache"); return $value['lagTimes']; // cache hit } $staleValue = $value ?: false; # (b) Check the shared cache and backfill APC $value = $this->mainCache->get($key); if ($value && $value['timestamp'] > microtime(true) - $ttl) { $this->srvCache->set($key, $value, $staleTTL); wfDebugLog('replication', __METHOD__ . ": got lag times ({$key}) from main cache"); return $value['lagTimes']; // cache hit } $staleValue = $value ?: $staleValue; # (c) Cache key missing or expired; regenerate and backfill if ($this->mainCache->lock($key, 0, 10)) { # Let this process alone update the cache value $cache = $this->mainCache; /** @noinspection PhpUnusedLocalVariableInspection */ $unlocker = new ScopedCallback(function () use($cache, $key) { $cache->unlock($key); }); } elseif ($staleValue) { # Could not acquire lock but an old cache exists, so use it return $staleValue['lagTimes']; } $lagTimes = array(); foreach ($serverIndexes as $i) { if ($i == 0) { # Master $lagTimes[$i] = 0; } elseif (false !== ($conn = $this->parent->getAnyOpenConnection($i))) { $lagTimes[$i] = $conn->getLag(); } elseif (false !== ($conn = $this->parent->openConnection($i, $wiki))) { $lagTimes[$i] = $conn->getLag(); # Close the connection to avoid sleeper connections piling up. # Note that the caller will pick one of these DBs and reconnect, # which is slightly inefficient, but this only matters for the lag # time cache miss cache, which is far less common that cache hits. $this->parent->closeConnection($conn); } } # Add a timestamp key so we know when it was cached $value = array('lagTimes' => $lagTimes, 'timestamp' => microtime(true)); $this->mainCache->set($key, $value, $staleTTL); $this->srvCache->set($key, $value, $staleTTL); wfDebugLog('replication', __METHOD__ . ": re-calculated lag times ({$key})"); return $value['lagTimes']; }