/**
* Load in previous master positions for the client
*/
protected function initPositions()
{
if ($this->initialized) {
return;
}
$this->initialized = true;
if ($this->wait) {
// If there is an expectation to see master positions with a certain min
// timestamp, then block until they appear, or until a timeout is reached.
if ($this->waitForPosTime > 0.0) {
$data = null;
$loop = new WaitConditionLoop(function () use(&$data) {
$data = $this->store->get($this->key);
return self::minPosTime($data) >= $this->waitForPosTime ? WaitConditionLoop::CONDITION_REACHED : WaitConditionLoop::CONDITION_CONTINUE;
}, $this->waitForPosTimeout);
$result = $loop->invoke();
$waitedMs = $loop->getLastWaitTime() * 1000.0;
if ($result == $loop::CONDITION_REACHED) {
$msg = "expected and found pos time {$this->waitForPosTime} ({$waitedMs}ms)";
$this->logger->debug($msg);
} else {
$msg = "expected but missed pos time {$this->waitForPosTime} ({$waitedMs}ms)";
$this->logger->info($msg);
}
} else {
$data = $this->store->get($this->key);
}
$this->startupPositions = $data ? $data['positions'] : [];
$this->logger->info(__METHOD__ . ": key is {$this->key} (read)\n");
} else {
$this->startupPositions = [];
$this->logger->info(__METHOD__ . ": key is {$this->key} (unread)\n");
}
}
/**
* @see JobQueueAggregator::doAllGetReadyWikiQueues()
*/
protected function doGetAllReadyWikiQueues()
{
$key = $this->getReadyQueueCacheKey();
// If the cache entry wasn't present, is stale, or in .1% of cases otherwise,
// regenerate the cache. Use any available stale cache if another process is
// currently regenerating the pending DB information.
$pendingDbInfo = $this->cache->get($key);
if (!is_array($pendingDbInfo) || time() - $pendingDbInfo['timestamp'] > $this->cacheTTL || mt_rand(0, 999) == 0) {
if ($this->cache->add("{$key}:rebuild", 1, 1800)) {
// lock
$pendingDbInfo = array('pendingDBs' => $this->findPendingWikiQueues(), 'timestamp' => time());
for ($attempts = 1; $attempts <= 25; ++$attempts) {
if ($this->cache->add("{$key}:lock", 1, 60)) {
// lock
$this->cache->set($key, $pendingDbInfo);
$this->cache->delete("{$key}:lock");
// unlock
break;
}
}
$this->cache->delete("{$key}:rebuild");
// unlock
}
}
return is_array($pendingDbInfo) ? $pendingDbInfo['pendingDBs'] : array();
// cache is both empty and locked
}
protected function doPop() {
$key = $this->getCacheKey( 'empty' );
$isEmpty = $this->cache->get( $key );
if ( $isEmpty === 'true' ) {
return false;
}
$partitionsTry = $this->partitionMap; // (partition => weight)
while ( count( $partitionsTry ) ) {
$partition = ArrayUtils::pickRandom( $partitionsTry );
if ( $partition === false ) {
break; // all partitions at 0 weight
}
$queue = $this->partitionQueues[$partition];
try {
$job = $queue->pop();
} catch ( JobQueueError $e ) {
$job = false;
MWExceptionHandler::logException( $e );
}
if ( $job ) {
$job->metadata['QueuePartition'] = $partition;
return $job;
} else {
unset( $partitionsTry[$partition] ); // blacklist partition
}
}
$this->cache->set( $key, 'true', JobQueueDB::CACHE_TTL_LONG );
return false;
}
/**
* @covers BagOStuff::incr
*/
public function testIncr()
{
$key = wfMemcKey('test');
$this->cache->add($key, 0);
$this->cache->incr($key);
$expectedValue = 1;
$actualValue = $this->cache->get($key);
$this->assertEquals($expectedValue, $actualValue, 'Value should be 1 after incrementing');
}
/**
* Reserve a row with a single UPDATE without holding row locks over RTTs...
*
* @param string $uuid 32 char hex string
* @param $rand integer Random unsigned integer (31 bits)
* @param bool $gte Search for job_random >= $random (otherwise job_random <= $random)
* @return Row|false
*/
protected function claimRandom($uuid, $rand, $gte)
{
list($dbw, $scope) = $this->getMasterDB();
// Check cache to see if the queue has <= OFFSET items
$tinyQueue = $this->cache->get($this->getCacheKey('small'));
$row = false;
// the row acquired
$invertedDirection = false;
// whether one job_random direction was already scanned
// This uses a replication safe method for acquiring jobs. One could use UPDATE+LIMIT
// instead, but that either uses ORDER BY (in which case it deadlocks in MySQL) or is
// not replication safe. Due to http://bugs.mysql.com/bug.php?id=6980, subqueries cannot
// be used here with MySQL.
do {
if ($tinyQueue) {
// queue has <= MAX_OFFSET rows
// For small queues, using OFFSET will overshoot and return no rows more often.
// Instead, this uses job_random to pick a row (possibly checking both directions).
$ineq = $gte ? '>=' : '<=';
$dir = $gte ? 'ASC' : 'DESC';
$row = $dbw->selectRow('job', '*', array('job_cmd' => $this->type, 'job_token' => '', "job_random {$ineq} {$dbw->addQuotes($rand)}"), __METHOD__, array('ORDER BY' => "job_random {$dir}"));
if (!$row && !$invertedDirection) {
$gte = !$gte;
$invertedDirection = true;
continue;
// try the other direction
}
} else {
// table *may* have >= MAX_OFFSET rows
// Bug 42614: "ORDER BY job_random" with a job_random inequality causes high CPU
// in MySQL if there are many rows for some reason. This uses a small OFFSET
// instead of job_random for reducing excess claim retries.
$row = $dbw->selectRow('job', '*', array('job_cmd' => $this->type, 'job_token' => ''), __METHOD__, array('OFFSET' => mt_rand(0, self::MAX_OFFSET)));
if (!$row) {
$tinyQueue = true;
// we know the queue must have <= MAX_OFFSET rows
$this->cache->set($this->getCacheKey('small'), 1, 30);
continue;
// use job_random
}
}
if ($row) {
// claim the job
$dbw->update('job', array('job_token' => $uuid, 'job_token_timestamp' => $dbw->timestamp(), 'job_attempts = job_attempts+1'), array('job_cmd' => $this->type, 'job_id' => $row->job_id, 'job_token' => ''), __METHOD__);
// This might get raced out by another runner when claiming the previously
// selected row. The use of job_random should minimize this problem, however.
if (!$dbw->affectedRows()) {
$row = false;
// raced out
}
} else {
break;
// nothing to do
}
} while (!$row);
return $row;
}
/**
* Get a hash of a file's contents, either by retrieving a previously-
* computed hash from the cache, or by computing a hash from the file.
*
* @private
* @param string $filePath Full path to the file.
* @param string $algo Name of selected hashing algorithm.
* @return string|bool Hash of file contents, or false if the file could not be read.
*/
public function getFileContentsHashInternal($filePath, $algo = 'md4')
{
$mtime = MediaWiki\quietCall('filemtime', $filePath);
if ($mtime === false) {
return false;
}
$cacheKey = wfGlobalCacheKey(__CLASS__, $filePath, $mtime, $algo);
$hash = $this->cache->get($cacheKey);
if ($hash) {
return $hash;
}
$contents = MediaWiki\quietCall('file_get_contents', $filePath);
if ($contents === false) {
return false;
}
$hash = hash($algo, $contents);
$this->cache->set($cacheKey, $hash, 60 * 60 * 24);
// 24h
return $hash;
}
/**
* @see SiteStore::getSites
*
* @since 1.25
*
* @return SiteList
*/
public function getSites()
{
if ($this->sites === null) {
$this->sites = $this->cache->get($this->getCacheKey());
if (!is_object($this->sites)) {
$this->sites = $this->siteStore->getSites();
$this->cache->set($this->getCacheKey(), $this->sites, $this->cacheTimeout);
}
}
return $this->sites;
}
/**
* Get a hash of a file's contents, either by retrieving a previously-
* computed hash from the cache, or by computing a hash from the file.
*
* @private
* @param string $filePath Full path to the file.
* @param string $algo Name of selected hashing algorithm.
* @return string|bool Hash of file contents, or false if the file could not be read.
*/
public function getFileContentsHashInternal($filePath, $algo = 'md4')
{
$mtime = filemtime($filePath);
if ($mtime === false) {
return false;
}
$cacheKey = $this->cache->makeGlobalKey(__CLASS__, $filePath, $mtime, $algo);
$hash = $this->cache->get($cacheKey);
if ($hash) {
return $hash;
}
$contents = file_get_contents($filePath);
if ($contents === false) {
return false;
}
$hash = hash($algo, $contents);
$this->cache->set($cacheKey, $hash, 60 * 60 * 24);
// 24h
return $hash;
}
/**
* @see JobQueue::isRootJobOldDuplicate()
* @param Job $job
* @return bool
*/
protected function doIsRootJobOldDuplicate(Job $job)
{
if (!$job->hasRootJobParams()) {
return false;
// job has no de-deplication info
}
$params = $job->getRootJobParams();
$key = $this->getRootJobCacheKey($params['rootJobSignature']);
// Get the last time this root job was enqueued
$timestamp = $this->dupCache->get($key);
// Check if a new root job was started at the location after this one's...
return $timestamp && $timestamp > $params['rootJobTimestamp'];
}
/**
* Log a lock request failure to the cache
*
* @param $lockDb string
* @return bool Success
*/
protected function cacheRecordFailure($lockDb)
{
if ($this->statusCache && $this->safeDelay > 0) {
$path = $this->getMissKey($lockDb);
$misses = $this->statusCache->get($path);
if ($misses) {
return $this->statusCache->incr($path);
} else {
return $this->statusCache->add($path, 1, $this->safeDelay);
}
}
return true;
}
/**
* @param string $code
* @param array $where List of wfDebug() comments
* @param integer $mode Use MessageCache::FOR_UPDATE to use DB_MASTER
* @return bool|string True on success or one of ("cantacquire", "disabled")
*/
protected function loadFromDBWithLock($code, array &$where, $mode = null)
{
global $wgUseLocalMessageCache;
# If cache updates on all levels fail, give up on message overrides.
# This is to avoid easy site outages; see $saveSuccess comments below.
$statusKey = wfMemcKey('messages', $code, 'status');
$status = $this->mMemc->get($statusKey);
if ($status === 'error') {
$where[] = "could not load; method is still globally disabled";
return 'disabled';
}
# Now let's regenerate
$where[] = 'loading from database';
# Lock the cache to prevent conflicting writes.
# This lock is non-blocking so stale cache can quickly be used.
# Note that load() will call a blocking getReentrantScopedLock()
# after this if it really need to wait for any current thread.
$cacheKey = wfMemcKey('messages', $code);
$scopedLock = $this->getReentrantScopedLock($cacheKey, 0);
if (!$scopedLock) {
$where[] = 'could not acquire main lock';
return 'cantacquire';
}
$cache = $this->loadFromDB($code, $mode);
$this->mCache[$code] = $cache;
$saveSuccess = $this->saveToCaches($cache, 'all', $code);
if (!$saveSuccess) {
# Cache save has failed.
# There are two main scenarios where this could be a problem:
#
# - The cache is more than the maximum size (typically
# 1MB compressed).
#
# - Memcached has no space remaining in the relevant slab
# class. This is unlikely with recent versions of
# memcached.
#
# Either way, if there is a local cache, nothing bad will
# happen. If there is no local cache, disabling the message
# cache for all requests avoids incurring a loadFromDB()
# overhead on every request, and thus saves the wiki from
# complete downtime under moderate traffic conditions.
if (!$wgUseLocalMessageCache) {
$this->mMemc->set($statusKey, 'error', 60 * 5);
$where[] = 'could not save cache, disabled globally for 5 minutes';
} else {
$where[] = "could not save global cache";
}
}
return true;
}
/**
* Get an authenticated connection handle to the Swift proxy
*
* @throws CloudFilesException
* @throws CloudFilesException|Exception
* @return CF_Connection|bool False on failure
*/
protected function getConnection()
{
if ($this->connException instanceof CloudFilesException) {
if (time() - $this->connErrorTime < 60) {
throw $this->connException;
// failed last attempt; don't bother
} else {
// actually retry this time
$this->connException = null;
$this->connErrorTime = 0;
}
}
// Session keys expire after a while, so we renew them periodically
$reAuth = time() - $this->sessionStarted > $this->authTTL;
// Authenticate with proxy and get a session key...
if (!$this->conn || $reAuth) {
$this->sessionStarted = 0;
$this->connContainerCache->clear();
$cacheKey = $this->getCredsCacheKey($this->auth->username);
$creds = $this->srvCache->get($cacheKey);
// credentials
if (is_array($creds)) {
// cache hit
$this->auth->load_cached_credentials($creds['auth_token'], $creds['storage_url'], $creds['cdnm_url']);
$this->sessionStarted = time() - ceil($this->authTTL / 2);
// skew for worst case
} else {
// cache miss
try {
$this->auth->authenticate();
$creds = $this->auth->export_credentials();
$this->srvCache->add($cacheKey, $creds, ceil($this->authTTL / 2));
// cache
$this->sessionStarted = time();
} catch (CloudFilesException $e) {
$this->connException = $e;
// don't keep re-trying
$this->connErrorTime = time();
throw $e;
// throw it back
}
}
if ($this->conn) {
// re-authorizing?
$this->conn->close();
// close active cURL handles in CF_Http object
}
$this->conn = new CF_Connection($this->auth);
}
return $this->conn;
}
/**
* Wait for a given slave to catch up to the master pos stored in $this
* @param int $index Server index
* @param bool $open Check the server even if a new connection has to be made
* @param int $timeout Max seconds to wait; default is mWaitTimeout
* @return bool
*/
protected function doWait($index, $open = false, $timeout = null)
{
$close = false;
// close the connection afterwards
// Check if we already know that the DB has reached this point
$server = $this->getServerName($index);
$key = $this->srvCache->makeGlobalKey(__CLASS__, 'last-known-pos', $server);
/** @var DBMasterPos $knownReachedPos */
$knownReachedPos = $this->srvCache->get($key);
if ($knownReachedPos && $knownReachedPos->hasReached($this->mWaitForPos)) {
wfDebugLog('replication', __METHOD__ . ": slave {$server} known to be caught up (pos >= {$knownReachedPos}).\n");
return true;
}
// Find a connection to wait on, creating one if needed and allowed
$conn = $this->getAnyOpenConnection($index);
if (!$conn) {
if (!$open) {
wfDebugLog('replication', __METHOD__ . ": no connection open for {$server}\n");
return false;
} else {
$conn = $this->openConnection($index, '');
if (!$conn) {
wfDebugLog('replication', __METHOD__ . ": failed to connect to {$server}\n");
return false;
}
// Avoid connection spam in waitForAll() when connections
// are made just for the sake of doing this lag check.
$close = true;
}
}
wfDebugLog('replication', __METHOD__ . ": Waiting for slave {$server} to catch up...\n");
$timeout = $timeout ?: $this->mWaitTimeout;
$result = $conn->masterPosWait($this->mWaitForPos, $timeout);
if ($result == -1 || is_null($result)) {
// Timed out waiting for slave, use master instead
$msg = __METHOD__ . ": Timed out waiting on {$server} pos {$this->mWaitForPos}";
wfDebugLog('replication', "{$msg}\n");
wfDebugLog('DBPerformance', "{$msg}:\n" . wfBacktrace(true));
$ok = false;
} else {
wfDebugLog('replication', __METHOD__ . ": Done\n");
$ok = true;
// Remember that the DB reached this point
$this->srvCache->set($key, $this->mWaitForPos, BagOStuff::TTL_DAY);
}
if ($close) {
$this->closeConnection($conn);
}
return $ok;
}
/**
* @return array|null Credential map
*/
protected function getAuthentication()
{
if ($this->authErrorTimestamp !== null) {
if (time() - $this->authErrorTimestamp < 60) {
return null;
// failed last attempt; don't bother
} else {
// actually retry this time
$this->authErrorTimestamp = null;
}
}
// Session keys expire after a while, so we renew them periodically
$reAuth = time() - $this->authSessionTimestamp > $this->authTTL;
// Authenticate with proxy and get a session key...
if (!$this->authCreds || $reAuth) {
$this->authSessionTimestamp = 0;
$cacheKey = $this->getCredsCacheKey($this->swiftUser);
$creds = $this->srvCache->get($cacheKey);
// credentials
// Try to use the credential cache
if (isset($creds['auth_token']) && isset($creds['storage_url'])) {
$this->authCreds = $creds;
// Skew the timestamp for worst case to avoid using stale credentials
$this->authSessionTimestamp = time() - ceil($this->authTTL / 2);
} else {
// cache miss
list($rcode, $rdesc, $rhdrs, $rbody, $rerr) = $this->http->run(array('method' => 'GET', 'url' => "{$this->swiftAuthUrl}/v1.0", 'headers' => array('x-auth-user' => $this->swiftUser, 'x-auth-key' => $this->swiftKey)));
if ($rcode >= 200 && $rcode <= 299) {
// OK
$this->authCreds = array('auth_token' => $rhdrs['x-auth-token'], 'storage_url' => $rhdrs['x-storage-url']);
$this->srvCache->set($cacheKey, $this->authCreds, ceil($this->authTTL / 2));
$this->authSessionTimestamp = time();
} elseif ($rcode === 401) {
$this->onError(null, __METHOD__, array(), "Authentication failed.", $rcode);
$this->authErrorTimestamp = time();
return null;
} else {
$this->onError(null, __METHOD__, array(), "HTTP return code: {$rcode}", $rcode);
$this->authErrorTimestamp = time();
return null;
}
}
// Ceph RGW does not use <account> in URLs (OpenStack Swift uses "/v1/<account>")
if (substr($this->authCreds['storage_url'], -3) === '/v1') {
$this->isRGW = true;
// take advantage of strong consistency in Ceph
}
}
return $this->authCreds;
}
/**
* Load in previous master positions for the client
*/
protected function initPositions()
{
if ($this->initialized) {
return;
}
$this->initialized = true;
if ($this->wait) {
$data = $this->store->get($this->key);
$this->startupPositions = $data ? $data['positions'] : array();
wfDebugLog('replication', __METHOD__ . ": key is {$this->key} (read)\n");
} else {
$this->startupPositions = array();
wfDebugLog('replication', __METHOD__ . ": key is {$this->key} (unread)\n");
}
}
/**
* Retrieves number of unread notifications that a user has, would return
* $wgEchoMaxNotificationCount + 1 at most
*
* @param $cached bool Set to false to bypass the cache.
* @param $dbSource int use master or slave database to pull count
* @return integer: Number of unread notifications.
*/
public function getNotificationCount($cached = true, $dbSource = DB_SLAVE)
{
global $wgEchoConfig;
//XXCHANGEDXX - allow anons [sc|rs]
// if ( $this->mUser->isAnon() ) {
// return 0;
// }
$memcKey = wfMemcKey('echo-notification-count', $this->mUser->getId(), $wgEchoConfig['version']);
if ($cached && $this->cache->get($memcKey) !== false) {
return (int) $this->cache->get($memcKey);
}
$count = $this->storage->getNotificationCount($this->mUser, $dbSource);
$this->cache->set($memcKey, $count, 86400);
return (int) $count;
}
/**
* Retrieve the ParserOutput from ParserCache.
* false if not found or outdated.
*
* @param WikiPage|Article $article
* @param ParserOptions $popts
* @param bool $useOutdated (default false)
*
* @return ParserOutput|bool False on failure
*/
public function get($article, $popts, $useOutdated = false)
{
global $wgCacheEpoch;
$canCache = $article->checkTouched();
if (!$canCache) {
// It's a redirect now
return false;
}
$touched = $article->getTouched();
$parserOutputKey = $this->getKey($article, $popts, $useOutdated);
if ($parserOutputKey === false) {
wfIncrStats('pcache.miss.absent');
return false;
}
$casToken = null;
/** @var ParserOutput $value */
$value = $this->mMemc->get($parserOutputKey, $casToken, BagOStuff::READ_VERIFIED);
if (!$value) {
wfDebug("ParserOutput cache miss.\n");
wfIncrStats("pcache.miss.absent");
return false;
}
wfDebug("ParserOutput cache found.\n");
// The edit section preference may not be the appropiate one in
// the ParserOutput, as we are not storing it in the parsercache
// key. Force it here. See bug 31445.
$value->setEditSectionTokens($popts->getEditSection());
$wikiPage = method_exists($article, 'getPage') ? $article->getPage() : $article;
if (!$useOutdated && $value->expired($touched)) {
wfIncrStats("pcache.miss.expired");
$cacheTime = $value->getCacheTime();
wfDebugLog("ParserCache", "ParserOutput key expired, touched {$touched}, " . "epoch {$wgCacheEpoch}, cached {$cacheTime}\n");
$value = false;
} elseif (!$useOutdated && $value->isDifferentRevision($article->getLatest())) {
wfIncrStats("pcache.miss.revid");
$revId = $article->getLatest();
$cachedRevId = $value->getCacheRevisionId();
wfDebugLog("ParserCache", "ParserOutput key is for an old revision, latest {$revId}, cached {$cachedRevId}\n");
$value = false;
} elseif (Hooks::run('RejectParserCacheValue', [$value, $wikiPage, $popts]) === false) {
wfIncrStats('pcache.miss.rejected');
wfDebugLog("ParserCache", "ParserOutput key valid, but rejected by RejectParserCacheValue hook handler.\n");
$value = false;
} else {
wfIncrStats("pcache.hit");
}
return $value;
}
/**
* Get a connection to the Swift proxy
*
* @return CF_Connection|false
* @throws InvalidResponseException
*/
protected function getConnection()
{
if ($this->conn === false) {
throw new InvalidResponseException();
// failed last attempt
}
// Session keys expire after a while, so we renew them periodically
if ($this->conn && time() - $this->connStarted > $this->authTTL) {
$this->closeConnection();
}
// Authenticate with proxy and get a session key...
if ($this->conn === null) {
$cacheKey = $this->getCredsCacheKey($this->auth->username);
$creds = $this->srvCache->get($cacheKey);
// credentials
if (is_array($creds)) {
// cache hit
$this->auth->load_cached_credentials($creds['auth_token'], $creds['storage_url'], $creds['cdnm_url']);
$this->connStarted = time() - ceil($this->authTTL / 2);
// skew for worst case
} else {
// cache miss
try {
$this->auth->authenticate();
$creds = $this->auth->export_credentials();
$this->srvCache->set($cacheKey, $creds, ceil($this->authTTL / 2));
// cache
$this->connStarted = time();
} catch (AuthenticationException $e) {
$this->conn = false;
// don't keep re-trying
$this->logException($e, __METHOD__, $creds);
} catch (InvalidResponseException $e) {
$this->conn = false;
// don't keep re-trying
$this->logException($e, __METHOD__, $creds);
}
}
$this->conn = new CF_Connection($this->auth);
}
if (!$this->conn) {
throw new InvalidResponseException();
// auth/connection problem
}
return $this->conn;
}
/**
* @param string $type
* @param string $method
* @return int
*/
protected function getCrossPartitionSum($type, $method)
{
$key = $this->getCacheKey($type);
$count = $this->cache->get($key);
if ($count !== false) {
return $count;
}
$failed = 0;
foreach ($this->partitionQueues as $queue) {
try {
$count += $queue->{$method}();
} catch (JobQueueError $e) {
++$failed;
MWExceptionHandler::logException($e);
}
}
$this->throwErrorIfAllPartitionsDown($failed);
$this->cache->set($key, $count, self::CACHE_TTL_SHORT);
return $count;
}
public function loadFromQueue()
{
if (!$this->queued) {
return;
}
// See if this queue is in APC
$key = wfMemcKey('registration', md5(json_encode($this->queued)), self::CACHE_VERSION);
$data = $this->cache->get($key);
if ($data) {
$this->exportExtractedData($data);
} else {
$data = $this->readFromQueue($this->queued);
$this->exportExtractedData($data);
// Do this late since we don't want to extract it since we already
// did that, but it should be cached
$data['globals']['wgAutoloadClasses'] += $data['autoload'];
unset($data['autoload']);
$this->cache->set($key, $data, 60 * 60 * 24);
}
$this->queued = array();
}
public function loadFromQueue()
{
global $wgVersion;
if (!$this->queued) {
return;
}
// A few more things to vary the cache on
$versions = ['registration' => self::CACHE_VERSION, 'mediawiki' => $wgVersion];
// See if this queue is in APC
$key = wfMemcKey('registration', md5(json_encode($this->queued + $versions)));
$data = $this->cache->get($key);
if ($data) {
$this->exportExtractedData($data);
} else {
$data = $this->readFromQueue($this->queued);
$this->exportExtractedData($data);
// Do this late since we don't want to extract it since we already
// did that, but it should be cached
$data['globals']['wgAutoloadClasses'] += $data['autoload'];
unset($data['autoload']);
$this->cache->set($key, $data, 60 * 60 * 24);
}
$this->queued = [];
}
/**
* Attempt to get a value from the cache. If the value is expired or missing,
* it will be generated with the callback function (if present), and the newly
* calculated value will be stored to the cache in a wrapper.
*
* @param BagOStuff $cache A cache object
* @param string $key The cache key
* @param int $expiry The expiry timestamp or interval in seconds
* @param bool|callable $callback The callback for generating the value, or false
* @param array $callbackParams The function parameters for the callback
* @param array $deps The dependencies to store on a cache miss. Note: these
* are not the dependencies used on a cache hit! Cache hits use the stored
* dependency array.
*
* @return mixed The value, or null if it was not present in the cache and no
* callback was defined.
*/
static function getValueFromCache($cache, $key, $expiry = 0, $callback = false, $callbackParams = array(), $deps = array())
{
$obj = $cache->get($key);
if (is_object($obj) && $obj instanceof DependencyWrapper && !$obj->isExpired()) {
$value = $obj->value;
} elseif ($callback) {
$value = call_user_func_array($callback, $callbackParams);
# Cache the newly-generated value
$wrapper = new DependencyWrapper($value, $deps);
$wrapper->storeToCache($cache, $key, $expiry);
} else {
$value = null;
}
return $value;
}
/**
* Loads messages from caches or from database in this order:
* (1) local message cache (if $wgUseLocalMessageCache is enabled)
* (2) memcached
* (3) from the database.
*
* When succesfully loading from (2) or (3), all higher level caches are
* updated for the newest version.
*
* Nothing is loaded if member variable mDisable is true, either manually
* set by calling code or if message loading fails (is this possible?).
*
* Returns true if cache is already populated or it was succesfully populated,
* or false if populating empty cache fails. Also returns true if MessageCache
* is disabled.
*
* @param bool|string $code Language to which load messages
* @param integer $mode Use MessageCache::FOR_UPDATE to skip process cache
* @throws MWException
* @return bool
*/
function load($code = false, $mode = null)
{
if (!is_string($code)) {
# This isn't really nice, so at least make a note about it and try to
# fall back
wfDebug(__METHOD__ . " called without providing a language code\n");
$code = 'en';
}
# Don't do double loading...
if (isset($this->mLoadedLanguages[$code]) && $mode != self::FOR_UPDATE) {
return true;
}
# 8 lines of code just to say (once) that message cache is disabled
if ($this->mDisable) {
static $shownDisabled = false;
if (!$shownDisabled) {
wfDebug(__METHOD__ . ": disabled\n");
$shownDisabled = true;
}
return true;
}
# Loading code starts
$success = false;
# Keep track of success
$staleCache = false;
# a cache array with expired data, or false if none has been loaded
$where = array();
# Debug info, delayed to avoid spamming debug log too much
# Hash of the contents is stored in memcache, to detect if data-center cache
# or local cache goes out of date (e.g. due to replace() on some other server)
list($hash, $hashVolatile) = $this->getValidationHash($code);
# Try the local cache and check against the cluster hash key...
$cache = $this->getLocalCache($code);
if (!$cache) {
$where[] = 'local cache is empty';
} elseif (!isset($cache['HASH']) || $cache['HASH'] !== $hash) {
$where[] = 'local cache has the wrong hash';
$staleCache = $cache;
} elseif ($this->isCacheExpired($cache)) {
$where[] = 'local cache is expired';
$staleCache = $cache;
} elseif ($hashVolatile) {
$where[] = 'local cache validation key is expired/volatile';
$staleCache = $cache;
} else {
$where[] = 'got from local cache';
$success = true;
$this->mCache[$code] = $cache;
}
if (!$success) {
$cacheKey = wfMemcKey('messages', $code);
# Key in memc for messages
# Try the global cache. If it is empty, try to acquire a lock. If
# the lock can't be acquired, wait for the other thread to finish
# and then try the global cache a second time.
for ($failedAttempts = 0; $failedAttempts < 2; $failedAttempts++) {
if ($hashVolatile && $staleCache) {
# Do not bother fetching the whole cache blob to avoid I/O.
# Instead, just try to get the non-blocking $statusKey lock
# below, and use the local stale value if it was not acquired.
$where[] = 'global cache is presumed expired';
} else {
$cache = $this->mMemc->get($cacheKey);
if (!$cache) {
$where[] = 'global cache is empty';
} elseif ($this->isCacheExpired($cache)) {
$where[] = 'global cache is expired';
$staleCache = $cache;
} elseif ($hashVolatile) {
# DB results are slave lag prone until the holdoff TTL passes.
# By then, updates should be reflected in loadFromDBWithLock().
# One thread renerates the cache while others use old values.
$where[] = 'global cache is expired/volatile';
$staleCache = $cache;
} else {
$where[] = 'got from global cache';
$this->mCache[$code] = $cache;
$this->saveToCaches($cache, 'local-only', $code);
$success = true;
}
}
if ($success) {
# Done, no need to retry
break;
}
# We need to call loadFromDB. Limit the concurrency to one process.
# This prevents the site from going down when the cache expires.
# Note that the slam-protection lock here is non-blocking.
if ($this->loadFromDBWithLock($code, $where)) {
$success = true;
break;
} elseif ($staleCache) {
# Use the stale cache while some other thread constructs the new one
$where[] = 'using stale cache';
$this->mCache[$code] = $staleCache;
$success = true;
break;
} elseif ($failedAttempts > 0) {
# Already retried once, still failed, so don't do another lock/unlock cycle
# This case will typically be hit if memcached is down, or if
# loadFromDB() takes longer than MSG_WAIT_TIMEOUT
$where[] = "could not acquire status key.";
break;
} else {
$statusKey = wfMemcKey('messages', $code, 'status');
$status = $this->mMemc->get($statusKey);
if ($status === 'error') {
# Disable cache
break;
} else {
# Wait for the other thread to finish, then retry. Normally,
# the memcached get() will then yeild the other thread's result.
$where[] = 'waited for other thread to complete';
$this->getReentrantScopedLock($cacheKey);
}
}
}
}
if (!$success) {
$where[] = 'loading FAILED - cache is disabled';
$this->mDisable = true;
$this->mCache = false;
# This used to throw an exception, but that led to nasty side effects like
# the whole wiki being instantly down if the memcached server died
} else {
# All good, just record the success
$this->mLoadedLanguages[$code] = true;
}
$info = implode(', ', $where);
wfDebugLog('MessageCache', __METHOD__ . ": Loading {$code}... {$info}\n");
return $success;
}
/**
* Get a message from the MediaWiki namespace, with caching. The key must
* first be converted to two-part lang/msg form if necessary.
*
* Unlike self::get(), this function doesn't resolve fallback chains, and
* some callers require this behavior. LanguageConverter::parseCachedTable()
* and self::get() are some examples in core.
*
* @param string $title Message cache key with initial uppercase letter.
* @param string $code Code denoting the language to try.
* @return string|bool The message, or false if it does not exist or on error
*/
function getMsgFromNamespace($title, $code)
{
$this->load($code);
if (isset($this->mCache[$code][$title])) {
$entry = $this->mCache[$code][$title];
if (substr($entry, 0, 1) === ' ') {
// The message exists, so make sure a string
// is returned.
return (string) substr($entry, 1);
} elseif ($entry === '!NONEXISTENT') {
return false;
} elseif ($entry === '!TOO BIG') {
// Fall through and try invididual message cache below
}
} else {
// XXX: This is not cached in process cache, should it?
$message = false;
Hooks::run('MessagesPreLoad', array($title, &$message));
if ($message !== false) {
return $message;
}
return false;
}
# Try the individual message cache
$titleKey = wfMemcKey('messages', 'individual', $title);
$entry = $this->mMemc->get($titleKey);
if ($entry) {
if (substr($entry, 0, 1) === ' ') {
$this->mCache[$code][$title] = $entry;
// The message exists, so make sure a string
// is returned.
return (string) substr($entry, 1);
} elseif ($entry === '!NONEXISTENT') {
$this->mCache[$code][$title] = '!NONEXISTENT';
return false;
} else {
# Corrupt/obsolete entry, delete it
$this->mMemc->delete($titleKey);
}
}
# Try loading it from the database
$revision = Revision::newFromTitle(Title::makeTitle(NS_MEDIAWIKI, $title), false, Revision::READ_LATEST);
if ($revision) {
$content = $revision->getContent();
if (!$content) {
// A possibly temporary loading failure.
wfDebugLog('MessageCache', __METHOD__ . ": failed to load message page text for {$title} ({$code})");
$message = null;
// no negative caching
} else {
// XXX: Is this the right way to turn a Content object into a message?
// NOTE: $content is typically either WikitextContent, JavaScriptContent or
// CssContent. MessageContent is *not* used for storing messages, it's
// only used for wrapping them when needed.
$message = $content->getWikitextForTransclusion();
if ($message === false || $message === null) {
wfDebugLog('MessageCache', __METHOD__ . ": message content doesn't provide wikitext " . "(content model: " . $content->getContentHandler() . ")");
$message = false;
// negative caching
} else {
$this->mCache[$code][$title] = ' ' . $message;
$this->mMemc->set($titleKey, ' ' . $message, $this->mExpiry);
}
}
} else {
$message = false;
// negative caching
}
if ($message === false) {
// negative caching
$this->mCache[$code][$title] = '!NONEXISTENT';
$this->mMemc->set($titleKey, '!NONEXISTENT', $this->mExpiry);
}
return $message;
}
protected function doGet($key, $flags = 0)
{
return $flags & self::READ_LATEST ? $this->writeStore->get($key, $flags) : $this->readStore->get($key, $flags);
}
protected function getServerStates(array $serverIndexes, $domain)
{
$writerIndex = $this->parent->getWriterIndex();
if (count($serverIndexes) == 1 && reset($serverIndexes) == $writerIndex) {
# Single server only, just return zero without caching
return ['lagTimes' => [$writerIndex => 0], 'weightScales' => [$writerIndex => 1.0]];
}
$key = $this->getCacheKey($serverIndexes);
# Randomize TTLs to reduce stampedes (4.0 - 5.0 sec)
$ttl = mt_rand(4000000.0, 5000000.0) / 1000000.0;
# Keep keys around longer as fallbacks
$staleTTL = 60;
# (a) Check the local APC cache
$value = $this->srvCache->get($key);
if ($value && $value['timestamp'] > microtime(true) - $ttl) {
$this->replLogger->debug(__METHOD__ . ": got lag times ({$key}) from local cache");
return $value;
// cache hit
}
$staleValue = $value ?: false;
# (b) Check the shared cache and backfill APC
$value = $this->mainCache->get($key);
if ($value && $value['timestamp'] > microtime(true) - $ttl) {
$this->srvCache->set($key, $value, $staleTTL);
$this->replLogger->debug(__METHOD__ . ": got lag times ({$key}) from main cache");
return $value;
// cache hit
}
$staleValue = $value ?: $staleValue;
# (c) Cache key missing or expired; regenerate and backfill
if ($this->mainCache->lock($key, 0, 10)) {
# Let this process alone update the cache value
$cache = $this->mainCache;
/** @noinspection PhpUnusedLocalVariableInspection */
$unlocker = new ScopedCallback(function () use($cache, $key) {
$cache->unlock($key);
});
} elseif ($staleValue) {
# Could not acquire lock but an old cache exists, so use it
return $staleValue;
}
$lagTimes = [];
$weightScales = [];
$movAveRatio = $this->movingAveRatio;
foreach ($serverIndexes as $i) {
if ($i == $this->parent->getWriterIndex()) {
$lagTimes[$i] = 0;
// master always has no lag
$weightScales[$i] = 1.0;
// nominal weight
continue;
}
$conn = $this->parent->getAnyOpenConnection($i);
if ($conn) {
$close = false;
// already open
} else {
$conn = $this->parent->openConnection($i, $domain);
$close = true;
// new connection
}
$lastWeight = isset($staleValue['weightScales'][$i]) ? $staleValue['weightScales'][$i] : 1.0;
$coefficient = $this->getWeightScale($i, $conn ?: null);
$newWeight = $movAveRatio * $coefficient + (1 - $movAveRatio) * $lastWeight;
// Scale from 10% to 100% of nominal weight
$weightScales[$i] = max($newWeight, 0.1);
if (!$conn) {
$lagTimes[$i] = false;
$host = $this->parent->getServerName($i);
$this->replLogger->error(__METHOD__ . ": host {$host} is unreachable");
continue;
}
if ($conn->getLBInfo('is static')) {
$lagTimes[$i] = 0;
} else {
$lagTimes[$i] = $conn->getLag();
if ($lagTimes[$i] === false) {
$host = $this->parent->getServerName($i);
$this->replLogger->error(__METHOD__ . ": host {$host} is not replicating?");
}
}
if ($close) {
# Close the connection to avoid sleeper connections piling up.
# Note that the caller will pick one of these DBs and reconnect,
# which is slightly inefficient, but this only matters for the lag
# time cache miss cache, which is far less common that cache hits.
$this->parent->closeConnection($conn);
}
}
# Add a timestamp key so we know when it was cached
$value = ['lagTimes' => $lagTimes, 'weightScales' => $weightScales, 'timestamp' => microtime(true)];
$this->mainCache->set($key, $value, $staleTTL);
$this->srvCache->set($key, $value, $staleTTL);
$this->replLogger->info(__METHOD__ . ": re-calculated lag times ({$key})");
return $value;
}
public function get($key, &$casToken = null, $flags = 0)
{
return $flags & self::READ_LATEST ? $this->writeStore->get($key, $casToken, $flags) : $this->readStore->get($key, $casToken, $flags);
}
public function get($key, &$casToken = null)
{
return $this->readStore->get($key, $casToken);
}
/**
* Checks if the DB has not recently had connection/query errors.
* This just avoids wasting time on doomed connection attempts.
*
* @param string $lockDb
* @return bool
*/
protected function cacheCheckFailures($lockDb)
{
return $this->statusCache && $this->safeDelay > 0 ? !$this->statusCache->get($this->getMissKey($lockDb)) : true;
}
public function getLagTimes($serverIndexes, $wiki)
{
if (count($serverIndexes) == 1 && reset($serverIndexes) == 0) {
# Single server only, just return zero without caching
return array(0 => 0);
}
$key = $this->getLagTimeCacheKey();
# Randomize TTLs to reduce stampedes (4.0 - 5.0 sec)
$ttl = mt_rand(4000000.0, 5000000.0) / 1000000.0;
# Keep keys around longer as fallbacks
$staleTTL = 60;
# (a) Check the local APC cache
$value = $this->srvCache->get($key);
if ($value && $value['timestamp'] > microtime(true) - $ttl) {
wfDebugLog('replication', __METHOD__ . ": got lag times ({$key}) from local cache");
return $value['lagTimes'];
// cache hit
}
$staleValue = $value ?: false;
# (b) Check the shared cache and backfill APC
$value = $this->mainCache->get($key);
if ($value && $value['timestamp'] > microtime(true) - $ttl) {
$this->srvCache->set($key, $value, $staleTTL);
wfDebugLog('replication', __METHOD__ . ": got lag times ({$key}) from main cache");
return $value['lagTimes'];
// cache hit
}
$staleValue = $value ?: $staleValue;
# (c) Cache key missing or expired; regenerate and backfill
if ($this->mainCache->lock($key, 0, 10)) {
# Let this process alone update the cache value
$cache = $this->mainCache;
/** @noinspection PhpUnusedLocalVariableInspection */
$unlocker = new ScopedCallback(function () use($cache, $key) {
$cache->unlock($key);
});
} elseif ($staleValue) {
# Could not acquire lock but an old cache exists, so use it
return $staleValue['lagTimes'];
}
$lagTimes = array();
foreach ($serverIndexes as $i) {
if ($i == 0) {
# Master
$lagTimes[$i] = 0;
} elseif (false !== ($conn = $this->parent->getAnyOpenConnection($i))) {
$lagTimes[$i] = $conn->getLag();
} elseif (false !== ($conn = $this->parent->openConnection($i, $wiki))) {
$lagTimes[$i] = $conn->getLag();
# Close the connection to avoid sleeper connections piling up.
# Note that the caller will pick one of these DBs and reconnect,
# which is slightly inefficient, but this only matters for the lag
# time cache miss cache, which is far less common that cache hits.
$this->parent->closeConnection($conn);
}
}
# Add a timestamp key so we know when it was cached
$value = array('lagTimes' => $lagTimes, 'timestamp' => microtime(true));
$this->mainCache->set($key, $value, $staleTTL);
$this->srvCache->set($key, $value, $staleTTL);
wfDebugLog('replication', __METHOD__ . ": re-calculated lag times ({$key})");
return $value['lagTimes'];
}
