/**
  * @see JobQueueAggregator::doAllGetReadyWikiQueues()
  */
 protected function doGetAllReadyWikiQueues()
 {
     $key = $this->getReadyQueueCacheKey();
     // If the cache entry wasn't present, is stale, or in .1% of cases otherwise,
     // regenerate the cache. Use any available stale cache if another process is
     // currently regenerating the pending DB information.
     $pendingDbInfo = $this->cache->get($key);
     if (!is_array($pendingDbInfo) || time() - $pendingDbInfo['timestamp'] > $this->cacheTTL || mt_rand(0, 999) == 0) {
         if ($this->cache->add("{$key}:rebuild", 1, 1800)) {
             // lock
             $pendingDbInfo = array('pendingDBs' => $this->findPendingWikiQueues(), 'timestamp' => time());
             for ($attempts = 1; $attempts <= 25; ++$attempts) {
                 if ($this->cache->add("{$key}:lock", 1, 60)) {
                     // lock
                     $this->cache->set($key, $pendingDbInfo);
                     $this->cache->delete("{$key}:lock");
                     // unlock
                     break;
                 }
             }
             $this->cache->delete("{$key}:rebuild");
             // unlock
         }
     }
     return is_array($pendingDbInfo) ? $pendingDbInfo['pendingDBs'] : array();
     // cache is both empty and locked
 }
示例#2
0
 /**
  * Load in previous master positions for the client
  */
 protected function initPositions()
 {
     if ($this->initialized) {
         return;
     }
     $this->initialized = true;
     if ($this->wait) {
         // If there is an expectation to see master positions with a certain min
         // timestamp, then block until they appear, or until a timeout is reached.
         if ($this->waitForPosTime > 0.0) {
             $data = null;
             $loop = new WaitConditionLoop(function () use(&$data) {
                 $data = $this->store->get($this->key);
                 return self::minPosTime($data) >= $this->waitForPosTime ? WaitConditionLoop::CONDITION_REACHED : WaitConditionLoop::CONDITION_CONTINUE;
             }, $this->waitForPosTimeout);
             $result = $loop->invoke();
             $waitedMs = $loop->getLastWaitTime() * 1000.0;
             if ($result == $loop::CONDITION_REACHED) {
                 $msg = "expected and found pos time {$this->waitForPosTime} ({$waitedMs}ms)";
                 $this->logger->debug($msg);
             } else {
                 $msg = "expected but missed pos time {$this->waitForPosTime} ({$waitedMs}ms)";
                 $this->logger->info($msg);
             }
         } else {
             $data = $this->store->get($this->key);
         }
         $this->startupPositions = $data ? $data['positions'] : [];
         $this->logger->info(__METHOD__ . ": key is {$this->key} (read)\n");
     } else {
         $this->startupPositions = [];
         $this->logger->info(__METHOD__ . ": key is {$this->key} (unread)\n");
     }
 }
 /**
  * Constructor. Parameters are:
  *   - server:      A server info structure in the format required by each
  *                  element in $wgDBServers.
  *
  *   - servers:     An array of server info structures describing a set of
  *                  database servers to distribute keys to. If this is
  *                  specified, the "server" option will be ignored.
  *
  *   - purgePeriod: The average number of object cache requests in between
  *                  garbage collection operations, where expired entries
  *                  are removed from the database. Or in other words, the
  *                  reciprocal of the probability of purging on any given
  *                  request. If this is set to zero, purging will never be
  *                  done.
  *
  *   - tableName:   The table name to use, default is "objectcache".
  *
  *   - shards:      The number of tables to use for data storage on each server.
  *                  If this is more than 1, table names will be formed in the style
  *                  objectcacheNNN where NNN is the shard index, between 0 and
  *                  shards-1. The number of digits will be the minimum number
  *                  required to hold the largest shard index. Data will be
  *                  distributed across all tables by key hash. This is for
  *                  MySQL bugs 61735 and 61736.
  *
  * @param array $params
  */
 public function __construct($params)
 {
     parent::__construct($params);
     if (isset($params['servers'])) {
         $this->serverInfos = $params['servers'];
         $this->numServers = count($this->serverInfos);
         $this->serverNames = array();
         foreach ($this->serverInfos as $i => $info) {
             $this->serverNames[$i] = isset($info['host']) ? $info['host'] : "#{$i}";
         }
     } elseif (isset($params['server'])) {
         $this->serverInfos = array($params['server']);
         $this->numServers = count($this->serverInfos);
     } else {
         $this->serverInfos = false;
         $this->numServers = 1;
     }
     if (isset($params['purgePeriod'])) {
         $this->purgePeriod = intval($params['purgePeriod']);
     }
     if (isset($params['tableName'])) {
         $this->tableName = $params['tableName'];
     }
     if (isset($params['shards'])) {
         $this->shards = intval($params['shards']);
     }
 }
示例#4
0
 /**
  * @param ParserOutput $parserOutput
  * @param WikiPage $page
  * @param ParserOptions $popts
  * @param string $cacheTime Time when the cache was generated
  * @param int $revId Revision ID that was parsed
  */
 public function save($parserOutput, $page, $popts, $cacheTime = null, $revId = null)
 {
     $expire = $parserOutput->getCacheExpiry();
     if ($expire > 0) {
         $cacheTime = $cacheTime ?: wfTimestampNow();
         if (!$revId) {
             $revision = $page->getRevision();
             $revId = $revision ? $revision->getId() : null;
         }
         $optionsKey = new CacheTime();
         $optionsKey->mUsedOptions = $parserOutput->getUsedOptions();
         $optionsKey->updateCacheExpiry($expire);
         $optionsKey->setCacheTime($cacheTime);
         $parserOutput->setCacheTime($cacheTime);
         $optionsKey->setCacheRevisionId($revId);
         $parserOutput->setCacheRevisionId($revId);
         $parserOutputKey = $this->getParserOutputKey($page, $popts->optionsHash($optionsKey->mUsedOptions, $page->getTitle()));
         // Save the timestamp so that we don't have to load the revision row on view
         $parserOutput->setTimestamp($page->getTimestamp());
         $msg = "Saved in parser cache with key {$parserOutputKey}" . " and timestamp {$cacheTime}" . " and revision id {$revId}" . "\n";
         $parserOutput->mText .= "\n<!-- {$msg} -->\n";
         wfDebug($msg);
         // Save the parser output
         $this->mMemc->set($parserOutputKey, $parserOutput, $expire);
         // ...and its pointer
         $this->mMemc->set($this->getOptionsKey($page), $optionsKey, $expire);
         Hooks::run('ParserCacheSaveComplete', array($this, $parserOutput, $page->getTitle(), $popts, $revId));
     } else {
         wfDebug("Parser output was marked as uncacheable and has not been saved.\n");
     }
 }
 /**
  * Recycle or destroy any jobs that have been claimed for too long
  *
  * @return int Number of jobs recycled/deleted
  */
 public function recycleAndDeleteStaleJobs()
 {
     $now = time();
     $count = 0;
     // affected rows
     $dbw = $this->getMasterDB();
     try {
         if (!$dbw->lock("jobqueue-recycle-{$this->type}", __METHOD__, 1)) {
             return $count;
             // already in progress
         }
         // Remove claims on jobs acquired for too long if enabled...
         if ($this->claimTTL > 0) {
             $claimCutoff = $dbw->timestamp($now - $this->claimTTL);
             // Get the IDs of jobs that have be claimed but not finished after too long.
             // These jobs can be recycled into the queue by expiring the claim. Selecting
             // the IDs first means that the UPDATE can be done by primary key (less deadlocks).
             $res = $dbw->select('job', 'job_id', array('job_cmd' => $this->type, "job_token != {$dbw->addQuotes('')}", "job_token_timestamp < {$dbw->addQuotes($claimCutoff)}", "job_attempts < {$dbw->addQuotes($this->maxTries)}"), __METHOD__);
             $ids = array_map(function ($o) {
                 return $o->job_id;
             }, iterator_to_array($res));
             if (count($ids)) {
                 // Reset job_token for these jobs so that other runners will pick them up.
                 // Set the timestamp to the current time, as it is useful to now that the job
                 // was already tried before (the timestamp becomes the "released" time).
                 $dbw->update('job', array('job_token' => '', 'job_token_timestamp' => $dbw->timestamp($now)), array('job_id' => $ids), __METHOD__);
                 $affected = $dbw->affectedRows();
                 $count += $affected;
                 JobQueue::incrStats('job-recycle', $this->type, $affected, $this->wiki);
                 // The tasks recycled jobs or release delayed jobs into the queue
                 $this->cache->set($this->getCacheKey('empty'), 'false', self::CACHE_TTL_LONG);
                 $this->aggr->notifyQueueNonEmpty($this->wiki, $this->type);
             }
         }
         // Just destroy any stale jobs...
         $pruneCutoff = $dbw->timestamp($now - self::MAX_AGE_PRUNE);
         $conds = array('job_cmd' => $this->type, "job_token != {$dbw->addQuotes('')}", "job_token_timestamp < {$dbw->addQuotes($pruneCutoff)}");
         if ($this->claimTTL > 0) {
             // only prune jobs attempted too many times...
             $conds[] = "job_attempts >= {$dbw->addQuotes($this->maxTries)}";
         }
         // Get the IDs of jobs that are considered stale and should be removed. Selecting
         // the IDs first means that the UPDATE can be done by primary key (less deadlocks).
         $res = $dbw->select('job', 'job_id', $conds, __METHOD__);
         $ids = array_map(function ($o) {
             return $o->job_id;
         }, iterator_to_array($res));
         if (count($ids)) {
             $dbw->delete('job', array('job_id' => $ids), __METHOD__);
             $affected = $dbw->affectedRows();
             $count += $affected;
             JobQueue::incrStats('job-abandon', $this->type, $affected, $this->wiki);
         }
         $dbw->unlock("jobqueue-recycle-{$this->type}", __METHOD__);
     } catch (DBError $e) {
         $this->throwDBException($e);
     }
     return $count;
 }
示例#6
0
 /**
  * @param array $params Additional parameters include:
  *   - maxKeys : only allow this many keys (using oldest-first eviction)
  */
 function __construct($params = [])
 {
     parent::__construct($params);
     $this->maxCacheKeys = isset($params['maxKeys']) ? $params['maxKeys'] : INF;
     if ($this->maxCacheKeys <= 0) {
         throw new InvalidArgumentException('$maxKeys parameter must be above zero');
     }
 }
示例#7
0
 protected function doFlushCaches()
 {
     static $types = array('empty', 'size', 'acquiredcount', 'delayedcount', 'abandonedcount');
     foreach ($types as $type) {
         $this->cache->delete($this->getCacheKey($type));
     }
     foreach ($this->partitionQueues as $queue) {
         $queue->doFlushCaches();
     }
 }
示例#8
0
 /**
  * Close the connection to the Swift proxy
  *
  * @return void
  */
 protected function closeConnection()
 {
     if ($this->conn) {
         $this->srvCache->delete($this->getCredsCacheKey($this->auth->username));
         $this->conn->close();
         // close active cURL handles in CF_Http object
         $this->conn = null;
         $this->connStarted = 0;
     }
 }
 /**
  * Get a hash of a file's contents, either by retrieving a previously-
  * computed hash from the cache, or by computing a hash from the file.
  *
  * @private
  * @param string $filePath Full path to the file.
  * @param string $algo Name of selected hashing algorithm.
  * @return string|bool Hash of file contents, or false if the file could not be read.
  */
 public function getFileContentsHashInternal($filePath, $algo = 'md4')
 {
     $mtime = filemtime($filePath);
     if ($mtime === false) {
         return false;
     }
     $cacheKey = $this->cache->makeGlobalKey(__CLASS__, $filePath, $mtime, $algo);
     $hash = $this->cache->get($cacheKey);
     if ($hash) {
         return $hash;
     }
     $contents = file_get_contents($filePath);
     if ($contents === false) {
         return false;
     }
     $hash = hash($algo, $contents);
     $this->cache->set($cacheKey, $hash, 60 * 60 * 24);
     // 24h
     return $hash;
 }
示例#10
0
 /**
  * Log an unexpected exception for this backend.
  * This also sets the Status object to have a fatal error.
  *
  * @param Status|null $status
  * @param string $func
  * @param array $params
  * @param string $err Error string
  * @param int $code HTTP status
  * @param string $desc HTTP status description
  */
 public function onError($status, $func, array $params, $err = '', $code = 0, $desc = '')
 {
     if ($status instanceof Status) {
         $status->fatal('backend-fail-internal', $this->name);
     }
     if ($code == 401) {
         // possibly a stale token
         $this->srvCache->delete($this->getCredsCacheKey($this->swiftUser));
     }
     wfDebugLog('SwiftBackend', "HTTP {$code} ({$desc}) in '{$func}' (given '" . FormatJson::encode($params) . "')" . ($err ? ": {$err}" : ""));
 }
 /**
  * Constructor. Parameters are:
  *
  *   - caches:   This should have a numbered array of cache parameter
  *               structures, in the style required by $wgObjectCaches. See
  *               the documentation of $wgObjectCaches for more detail.
  *
  * @param array $params
  * @throws InvalidArgumentException
  */
 public function __construct($params)
 {
     parent::__construct($params);
     if (!isset($params['caches'])) {
         throw new InvalidArgumentException(__METHOD__ . ': the caches parameter is required');
     }
     $this->caches = array();
     foreach ($params['caches'] as $cacheInfo) {
         $this->caches[] = ObjectCache::newFromParams($cacheInfo);
     }
 }
示例#12
0
 /**
  * Get a hash of a file's contents, either by retrieving a previously-
  * computed hash from the cache, or by computing a hash from the file.
  *
  * @private
  * @param string $filePath Full path to the file.
  * @param string $algo Name of selected hashing algorithm.
  * @return string|bool Hash of file contents, or false if the file could not be read.
  */
 public function getFileContentsHashInternal($filePath, $algo = 'md4')
 {
     $mtime = MediaWiki\quietCall('filemtime', $filePath);
     if ($mtime === false) {
         return false;
     }
     $cacheKey = wfGlobalCacheKey(__CLASS__, $filePath, $mtime, $algo);
     $hash = $this->cache->get($cacheKey);
     if ($hash) {
         return $hash;
     }
     $contents = MediaWiki\quietCall('file_get_contents', $filePath);
     if ($contents === false) {
         return false;
     }
     $hash = hash($algo, $contents);
     $this->cache->set($cacheKey, $hash, 60 * 60 * 24);
     // 24h
     return $hash;
 }
 /**
  * $params include:
  *   - caches:      This should have a numbered array of cache parameter
  *                  structures, in the style required by $wgObjectCaches. See
  *                  the documentation of $wgObjectCaches for more detail.
  *                  BagOStuff objects can also be used as values.
  *                  The first cache is the primary one, being the first to
  *                  be read in the fallback chain. Writes happen to all stores
  *                  in the order they are defined. However, lock()/unlock() calls
  *                  only use the primary store.
  *   - replication: Either 'sync' or 'async'. This controls whether writes to
  *                  secondary stores are deferred when possible. Async writes
  *                  require the HHVM register_postsend_function() function.
  *                  Async writes can increase the chance of some race conditions
  *                  or cause keys to expire seconds later than expected. It is
  *                  safe to use for modules when cached values: are immutable,
  *                  invalidation uses logical TTLs, invalidation uses etag/timestamp
  *                  validation against the DB, or merge() is used to handle races.
  *
  * @param array $params
  * @throws InvalidArgumentException
  */
 public function __construct($params)
 {
     parent::__construct($params);
     if (empty($params['caches']) || !is_array($params['caches'])) {
         throw new InvalidArgumentException(__METHOD__ . ': "caches" parameter must be an array of caches');
     }
     $this->caches = array();
     foreach ($params['caches'] as $cacheInfo) {
         $this->caches[] = $cacheInfo instanceof BagOStuff ? $cacheInfo : ObjectCache::newFromParams($cacheInfo);
     }
     $this->asyncWrites = isset($params['replication']) && $params['replication'] === 'async';
 }
示例#14
0
 /**
  * @see JobQueue::isRootJobOldDuplicate()
  * @param Job $job
  * @return bool
  */
 protected function doIsRootJobOldDuplicate(Job $job)
 {
     if (!$job->hasRootJobParams()) {
         return false;
         // job has no de-deplication info
     }
     $params = $job->getRootJobParams();
     $key = $this->getRootJobCacheKey($params['rootJobSignature']);
     // Get the last time this root job was enqueued
     $timestamp = $this->dupCache->get($key);
     // Check if a new root job was started at the location after this one's...
     return $timestamp && $timestamp > $params['rootJobTimestamp'];
 }
示例#15
0
 /**
  * Log a lock request failure to the cache
  *
  * @param $lockDb string
  * @return bool Success
  */
 protected function cacheRecordFailure($lockDb)
 {
     if ($this->statusCache && $this->safeDelay > 0) {
         $path = $this->getMissKey($lockDb);
         $misses = $this->statusCache->get($path);
         if ($misses) {
             return $this->statusCache->incr($path);
         } else {
             return $this->statusCache->add($path, 1, $this->safeDelay);
         }
     }
     return true;
 }
示例#16
0
 /**
  * Wait for a given slave to catch up to the master pos stored in $this
  * @param int $index Server index
  * @param bool $open Check the server even if a new connection has to be made
  * @param int $timeout Max seconds to wait; default is mWaitTimeout
  * @return bool
  */
 protected function doWait($index, $open = false, $timeout = null)
 {
     $close = false;
     // close the connection afterwards
     // Check if we already know that the DB has reached this point
     $server = $this->getServerName($index);
     $key = $this->srvCache->makeGlobalKey(__CLASS__, 'last-known-pos', $server);
     /** @var DBMasterPos $knownReachedPos */
     $knownReachedPos = $this->srvCache->get($key);
     if ($knownReachedPos && $knownReachedPos->hasReached($this->mWaitForPos)) {
         wfDebugLog('replication', __METHOD__ . ": slave {$server} known to be caught up (pos >= {$knownReachedPos}).\n");
         return true;
     }
     // Find a connection to wait on, creating one if needed and allowed
     $conn = $this->getAnyOpenConnection($index);
     if (!$conn) {
         if (!$open) {
             wfDebugLog('replication', __METHOD__ . ": no connection open for {$server}\n");
             return false;
         } else {
             $conn = $this->openConnection($index, '');
             if (!$conn) {
                 wfDebugLog('replication', __METHOD__ . ": failed to connect to {$server}\n");
                 return false;
             }
             // Avoid connection spam in waitForAll() when connections
             // are made just for the sake of doing this lag check.
             $close = true;
         }
     }
     wfDebugLog('replication', __METHOD__ . ": Waiting for slave {$server} to catch up...\n");
     $timeout = $timeout ?: $this->mWaitTimeout;
     $result = $conn->masterPosWait($this->mWaitForPos, $timeout);
     if ($result == -1 || is_null($result)) {
         // Timed out waiting for slave, use master instead
         $msg = __METHOD__ . ": Timed out waiting on {$server} pos {$this->mWaitForPos}";
         wfDebugLog('replication', "{$msg}\n");
         wfDebugLog('DBPerformance', "{$msg}:\n" . wfBacktrace(true));
         $ok = false;
     } else {
         wfDebugLog('replication', __METHOD__ . ": Done\n");
         $ok = true;
         // Remember that the DB reached this point
         $this->srvCache->set($key, $this->mWaitForPos, BagOStuff::TTL_DAY);
     }
     if ($close) {
         $this->closeConnection($conn);
     }
     return $ok;
 }
示例#17
0
 /**
  * Load in previous master positions for the client
  */
 protected function initPositions()
 {
     if ($this->initialized) {
         return;
     }
     $this->initialized = true;
     if ($this->wait) {
         $data = $this->store->get($this->key);
         $this->startupPositions = $data ? $data['positions'] : array();
         wfDebugLog('replication', __METHOD__ . ": key is {$this->key} (read)\n");
     } else {
         $this->startupPositions = array();
         wfDebugLog('replication', __METHOD__ . ": key is {$this->key} (unread)\n");
     }
 }
示例#18
0
 /**
  * Retrieves number of unread notifications that a user has, would return
  * $wgEchoMaxNotificationCount + 1 at most
  *
  * @param $cached bool Set to false to bypass the cache.
  * @param $dbSource int use master or slave database to pull count
  * @return integer: Number of unread notifications.
  */
 public function getNotificationCount($cached = true, $dbSource = DB_SLAVE)
 {
     global $wgEchoConfig;
     //XXCHANGEDXX - allow anons [sc|rs]
     // if ( $this->mUser->isAnon() ) {
     // return 0;
     // }
     $memcKey = wfMemcKey('echo-notification-count', $this->mUser->getId(), $wgEchoConfig['version']);
     if ($cached && $this->cache->get($memcKey) !== false) {
         return (int) $this->cache->get($memcKey);
     }
     $count = $this->storage->getNotificationCount($this->mUser, $dbSource);
     $this->cache->set($memcKey, $count, 86400);
     return (int) $count;
 }
示例#19
0
 /**
  * @covers BagOStuff::getScopedLock
  */
 public function testGetScopedLock()
 {
     $key = wfMemcKey('test');
     $value1 = $this->cache->getScopedLock($key, 0);
     $value2 = $this->cache->getScopedLock($key, 0);
     $this->assertType(ScopedCallback::class, $value1, 'First call returned lock');
     $this->assertNull($value2, 'Duplicate call returned no lock');
     unset($value1);
     $value3 = $this->cache->getScopedLock($key, 0);
     $this->assertType(ScopedCallback::class, $value3, 'Lock returned callback after release');
     unset($value3);
     $value1 = $this->cache->getScopedLock($key, 0, 5, 'reentry');
     $value2 = $this->cache->getScopedLock($key, 0, 5, 'reentry');
     $this->assertType(ScopedCallback::class, $value1, 'First reentrant call returned lock');
     $this->assertType(ScopedCallback::class, $value1, 'Second reentrant call returned lock');
 }
示例#20
0
 public function __construct($params)
 {
     if (empty($params['url'])) {
         throw new InvalidArgumentException('URL parameter is required');
     }
     parent::__construct($params);
     if (empty($params['client'])) {
         $this->client = new MultiHttpClient([]);
     } else {
         $this->client = $params['client'];
     }
     // Make sure URL ends with /
     $this->url = rtrim($params['url'], '/') . '/';
     // Default config, R+W > N; no locks on reads though; writes go straight to state-machine
     $this->attrMap[self::ATTR_SYNCWRITES] = self::QOS_SYNCWRITES_QC;
 }
示例#21
0
 /**
  * Constructor
  *
  * Available parameters are:
  *   - nativeSerialize:     If true, pass objects to apc_store(), and trust it
  *                          to serialize them correctly. If false, serialize
  *                          all values in PHP.
  *
  * @param array $params
  */
 public function __construct(array $params = array())
 {
     parent::__construct($params);
     if (isset($params['nativeSerialize'])) {
         $this->nativeSerialize = $params['nativeSerialize'];
     } elseif (extension_loaded('apcu') && ini_get('apc.serializer') === 'default') {
         // APCu has a memory corruption bug when the serializer is set to 'default'.
         // See T120267, and upstream bug reports:
         //  - https://github.com/krakjoe/apcu/issues/38
         //  - https://github.com/krakjoe/apcu/issues/35
         //  - https://github.com/krakjoe/apcu/issues/111
         $this->logger->warning('The APCu extension is loaded and the apc.serializer INI setting ' . 'is set to "default". This can cause memory corruption! ' . 'You should change apc.serializer to "php" instead. ' . 'See <https://github.com/krakjoe/apcu/issues/38>.');
         $this->nativeSerialize = false;
     } else {
         $this->nativeSerialize = true;
     }
 }
 /**
  * Construct a RedisBagOStuff object. Parameters are:
  *
  *   - servers: An array of server names. A server name may be a hostname,
  *     a hostname/port combination or the absolute path of a UNIX socket.
  *     If a hostname is specified but no port, the standard port number
  *     6379 will be used. Required.
  *
  *   - connectTimeout: The timeout for new connections, in seconds. Optional,
  *     default is 1 second.
  *
  *   - persistent: Set this to true to allow connections to persist across
  *     multiple web requests. False by default.
  *
  *   - password: The authentication password, will be sent to Redis in
  *     clear text. Optional, if it is unspecified, no AUTH command will be
  *     sent.
  *
  *   - automaticFailover: If this is false, then each key will be mapped to
  *     a single server, and if that server is down, any requests for that key
  *     will fail. If this is true, a connection failure will cause the client
  *     to immediately try the next server in the list (as determined by a
  *     consistent hashing algorithm). True by default. This has the
  *     potential to create consistency issues if a server is slow enough to
  *     flap, for example if it is in swap death.
  * @param array $params
  */
 function __construct($params)
 {
     parent::__construct($params);
     $redisConf = array('serializer' => 'none');
     // manage that in this class
     foreach (array('connectTimeout', 'persistent', 'password') as $opt) {
         if (isset($params[$opt])) {
             $redisConf[$opt] = $params[$opt];
         }
     }
     $this->redisPool = RedisConnectionPool::singleton($redisConf);
     $this->servers = $params['servers'];
     if (isset($params['automaticFailover'])) {
         $this->automaticFailover = $params['automaticFailover'];
     } else {
         $this->automaticFailover = true;
     }
 }
 /**
  * Do a batch lookup from cache for file stats for all paths
  * used in a list of storage paths or FileOp objects.
  * This loads the persistent cache values into the process cache.
  *
  * @param array $items List of storage paths
  */
 protected final function primeFileCache(array $items)
 {
     $ps = Profiler::instance()->scopedProfileIn(__METHOD__ . "-{$this->name}");
     $paths = array();
     // list of storage paths
     $pathNames = array();
     // (cache key => storage path)
     // Get all the paths/containers from the items...
     foreach ($items as $item) {
         if (self::isStoragePath($item)) {
             $paths[] = FileBackend::normalizeStoragePath($item);
         }
     }
     // Get rid of any paths that failed normalization...
     $paths = array_filter($paths, 'strlen');
     // remove nulls
     // Get all the corresponding cache keys for paths...
     foreach ($paths as $path) {
         list(, $rel, ) = $this->resolveStoragePath($path);
         if ($rel !== null) {
             // valid path for this backend
             $pathNames[$this->fileCacheKey($path)] = $path;
         }
     }
     // Get all cache entries for these container cache keys...
     $values = $this->memCache->getMulti(array_keys($pathNames));
     foreach ($values as $cacheKey => $val) {
         $path = $pathNames[$cacheKey];
         if (is_array($val)) {
             $val['latest'] = false;
             // never completely trust cache
             $this->cheapCache->set($path, 'stat', $val);
             if (isset($val['sha1'])) {
                 // some backends store SHA-1 as metadata
                 $this->cheapCache->set($path, 'sha1', array('hash' => $val['sha1'], 'latest' => false));
             }
             if (isset($val['xattr'])) {
                 // some backends store headers/metadata
                 $val['xattr'] = self::normalizeXAttributes($val['xattr']);
                 $this->cheapCache->set($path, 'xattr', array('map' => $val['xattr'], 'latest' => false));
             }
         }
     }
 }
示例#24
0
 /**
  * Do a batch lookup from cache for file stats for all paths
  * used in a list of storage paths or FileOp objects.
  * This loads the persistent cache values into the process cache.
  *
  * @param array $items List of storage paths or FileOps
  * @return void
  */
 protected final function primeFileCache(array $items)
 {
     wfProfileIn(__METHOD__);
     wfProfileIn(__METHOD__ . '-' . $this->name);
     $paths = array();
     // list of storage paths
     $pathNames = array();
     // (cache key => storage path)
     // Get all the paths/containers from the items...
     foreach ($items as $item) {
         if ($item instanceof FileOp) {
             $paths = array_merge($paths, $item->storagePathsRead());
             $paths = array_merge($paths, $item->storagePathsChanged());
         } elseif (self::isStoragePath($item)) {
             $paths[] = FileBackend::normalizeStoragePath($item);
         }
     }
     // Get rid of any paths that failed normalization...
     $paths = array_filter($paths, 'strlen');
     // remove nulls
     // Get all the corresponding cache keys for paths...
     foreach ($paths as $path) {
         list(, $rel, ) = $this->resolveStoragePath($path);
         if ($rel !== null) {
             // valid path for this backend
             $pathNames[$this->fileCacheKey($path)] = $path;
         }
     }
     // Get all cache entries for these container cache keys...
     $values = $this->memCache->getMulti(array_keys($pathNames));
     foreach ($values as $cacheKey => $val) {
         if (is_array($val)) {
             $path = $pathNames[$cacheKey];
             $this->cheapCache->set($path, 'stat', $val);
             if (isset($val['sha1'])) {
                 // some backends store SHA-1 as metadata
                 $this->cheapCache->set($path, 'sha1', array('hash' => $val['sha1'], 'latest' => $val['latest']));
             }
         }
     }
     wfProfileOut(__METHOD__ . '-' . $this->name);
     wfProfileOut(__METHOD__);
 }
示例#25
0
 /**
  * Log an unexpected exception for this backend.
  * This also sets the Status object to have a fatal error.
  *
  * @param $e Exception
  * @param $status Status|null
  * @param $func string
  * @param array $params
  * @return void
  */
 protected function handleException(Exception $e, $status, $func, array $params)
 {
     if ($status instanceof Status) {
         if ($e instanceof AuthenticationException) {
             $status->fatal('backend-fail-connect', $this->name);
         } else {
             $status->fatal('backend-fail-internal', $this->name);
         }
     }
     if ($e->getMessage()) {
         trigger_error("{$func}: " . $e->getMessage(), E_USER_WARNING);
     }
     if ($e instanceof InvalidResponseException) {
         // possibly a stale token
         $this->srvCache->delete($this->getCredsCacheKey($this->auth->username));
         $this->closeConnection();
         // force a re-connect and re-auth next time
     }
     wfDebugLog('SwiftBackend', get_class($e) . " in '{$func}' (given '" . FormatJson::encode($params) . "')" . ($e->getMessage() ? ": {$e->getMessage()}" : ""));
 }
 public function loadFromQueue()
 {
     if (!$this->queued) {
         return;
     }
     // See if this queue is in APC
     $key = wfMemcKey('registration', md5(json_encode($this->queued)), self::CACHE_VERSION);
     $data = $this->cache->get($key);
     if ($data) {
         $this->exportExtractedData($data);
     } else {
         $data = $this->readFromQueue($this->queued);
         $this->exportExtractedData($data);
         // Do this late since we don't want to extract it since we already
         // did that, but it should be cached
         $data['globals']['wgAutoloadClasses'] += $data['autoload'];
         unset($data['autoload']);
         $this->cache->set($key, $data, 60 * 60 * 24);
     }
     $this->queued = array();
 }
示例#27
0
 /**
  * Construct a RedisBagOStuff object. Parameters are:
  *
  *   - servers: An array of server names. A server name may be a hostname,
  *     a hostname/port combination or the absolute path of a UNIX socket.
  *     If a hostname is specified but no port, the standard port number
  *     6379 will be used. Arrays keys can be used to specify the tag to
  *     hash on in place of the host/port. Required.
  *
  *   - connectTimeout: The timeout for new connections, in seconds. Optional,
  *     default is 1 second.
  *
  *   - persistent: Set this to true to allow connections to persist across
  *     multiple web requests. False by default.
  *
  *   - password: The authentication password, will be sent to Redis in
  *     clear text. Optional, if it is unspecified, no AUTH command will be
  *     sent.
  *
  *   - automaticFailover: If this is false, then each key will be mapped to
  *     a single server, and if that server is down, any requests for that key
  *     will fail. If this is true, a connection failure will cause the client
  *     to immediately try the next server in the list (as determined by a
  *     consistent hashing algorithm). True by default. This has the
  *     potential to create consistency issues if a server is slow enough to
  *     flap, for example if it is in swap death.
  * @param array $params
  */
 function __construct($params)
 {
     parent::__construct($params);
     $redisConf = ['serializer' => 'none'];
     // manage that in this class
     foreach (['connectTimeout', 'persistent', 'password'] as $opt) {
         if (isset($params[$opt])) {
             $redisConf[$opt] = $params[$opt];
         }
     }
     $this->redisPool = RedisConnectionPool::singleton($redisConf);
     $this->servers = $params['servers'];
     foreach ($this->servers as $key => $server) {
         $this->serverTagMap[is_int($key) ? $server : $key] = $server;
     }
     if (isset($params['automaticFailover'])) {
         $this->automaticFailover = $params['automaticFailover'];
     } else {
         $this->automaticFailover = true;
     }
 }
 /**
  * $params include:
  *   - caches: A numbered array of either ObjectFactory::getObjectFromSpec
  *      arrays yeilding BagOStuff objects or direct BagOStuff objects.
  *      If using the former, the 'args' field *must* be set.
  *      The first cache is the primary one, being the first to
  *      be read in the fallback chain. Writes happen to all stores
  *      in the order they are defined. However, lock()/unlock() calls
  *      only use the primary store.
  *   - replication: Either 'sync' or 'async'. This controls whether writes
  *      to secondary stores are deferred when possible. Async writes
  *      require setting 'asyncHandler'. HHVM register_postsend_function() function.
  *      Async writes can increase the chance of some race conditions
  *      or cause keys to expire seconds later than expected. It is
  *      safe to use for modules when cached values: are immutable,
  *      invalidation uses logical TTLs, invalidation uses etag/timestamp
  *      validation against the DB, or merge() is used to handle races.
  * @param array $params
  * @throws InvalidArgumentException
  */
 public function __construct($params)
 {
     parent::__construct($params);
     if (empty($params['caches']) || !is_array($params['caches'])) {
         throw new InvalidArgumentException(__METHOD__ . ': "caches" parameter must be an array of caches');
     }
     $this->caches = [];
     foreach ($params['caches'] as $cacheInfo) {
         if ($cacheInfo instanceof BagOStuff) {
             $this->caches[] = $cacheInfo;
         } else {
             if (!isset($cacheInfo['args'])) {
                 // B/C for when $cacheInfo was for ObjectCache::newFromParams().
                 // Callers intenting this to be for ObjectFactory::getObjectFromSpec
                 // should have set "args" per the docs above. Doings so avoids extra
                 // (likely harmless) params (factory/class/calls) ending up in "args".
                 $cacheInfo['args'] = [$cacheInfo];
             }
             $this->caches[] = ObjectFactory::getObjectFromSpec($cacheInfo);
         }
     }
     $this->asyncWrites = isset($params['replication']) && $params['replication'] === 'async' && is_callable($this->asyncHandler);
 }
示例#29
0
 public function loadFromQueue()
 {
     global $wgVersion;
     if (!$this->queued) {
         return;
     }
     // A few more things to vary the cache on
     $versions = ['registration' => self::CACHE_VERSION, 'mediawiki' => $wgVersion];
     // See if this queue is in APC
     $key = wfMemcKey('registration', md5(json_encode($this->queued + $versions)));
     $data = $this->cache->get($key);
     if ($data) {
         $this->exportExtractedData($data);
     } else {
         $data = $this->readFromQueue($this->queued);
         $this->exportExtractedData($data);
         // Do this late since we don't want to extract it since we already
         // did that, but it should be cached
         $data['globals']['wgAutoloadClasses'] += $data['autoload'];
         unset($data['autoload']);
         $this->cache->set($key, $data, 60 * 60 * 24);
     }
     $this->queued = [];
 }
示例#30
0
 /**
  * @param string $key A language message cache key that stores blobs
  * @param integer $timeout Wait timeout in seconds
  * @return null|ScopedCallback
  */
 protected function getReentrantScopedLock($key, $timeout = self::WAIT_SEC)
 {
     return $this->mMemc->getScopedLock($key, $timeout, self::LOCK_TTL, __METHOD__);
 }