/**
* @see JobQueue::doIsEmpty()
* @return bool
*/
protected function doIsEmpty()
{
$key = $this->getCacheKey('empty');
$isEmpty = $this->cache->get($key);
if ($isEmpty === 'true') {
return true;
} elseif ($isEmpty === 'false') {
return false;
}
list($dbr, $scope) = $this->getSlaveDB();
$found = $dbr->selectField('job', '1', array('job_cmd' => $this->type, 'job_token' => ''), __METHOD__);
$this->cache->add($key, $found ? 'false' : 'true', self::CACHE_TTL_LONG);
return !$found;
}
/**
* @see JobQueueAggregator::doAllGetReadyWikiQueues()
*/
protected function doGetAllReadyWikiQueues()
{
$key = $this->getReadyQueueCacheKey();
// If the cache entry wasn't present, is stale, or in .1% of cases otherwise,
// regenerate the cache. Use any available stale cache if another process is
// currently regenerating the pending DB information.
$pendingDbInfo = $this->cache->get($key);
if (!is_array($pendingDbInfo) || time() - $pendingDbInfo['timestamp'] > $this->cacheTTL || mt_rand(0, 999) == 0) {
if ($this->cache->add("{$key}:rebuild", 1, 1800)) {
// lock
$pendingDbInfo = array('pendingDBs' => $this->findPendingWikiQueues(), 'timestamp' => time());
for ($attempts = 1; $attempts <= 25; ++$attempts) {
if ($this->cache->add("{$key}:lock", 1, 60)) {
// lock
$this->cache->set($key, $pendingDbInfo);
$this->cache->delete("{$key}:lock");
// unlock
break;
}
}
$this->cache->delete("{$key}:rebuild");
// unlock
}
}
return is_array($pendingDbInfo) ? $pendingDbInfo['pendingDBs'] : array();
// cache is both empty and locked
}
/**
* Set the cached stat info for a file path.
* Negatives (404s) are not cached. By not caching negatives, we can skip cache
* salting for the case when a file is created at a path were there was none before.
*
* @param $path string Storage path
* @param $val mixed Information to cache
*/
protected final function setFileCache($path, $val)
{
$path = FileBackend::normalizeStoragePath($path);
if ($path === null) {
return;
// invalid storage path
}
$this->memCache->add($this->fileCacheKey($path), $val, 7 * 86400);
}
/**
* Set the cached stat info for a file path.
* Negatives (404s) are not cached. By not caching negatives, we can skip cache
* salting for the case when a file is created at a path were there was none before.
*
* @param string $path Storage path
* @param array $val Stat information to cache
* @return void
*/
final protected function setFileCache( $path, array $val ) {
$path = FileBackend::normalizeStoragePath( $path );
if ( $path === null ) {
return; // invalid storage path
}
$age = time() - wfTimestamp( TS_UNIX, $val['mtime'] );
$ttl = min( 7 * 86400, max( 300, floor( .1 * $age ) ) );
$this->memCache->add( $this->fileCacheKey( $path ), $val, $ttl );
}
/**
* Log a lock request failure to the cache
*
* @param $lockDb string
* @return bool Success
*/
protected function cacheRecordFailure($lockDb)
{
if ($this->statusCache && $this->safeDelay > 0) {
$path = $this->getMissKey($lockDb);
$misses = $this->statusCache->get($path);
if ($misses) {
return $this->statusCache->incr($path);
} else {
return $this->statusCache->add($path, 1, $this->safeDelay);
}
}
return true;
}
/**
* @covers BagOStuff::getMulti
*/
public function testGetMulti()
{
$value1 = array('this' => 'is', 'a' => 'test');
$value2 = array('this' => 'is', 'another' => 'test');
$key1 = wfMemcKey('test1');
$key2 = wfMemcKey('test2');
$this->cache->add($key1, $value1);
$this->cache->add($key2, $value2);
$this->assertEquals($this->cache->getMulti(array($key1, $key2)), array($key1 => $value1, $key2 => $value2));
// cleanup
$this->cache->delete($key1);
$this->cache->delete($key2);
}
/**
* Get an authenticated connection handle to the Swift proxy
*
* @throws CloudFilesException
* @throws CloudFilesException|Exception
* @return CF_Connection|bool False on failure
*/
protected function getConnection()
{
if ($this->connException instanceof CloudFilesException) {
if (time() - $this->connErrorTime < 60) {
throw $this->connException;
// failed last attempt; don't bother
} else {
// actually retry this time
$this->connException = null;
$this->connErrorTime = 0;
}
}
// Session keys expire after a while, so we renew them periodically
$reAuth = time() - $this->sessionStarted > $this->authTTL;
// Authenticate with proxy and get a session key...
if (!$this->conn || $reAuth) {
$this->sessionStarted = 0;
$this->connContainerCache->clear();
$cacheKey = $this->getCredsCacheKey($this->auth->username);
$creds = $this->srvCache->get($cacheKey);
// credentials
if (is_array($creds)) {
// cache hit
$this->auth->load_cached_credentials($creds['auth_token'], $creds['storage_url'], $creds['cdnm_url']);
$this->sessionStarted = time() - ceil($this->authTTL / 2);
// skew for worst case
} else {
// cache miss
try {
$this->auth->authenticate();
$creds = $this->auth->export_credentials();
$this->srvCache->add($cacheKey, $creds, ceil($this->authTTL / 2));
// cache
$this->sessionStarted = time();
} catch (CloudFilesException $e) {
$this->connException = $e;
// don't keep re-trying
$this->connErrorTime = time();
throw $e;
// throw it back
}
}
if ($this->conn) {
// re-authorizing?
$this->conn->close();
// close active cURL handles in CF_Http object
}
$this->conn = new CF_Connection($this->auth);
}
return $this->conn;
}
protected function doIsEmpty()
{
$key = $this->getCacheKey('empty');
$isEmpty = $this->cache->get($key);
if ($isEmpty === 'true') {
return true;
} elseif ($isEmpty === 'false') {
return false;
}
foreach ($this->partitionQueues as $queue) {
if (!$queue->doIsEmpty()) {
$this->cache->add($key, 'false', self::CACHE_TTL_LONG);
return false;
}
}
$this->cache->add($key, 'true', self::CACHE_TTL_LONG);
return true;
}
/**
* @see JobQueue::doIsEmpty()
* @return bool
*/
protected function doIsEmpty()
{
$key = $this->getCacheKey('empty');
$isEmpty = $this->cache->get($key);
if ($isEmpty === 'true') {
return true;
} elseif ($isEmpty === 'false') {
return false;
}
$dbr = $this->getSlaveDB();
try {
$found = $dbr->selectField('job', '1', array('job_cmd' => $this->type, 'job_token' => ''), __METHOD__);
} catch (DBError $e) {
$this->throwDBException($e);
}
$this->cache->add($key, $found ? 'false' : 'true', self::CACHE_TTL_LONG);
return !$found;
}
/**
* @covers BagOStuff::getMulti
*/
public function testGetMulti()
{
$value1 = array('this' => 'is', 'a' => 'test');
$value2 = array('this' => 'is', 'another' => 'test');
$value3 = array('testing a key that may be encoded when sent to cache backend');
$key1 = wfMemcKey('test1');
$key2 = wfMemcKey('test2');
$key3 = wfMemcKey('will-%-encode');
// internally, MemcachedBagOStuffs will encode to will-%25-encode
$this->cache->add($key1, $value1);
$this->cache->add($key2, $value2);
$this->cache->add($key3, $value3);
$this->assertEquals(array($key1 => $value1, $key2 => $value2, $key3 => $value3), $this->cache->getMulti(array($key1, $key2, $key3)));
// cleanup
$this->cache->delete($key1);
$this->cache->delete($key2);
$this->cache->delete($key3);
}
/**
* Set the cached stat info for a file path.
* Negatives (404s) are not cached. By not caching negatives, we can skip cache
* salting for the case when a file is created at a path were there was none before.
*
* @param string $path Storage path
* @param array $val Stat information to cache
*/
protected final function setFileCache($path, array $val)
{
$path = FileBackend::normalizeStoragePath($path);
if ($path === null) {
return;
// invalid storage path
}
$age = time() - wfTimestamp(TS_UNIX, $val['mtime']);
$ttl = min(7 * 86400, max(300, floor(0.1 * $age)));
$key = $this->fileCacheKey($path);
// Set the cache unless it is currently salted with the value "PURGED".
// Using add() handles this except it also is a no-op in that case where
// the current value is not "latest" but $val is, so use CAS in that case.
if (!$this->memCache->add($key, $val, $ttl) && !empty($val['latest'])) {
$this->memCache->merge($key, function (BagOStuff $cache, $key, $cValue) use($val) {
return is_array($cValue) && empty($cValue['latest']) ? $val : false;
// do nothing (cache is salted or some error happened)
}, $ttl, 1);
}
}
/**
* Represents a write lock on the messages key.
*
* Will retry MessageCache::MSG_WAIT_TIMEOUT times, each operations having
* a timeout of MessageCache::MSG_LOCK_TIMEOUT.
*
* @param string $key
* @return bool Success
*/
function lock($key)
{
$lockKey = $key . ':lock';
$acquired = false;
$testDone = false;
for ($i = 0; $i < MSG_WAIT_TIMEOUT && !$acquired; $i++) {
$acquired = $this->mMemc->add($lockKey, 1, MSG_LOCK_TIMEOUT);
if ($acquired) {
break;
}
# Fail fast if memcached is totally down
if (!$testDone) {
$testDone = true;
if (!$this->mMemc->set(wfMemcKey('test'), 'test', 1)) {
break;
}
}
sleep(1);
}
return $acquired;
}
/**
* @covers BagOStuff::getMulti
*/
public function testGetMulti()
{
$value1 = ['this' => 'is', 'a' => 'test'];
$value2 = ['this' => 'is', 'another' => 'test'];
$value3 = ['testing a key that may be encoded when sent to cache backend'];
$value4 = ['another test where chars in key will be encoded'];
$key1 = wfMemcKey('test1');
$key2 = wfMemcKey('test2');
// internally, MemcachedBagOStuffs will encode to will-%25-encode
$key3 = wfMemcKey('will-%-encode');
$key4 = wfMemcKey('flowdb:flow_ref:wiki:by-source:v3:Parser\'s_"broken"_+_(page)_&_grill:testwiki:1:4.7');
$this->cache->add($key1, $value1);
$this->cache->add($key2, $value2);
$this->cache->add($key3, $value3);
$this->cache->add($key4, $value4);
$this->assertEquals([$key1 => $value1, $key2 => $value2, $key3 => $value3, $key4 => $value4], $this->cache->getMulti([$key1, $key2, $key3, $key4]));
// cleanup
$this->cache->delete($key1);
$this->cache->delete($key2);
$this->cache->delete($key3);
$this->cache->delete($key4);
}
protected function doIsEmpty()
{
$key = $this->getCacheKey('empty');
$isEmpty = $this->cache->get($key);
if ($isEmpty === 'true') {
return true;
} elseif ($isEmpty === 'false') {
return false;
}
$empty = true;
$failed = 0;
foreach ($this->partitionQueues as $queue) {
try {
$empty = $empty && $queue->doIsEmpty();
} catch (JobQueueError $e) {
++$failed;
MWExceptionHandler::logException($e);
}
}
$this->throwErrorIfAllPartitionsDown($failed);
$this->cache->add($key, $empty ? 'true' : 'false', self::CACHE_TTL_LONG);
return $empty;
}
protected function doIsEmpty() {
$key = $this->getCacheKey( 'empty' );
$isEmpty = $this->cache->get( $key );
if ( $isEmpty === 'true' ) {
return true;
} elseif ( $isEmpty === 'false' ) {
return false;
}
foreach ( $this->partitionQueues as $queue ) {
try {
if ( !$queue->doIsEmpty() ) {
$this->cache->add( $key, 'false', self::CACHE_TTL_LONG );
return false;
}
} catch ( JobQueueError $e ) {
MWExceptionHandler::logException( $e );
}
}
$this->cache->add( $key, 'true', self::CACHE_TTL_LONG );
return true;
}
/**
* Set the cached stat info for a file path.
* Negatives (404s) are not cached. By not caching negatives, we can skip cache
* salting for the case when a file is created at a path were there was none before.
*
* @param $path string Storage path
* @param $val mixed Information to cache
*/
protected final function setFileCache($path, $val)
{
$this->memCache->add($this->fileCacheKey($path), $val, 7 * 86400);
}
public function add($key, $value, $exptime = 0)
{
return $this->writeStore->add($key, $value, $exptime);
}