/**
* @see FileBackendStore::__construct()
* Additional $config params include:
* - basePath : File system directory that holds containers.
* - containerPaths : Map of container names to custom file system directories.
* This should only be used for backwards-compatibility.
* - fileMode : Octal UNIX file permissions to use on files stored.
*/
public function __construct(array $config)
{
parent::__construct($config);
// Remove any possible trailing slash from directories
if (isset($config['basePath'])) {
$this->basePath = rtrim($config['basePath'], '/');
// remove trailing slash
} else {
$this->basePath = null;
// none; containers must have explicit paths
}
if (isset($config['containerPaths'])) {
$this->containerPaths = (array) $config['containerPaths'];
foreach ($this->containerPaths as &$path) {
$path = rtrim($path, '/');
// remove trailing slash
}
}
$this->fileMode = isset($config['fileMode']) ? $config['fileMode'] : 0644;
if (isset($config['fileOwner']) && function_exists('posix_getuid')) {
$this->fileOwner = $config['fileOwner'];
$info = posix_getpwuid(posix_getuid());
$this->currentUser = $info['name'];
// cache this, assuming it doesn't change
}
}
/**
* Get the SHA-1 of a file in storage when this operation is attempted
*
* @param $source string Storage path
* @param $predicates Array
* @return string|false
*/
protected final function fileSha1($source, array $predicates)
{
if (isset($predicates['sha1'][$source])) {
return $predicates['sha1'][$source];
// previous op assures this
} else {
$params = array('src' => $source, 'latest' => $this->useLatest);
return $this->backend->getFileSha1Base36($params);
}
}
/**
* @see FileBackendStore::__construct()
* Additional $config params include:
* swiftAuthUrl : Swift authentication server URL
* swiftUser : Swift user used by MediaWiki (account:username)
* swiftKey : Swift authentication key for the above user
* swiftAuthTTL : Swift authentication TTL (seconds)
* swiftAnonUser : Swift user used for end-user requests (account:username)
* shardViaHashLevels : Map of container names to sharding config with:
* 'base' : base of hash characters, 16 or 36
* 'levels' : the number of hash levels (and digits)
* 'repeat' : hash subdirectories are prefixed with all the
* parent hash directory names (e.g. "a/ab/abc")
*/
public function __construct(array $config)
{
parent::__construct($config);
// Required settings
$this->auth = new CF_Authentication($config['swiftUser'], $config['swiftKey'], null, $config['swiftAuthUrl']);
// Optional settings
$this->authTTL = isset($config['swiftAuthTTL']) ? $config['swiftAuthTTL'] : 120;
// some sane number
$this->swiftAnonUser = isset($config['swiftAnonUser']) ? $config['swiftAnonUser'] : '';
$this->shardViaHashLevels = isset($config['shardViaHashLevels']) ? $config['shardViaHashLevels'] : '';
}
/**
* Get the SHA-1 of a file in storage when this operation is attempted
*
* @param string $source Storage path
* @param array $predicates
* @return string|bool False on failure
*/
protected final function fileSha1($source, array $predicates)
{
if (isset($predicates['sha1'][$source])) {
return $predicates['sha1'][$source];
// previous op assures this
} elseif (isset($predicates['exists'][$source]) && !$predicates['exists'][$source]) {
return false;
// previous op assures this
} else {
$params = ['src' => $source, 'latest' => true];
return $this->backend->getFileSha1Base36($params);
}
}
/**
* @see FileBackendStore::__construct()
* Additional $config params include:
* basePath : File system directory that holds containers.
* containerPaths : Map of container names to custom file system directories.
* This should only be used for backwards-compatibility.
* fileMode : Octal UNIX file permissions to use on files stored.
*/
public function __construct(array $config)
{
parent::__construct($config);
// Remove any possible trailing slash from directories
if (isset($config['basePath'])) {
$this->basePath = rtrim($config['basePath'], '/');
// remove trailing slash
} else {
$this->basePath = null;
// none; containers must have explicit paths
}
if (isset($config['containerPaths'])) {
$this->containerPaths = (array) $config['containerPaths'];
foreach ($this->containerPaths as &$path) {
$path = rtrim($path, '/');
// remove trailing slash
}
}
$this->fileMode = isset($config['fileMode']) ? $config['fileMode'] : 0644;
}
/**
* @see FileBackendStore::__construct()
* Additional $config params include:
* swiftAuthUrl : Swift authentication server URL
* swiftUser : Swift user used by MediaWiki (account:username)
* swiftKey : Swift authentication key for the above user
* swiftAuthTTL : Swift authentication TTL (seconds)
* swiftAnonUser : Swift user used for end-user requests (account:username)
* shardViaHashLevels : Map of container names to sharding config with:
* 'base' : base of hash characters, 16 or 36
* 'levels' : the number of hash levels (and digits)
* 'repeat' : hash subdirectories are prefixed with all the
* parent hash directory names (e.g. "a/ab/abc")
* swiftTimeout : number of seconds timeout consistent with php-cloudfiles. Default: 10
*/
public function __construct(array $config)
{
parent::__construct($config);
// Required settings
$this->auth = new CF_Authentication($config['swiftUser'], $config['swiftKey'], null, $config['swiftAuthUrl']);
/* <Wikia> */
if (!empty($config['debug'])) {
$this->auth->setDebug($config['debug']);
}
$this->swiftTimeout = isset($config['swiftTimeout']) ? intval($config['swiftTimeout']) : 10;
/* </Wikia> */
// Optional settings
$this->authTTL = isset($config['swiftAuthTTL']) ? $config['swiftAuthTTL'] : 120;
// some sane number
$this->swiftAnonUser = isset($config['swiftAnonUser']) ? $config['swiftAnonUser'] : '';
$this->shardViaHashLevels = isset($config['shardViaHashLevels']) ? $config['shardViaHashLevels'] : '';
/* <Wikia> */
// caching credentials
if (!empty($config['cacheAuthInfo']) && $config['cacheAuthInfo'] === true) {
$this->srvCache = wfGetMainCache();
}
$this->srvCache = $this->srvCache ? $this->srvCache : new EmptyBagOStuff();
/* </Wikia> */
}
/**
* @see FileBackendStore::__construct()
* Additional $config params include:
* - swiftAuthUrl : Swift authentication server URL
* - swiftUser : Swift user used by MediaWiki (account:username)
* - swiftKey : Swift authentication key for the above user
* - swiftAuthTTL : Swift authentication TTL (seconds)
* - swiftTempUrlKey : Swift "X-Account-Meta-Temp-URL-Key" value on the account.
* Do not set this until it has been set in the backend.
* - swiftAnonUser : Swift user used for end-user requests (account:username).
* If set, then views of public containers are assumed to go
* through this user. If not set, then public containers are
* accessible to unauthenticated requests via ".r:*" in the ACL.
* - swiftUseCDN : Whether a Cloud Files Content Delivery Network is set up
* - swiftCDNExpiry : How long (in seconds) to store content in the CDN.
* If files may likely change, this should probably not exceed
* a few days. For example, deletions may take this long to apply.
* If object purging is enabled, however, this is not an issue.
* - swiftCDNPurgable : Whether object purge requests are allowed by the CDN.
* - shardViaHashLevels : Map of container names to sharding config with:
* - base : base of hash characters, 16 or 36
* - levels : the number of hash levels (and digits)
* - repeat : hash subdirectories are prefixed with all the
* parent hash directory names (e.g. "a/ab/abc")
* - cacheAuthInfo : Whether to cache authentication tokens in APC, XCache, ect.
* If those are not available, then the main cache will be used.
* This is probably insecure in shared hosting environments.
* - rgwS3AccessKey : Ragos Gateway S3 "access key" value on the account.
* Do not set this until it has been set in the backend.
* This is used for generating expiring pre-authenticated URLs.
* Only use this when using rgw and to work around
* http://tracker.newdream.net/issues/3454.
* - rgwS3SecretKey : Ragos Gateway S3 "secret key" value on the account.
* Do not set this until it has been set in the backend.
* This is used for generating expiring pre-authenticated URLs.
* Only use this when using rgw and to work around
* http://tracker.newdream.net/issues/3454.
*/
public function __construct(array $config)
{
parent::__construct($config);
if (!class_exists('CF_Constants')) {
throw new MWException('SwiftCloudFiles extension not installed.');
}
// Required settings
$this->auth = new CF_Authentication($config['swiftUser'], $config['swiftKey'], null, $config['swiftAuthUrl']);
// Optional settings
$this->authTTL = isset($config['swiftAuthTTL']) ? $config['swiftAuthTTL'] : 5 * 60;
// some sane number
$this->swiftAnonUser = isset($config['swiftAnonUser']) ? $config['swiftAnonUser'] : '';
$this->swiftTempUrlKey = isset($config['swiftTempUrlKey']) ? $config['swiftTempUrlKey'] : '';
$this->shardViaHashLevels = isset($config['shardViaHashLevels']) ? $config['shardViaHashLevels'] : '';
$this->swiftUseCDN = isset($config['swiftUseCDN']) ? $config['swiftUseCDN'] : false;
$this->swiftCDNExpiry = isset($config['swiftCDNExpiry']) ? $config['swiftCDNExpiry'] : 12 * 3600;
// 12 hours is safe (tokens last 24 hours per http://docs.openstack.org)
$this->swiftCDNPurgable = isset($config['swiftCDNPurgable']) ? $config['swiftCDNPurgable'] : true;
$this->rgwS3AccessKey = isset($config['rgwS3AccessKey']) ? $config['rgwS3AccessKey'] : '';
$this->rgwS3SecretKey = isset($config['rgwS3SecretKey']) ? $config['rgwS3SecretKey'] : '';
// Cache container information to mask latency
$this->memCache = wfGetMainCache();
// Process cache for container info
$this->connContainerCache = new ProcessCacheLRU(300);
// Cache auth token information to avoid RTTs
if (!empty($config['cacheAuthInfo'])) {
if (PHP_SAPI === 'cli') {
$this->srvCache = wfGetMainCache();
// preferrably memcached
} else {
try {
// look for APC, XCache, WinCache, ect...
$this->srvCache = ObjectCache::newAccelerator(array());
} catch (Exception $e) {
}
}
}
$this->srvCache = $this->srvCache ? $this->srvCache : new EmptyBagOStuff();
}
public function accept()
{
$rel = $this->getInnerIterator()->current();
// path relative to given directory
$path = $this->params['dir'] . "/{$rel}";
// full storage path
if ($this->backend->isSingleShardPathInternal($path)) {
return true;
// path is only on one shard; no issue with duplicates
} elseif (isset($this->multiShardPaths[$rel])) {
// Don't keep listing paths that are on multiple shards
return false;
} else {
$this->multiShardPaths[$rel] = 1;
return true;
}
}
/**
* @see FileBackendStore::__construct()
* Additional $config params include:
* - swiftAuthUrl : Swift authentication server URL
* - swiftUser : Swift user used by MediaWiki (account:username)
* - swiftKey : Swift authentication key for the above user
* - swiftAuthTTL : Swift authentication TTL (seconds)
* - swiftTempUrlKey : Swift "X-Account-Meta-Temp-URL-Key" value on the account.
* Do not set this until it has been set in the backend.
* - shardViaHashLevels : Map of container names to sharding config with:
* - base : base of hash characters, 16 or 36
* - levels : the number of hash levels (and digits)
* - repeat : hash subdirectories are prefixed with all the
* parent hash directory names (e.g. "a/ab/abc")
* - cacheAuthInfo : Whether to cache authentication tokens in APC, XCache, ect.
* If those are not available, then the main cache will be used.
* This is probably insecure in shared hosting environments.
* - rgwS3AccessKey : Rados Gateway S3 "access key" value on the account.
* Do not set this until it has been set in the backend.
* This is used for generating expiring pre-authenticated URLs.
* Only use this when using rgw and to work around
* http://tracker.newdream.net/issues/3454.
* - rgwS3SecretKey : Rados Gateway S3 "secret key" value on the account.
* Do not set this until it has been set in the backend.
* This is used for generating expiring pre-authenticated URLs.
* Only use this when using rgw and to work around
* http://tracker.newdream.net/issues/3454.
*/
public function __construct(array $config)
{
parent::__construct($config);
// Required settings
$this->swiftAuthUrl = $config['swiftAuthUrl'];
$this->swiftUser = $config['swiftUser'];
$this->swiftKey = $config['swiftKey'];
// Optional settings
$this->authTTL = isset($config['swiftAuthTTL']) ? $config['swiftAuthTTL'] : 15 * 60;
// some sane number
$this->swiftTempUrlKey = isset($config['swiftTempUrlKey']) ? $config['swiftTempUrlKey'] : '';
$this->shardViaHashLevels = isset($config['shardViaHashLevels']) ? $config['shardViaHashLevels'] : '';
$this->rgwS3AccessKey = isset($config['rgwS3AccessKey']) ? $config['rgwS3AccessKey'] : '';
$this->rgwS3SecretKey = isset($config['rgwS3SecretKey']) ? $config['rgwS3SecretKey'] : '';
// HTTP helper client
$this->http = new MultiHttpClient(array());
// Cache container information to mask latency
if (isset($config['wanCache']) && $config['wanCache'] instanceof WANObjectCache) {
$this->memCache = $config['wanCache'];
}
// Process cache for container info
$this->containerStatCache = new ProcessCacheLRU(300);
// Cache auth token information to avoid RTTs
if (!empty($config['cacheAuthInfo'])) {
if (PHP_SAPI === 'cli') {
// Preferrably memcached
$this->srvCache = ObjectCache::getLocalClusterInstance();
} else {
// Look for APC, XCache, WinCache, ect...
$this->srvCache = ObjectCache::getLocalServerInstance(CACHE_NONE);
}
} else {
$this->srvCache = new EmptyBagOStuff();
}
}
/**
* Filter out duplicate items by advancing to the next ones
*/
protected function filterViaNext()
{
while ($this->valid()) {
$rel = $this->iter->current();
// path relative to given directory
$path = $this->params['dir'] . "/{$rel}";
// full storage path
if ($this->backend->isSingleShardPathInternal($path)) {
break;
// path is only on one shard; no issue with duplicates
} elseif (isset($this->multiShardPaths[$rel])) {
// Don't keep listing paths that are on multiple shards
$this->iter instanceof Iterator ? $this->iter->next() : next($this->iter);
} else {
$this->multiShardPaths[$rel] = 1;
break;
}
}
}
/**
* Substitute the backend of storage paths with an internal backend's name
*
* @param array|string $paths List of paths or single string path
* @param FileBackendStore $backend
* @return Array|string
*/
protected function substPaths( $paths, FileBackendStore $backend ) {
return preg_replace(
'!^mwstore://' . preg_quote( $this->name ) . '/!',
StringUtils::escapeRegexReplacement( "mwstore://{$backend->getName()}/" ),
$paths // string or array
);
}
/**
* @see FileBackendStore::__construct()
* Additional $config params include:
* - swiftAuthUrl : Swift authentication server URL
* - swiftUser : Swift user used by MediaWiki (account:username)
* - swiftKey : Swift authentication key for the above user
* - swiftAuthTTL : Swift authentication TTL (seconds)
* - swiftTempUrlKey : Swift "X-Account-Meta-Temp-URL-Key" value on the account.
* Do not set this until it has been set in the backend.
* - shardViaHashLevels : Map of container names to sharding config with:
* - base : base of hash characters, 16 or 36
* - levels : the number of hash levels (and digits)
* - repeat : hash subdirectories are prefixed with all the
* parent hash directory names (e.g. "a/ab/abc")
* - cacheAuthInfo : Whether to cache authentication tokens in APC, XCache, ect.
* If those are not available, then the main cache will be used.
* This is probably insecure in shared hosting environments.
* - rgwS3AccessKey : Rados Gateway S3 "access key" value on the account.
* Do not set this until it has been set in the backend.
* This is used for generating expiring pre-authenticated URLs.
* Only use this when using rgw and to work around
* http://tracker.newdream.net/issues/3454.
* - rgwS3SecretKey : Rados Gateway S3 "secret key" value on the account.
* Do not set this until it has been set in the backend.
* This is used for generating expiring pre-authenticated URLs.
* Only use this when using rgw and to work around
* http://tracker.newdream.net/issues/3454.
*/
public function __construct(array $config)
{
parent::__construct($config);
// Required settings
$this->swiftAuthUrl = $config['swiftAuthUrl'];
$this->swiftUser = $config['swiftUser'];
$this->swiftKey = $config['swiftKey'];
// Optional settings
$this->authTTL = isset($config['swiftAuthTTL']) ? $config['swiftAuthTTL'] : 5 * 60;
// some sane number
$this->swiftTempUrlKey = isset($config['swiftTempUrlKey']) ? $config['swiftTempUrlKey'] : '';
$this->shardViaHashLevels = isset($config['shardViaHashLevels']) ? $config['shardViaHashLevels'] : '';
$this->rgwS3AccessKey = isset($config['rgwS3AccessKey']) ? $config['rgwS3AccessKey'] : '';
$this->rgwS3SecretKey = isset($config['rgwS3SecretKey']) ? $config['rgwS3SecretKey'] : '';
// HTTP helper client
$this->http = new MultiHttpClient(array());
// Cache container information to mask latency
$this->memCache = wfGetMainCache();
// Process cache for container info
$this->containerStatCache = new ProcessCacheLRU(300);
// Cache auth token information to avoid RTTs
if (!empty($config['cacheAuthInfo'])) {
if (PHP_SAPI === 'cli') {
$this->srvCache = wfGetMainCache();
// preferrably memcached
} else {
try {
// look for APC, XCache, WinCache, ect...
$this->srvCache = ObjectCache::newAccelerator(array());
} catch (Exception $e) {
}
}
}
$this->srvCache = $this->srvCache ?: new EmptyBagOStuff();
}
/**
* @see FileBackend::__construct()
* Additional $config params include:
* azureHost : Windows Azure server URL
* azureAccount : Windows Azure user used by MediaWiki
* azureKey : Authentication key for the above user (used to get sessions)
* //azureContainer : Identifier of the container. (Optional. If not provided wikiId will be used as container name)
* containerPaths : Map of container names to Azure container names
*/
public function __construct(array $config)
{
parent::__construct($config);
$this->storageClient = new Microsoft_WindowsAzure_Storage_Blob($config['azureHost'], $config['azureAccount'], $config['azureKey']);
$this->containerPaths = (array) $config['containerPaths'];
}
