protected function setUp()
{
global $wgFileBackends;
parent::setUp();
$uniqueId = time() . '-' . mt_rand();
$tmpPrefix = wfTempDir() . '/filebackend-unittest-' . $uniqueId;
if ($this->getCliArg('use-filebackend=')) {
if (self::$backendToUse) {
$this->singleBackend = self::$backendToUse;
} else {
$name = $this->getCliArg('use-filebackend=');
$useConfig = array();
foreach ($wgFileBackends as $conf) {
if ($conf['name'] == $name) {
$useConfig = $conf;
break;
}
}
$useConfig['name'] = 'localtesting';
// swap name
$useConfig['shardViaHashLevels'] = array('unittest-cont1' => array('levels' => 1, 'base' => 16, 'repeat' => 1));
if (isset($useConfig['fileJournal'])) {
$useConfig['fileJournal'] = FileJournal::factory($useConfig['fileJournal'], $name);
}
$useConfig['lockManager'] = LockManagerGroup::singleton()->get($useConfig['lockManager']);
$class = $useConfig['class'];
self::$backendToUse = new $class($useConfig);
$this->singleBackend = self::$backendToUse;
}
} else {
$this->singleBackend = new FSFileBackend(array('name' => 'localtesting', 'lockManager' => LockManagerGroup::singleton()->get('fsLockManager'), 'wikiId' => wfWikiID(), 'containerPaths' => array('unittest-cont1' => "{$tmpPrefix}-localtesting-cont1", 'unittest-cont2' => "{$tmpPrefix}-localtesting-cont2")));
}
$this->multiBackend = new FileBackendMultiWrite(array('name' => 'localtesting', 'lockManager' => LockManagerGroup::singleton()->get('fsLockManager'), 'parallelize' => 'implicit', 'wikiId' => wfWikiId() . $uniqueId, 'backends' => array(array('name' => 'localmultitesting1', 'class' => 'FSFileBackend', 'containerPaths' => array('unittest-cont1' => "{$tmpPrefix}-localtestingmulti1-cont1", 'unittest-cont2' => "{$tmpPrefix}-localtestingmulti1-cont2"), 'isMultiMaster' => false), array('name' => 'localmultitesting2', 'class' => 'FSFileBackend', 'containerPaths' => array('unittest-cont1' => "{$tmpPrefix}-localtestingmulti2-cont1", 'unittest-cont2' => "{$tmpPrefix}-localtestingmulti2-cont2"), 'isMultiMaster' => true))));
$this->filesToPrune = array();
}
protected function setUp()
{
global $wgFileBackends;
parent::setUp();
$tmpDir = $this->getNewTempDirectory();
if ($this->getCliArg('use-filebackend')) {
if (self::$backendToUse) {
$this->singleBackend = self::$backendToUse;
} else {
$name = $this->getCliArg('use-filebackend');
$useConfig = [];
foreach ($wgFileBackends as $conf) {
if ($conf['name'] == $name) {
$useConfig = $conf;
break;
}
}
$useConfig['name'] = 'localtesting';
// swap name
$useConfig['shardViaHashLevels'] = ['unittest-cont1' => ['levels' => 1, 'base' => 16, 'repeat' => 1]];
if (isset($useConfig['fileJournal'])) {
$useConfig['fileJournal'] = FileJournal::factory($useConfig['fileJournal'], $name);
}
$useConfig['lockManager'] = LockManagerGroup::singleton()->get($useConfig['lockManager']);
$class = $useConfig['class'];
self::$backendToUse = new $class($useConfig);
$this->singleBackend = self::$backendToUse;
}
} else {
$this->singleBackend = new FSFileBackend(['name' => 'localtesting', 'lockManager' => LockManagerGroup::singleton()->get('fsLockManager'), 'wikiId' => wfWikiID(), 'containerPaths' => ['unittest-cont1' => "{$tmpDir}/localtesting-cont1", 'unittest-cont2' => "{$tmpDir}/localtesting-cont2"]]);
}
$this->multiBackend = new FileBackendMultiWrite(['name' => 'localtesting', 'lockManager' => LockManagerGroup::singleton()->get('fsLockManager'), 'parallelize' => 'implicit', 'wikiId' => wfWikiID() . wfRandomString(), 'backends' => [['name' => 'localmultitesting1', 'class' => 'FSFileBackend', 'containerPaths' => ['unittest-cont1' => "{$tmpDir}/localtestingmulti1-cont1", 'unittest-cont2' => "{$tmpDir}/localtestingmulti1-cont2"], 'isMultiMaster' => false], ['name' => 'localmultitesting2', 'class' => 'FSFileBackend', 'containerPaths' => ['unittest-cont1' => "{$tmpDir}/localtestingmulti2-cont1", 'unittest-cont2' => "{$tmpDir}/localtestingmulti2-cont2"], 'isMultiMaster' => true]]]);
}
/**
* Get the backend object with a given name
*
* @param string $name
* @return FileBackend
* @throws FileBackendException
*/
public function get($name)
{
if (!isset($this->backends[$name])) {
throw new FileBackendException("No backend defined with the name `{$name}`.");
}
// Lazy-load the actual backend instance
if (!isset($this->backends[$name]['instance'])) {
$class = $this->backends[$name]['class'];
$config = $this->backends[$name]['config'];
$config['wikiId'] = isset($config['wikiId']) ? $config['wikiId'] : wfWikiID();
// e.g. "my_wiki-en_"
$config['lockManager'] = LockManagerGroup::singleton($config['wikiId'])->get($config['lockManager']);
$config['fileJournal'] = isset($config['fileJournal']) ? FileJournal::factory($config['fileJournal'], $name) : FileJournal::factory(array('class' => 'NullFileJournal'), $name);
$config['wanCache'] = ObjectCache::getMainWANInstance();
$this->backends[$name]['instance'] = new $class($config);
}
return $this->backends[$name]['instance'];
}
/**
* Create a new backend instance from configuration.
* This should only be called from within FileBackendGroup.
*
* $config includes:
* - name : The unique name of this backend.
* This should consist of alphanumberic, '-', and '_' characters.
* This name should not be changed after use (e.g. with journaling).
* Note that the name is *not* used in actual container names.
* - wikiId : Prefix to container names that is unique to this backend.
* If not provided, this defaults to the current wiki ID.
* It should only consist of alphanumberic, '-', and '_' characters.
* This ID is what avoids collisions if multiple logical backends
* use the same storage system, so this should be set carefully.
* - lockManager : Registered name of a file lock manager to use.
* - fileJournal : File journal configuration; see FileJournal::factory().
* Journals simply log changes to files stored in the backend.
* - readOnly : Write operations are disallowed if this is a non-empty string.
* It should be an explanation for the backend being read-only.
* - parallelize : When to do file operations in parallel (when possible).
* Allowed values are "implicit", "explicit" and "off".
* - concurrency : How many file operations can be done in parallel.
*
* @param array $config
* @throws MWException
*/
public function __construct( array $config ) {
$this->name = $config['name'];
if ( !preg_match( '!^[a-zA-Z0-9-_]{1,255}$!', $this->name ) ) {
throw new MWException( "Backend name `{$this->name}` is invalid." );
}
$this->wikiId = isset( $config['wikiId'] )
? $config['wikiId']
: wfWikiID(); // e.g. "my_wiki-en_"
$this->lockManager = ( $config['lockManager'] instanceof LockManager )
? $config['lockManager']
: LockManagerGroup::singleton( $this->wikiId )->get( $config['lockManager'] );
$this->fileJournal = isset( $config['fileJournal'] )
? ( ( $config['fileJournal'] instanceof FileJournal )
? $config['fileJournal']
: FileJournal::factory( $config['fileJournal'], $this->name ) )
: FileJournal::factory( array( 'class' => 'NullFileJournal' ), $this->name );
$this->readOnly = isset( $config['readOnly'] )
? (string)$config['readOnly']
: '';
$this->parallelize = isset( $config['parallelize'] )
? (string)$config['parallelize']
: 'off';
$this->concurrency = isset( $config['concurrency'] )
? (int)$config['concurrency']
: 50;
}
/**
* Construct a new instance from configuration.
* $config includes:
* 'wiki' : wiki name to use for LoadBalancer
*
* @param $config Array
*/
protected function __construct(array $config)
{
parent::__construct($config);
$this->wiki = $config['wiki'];
}
/**
* Attempt to perform a series of file operations.
* Callers are responsible for handling file locking.
*
* $opts is an array of options, including:
* - force : Errors that would normally cause a rollback do not.
* The remaining operations are still attempted if any fail.
* - nonJournaled : Don't log this operation batch in the file journal.
* - concurrency : Try to do this many operations in parallel when possible.
*
* The resulting Status will be "OK" unless:
* - a) unexpected operation errors occurred (network partitions, disk full...)
* - b) significant operation errors occurred and 'force' was not set
*
* @param array $performOps List of FileOp operations
* @param array $opts Batch operation options
* @param FileJournal $journal Journal to log operations to
* @return Status
*/
public static function attempt( array $performOps, array $opts, FileJournal $journal ) {
wfProfileIn( __METHOD__ );
$status = Status::newGood();
$n = count( $performOps );
if ( $n > self::MAX_BATCH_SIZE ) {
$status->fatal( 'backend-fail-batchsize', $n, self::MAX_BATCH_SIZE );
wfProfileOut( __METHOD__ );
return $status;
}
$batchId = $journal->getTimestampedUUID();
$ignoreErrors = !empty( $opts['force'] );
$journaled = empty( $opts['nonJournaled'] );
$maxConcurrency = isset( $opts['concurrency'] ) ? $opts['concurrency'] : 1;
$entries = array(); // file journal entry list
$predicates = FileOp::newPredicates(); // account for previous ops in prechecks
$curBatch = array(); // concurrent FileOp sub-batch accumulation
$curBatchDeps = FileOp::newDependencies(); // paths used in FileOp sub-batch
$pPerformOps = array(); // ordered list of concurrent FileOp sub-batches
$lastBackend = null; // last op backend name
// Do pre-checks for each operation; abort on failure...
foreach ( $performOps as $index => $fileOp ) {
$backendName = $fileOp->getBackend()->getName();
$fileOp->setBatchId( $batchId ); // transaction ID
// Decide if this op can be done concurrently within this sub-batch
// or if a new concurrent sub-batch must be started after this one...
if ( $fileOp->dependsOn( $curBatchDeps )
|| count( $curBatch ) >= $maxConcurrency
|| ( $backendName !== $lastBackend && count( $curBatch ) )
) {
$pPerformOps[] = $curBatch; // push this batch
$curBatch = array(); // start a new sub-batch
$curBatchDeps = FileOp::newDependencies();
}
$lastBackend = $backendName;
$curBatch[$index] = $fileOp; // keep index
// Update list of affected paths in this batch
$curBatchDeps = $fileOp->applyDependencies( $curBatchDeps );
// Simulate performing the operation...
$oldPredicates = $predicates;
$subStatus = $fileOp->precheck( $predicates ); // updates $predicates
$status->merge( $subStatus );
if ( $subStatus->isOK() ) {
if ( $journaled ) { // journal log entries
$entries = array_merge( $entries,
$fileOp->getJournalEntries( $oldPredicates, $predicates ) );
}
} else { // operation failed?
$status->success[$index] = false;
++$status->failCount;
if ( !$ignoreErrors ) {
wfProfileOut( __METHOD__ );
return $status; // abort
}
}
}
// Push the last sub-batch
if ( count( $curBatch ) ) {
$pPerformOps[] = $curBatch;
}
// Log the operations in the file journal...
if ( count( $entries ) ) {
$subStatus = $journal->logChangeBatch( $entries, $batchId );
if ( !$subStatus->isOK() ) {
wfProfileOut( __METHOD__ );
return $subStatus; // abort
}
}
if ( $ignoreErrors ) { // treat precheck() fatals as mere warnings
$status->setResult( true, $status->value );
}
// Attempt each operation (in parallel if allowed and possible)...
self::runParallelBatches( $pPerformOps, $status );
wfProfileOut( __METHOD__ );
return $status;
}
/**
* Create a new backend instance from configuration.
* This should only be called from within FileBackendGroup.
*
* @param array $config Parameters include:
* - name : The unique name of this backend.
* This should consist of alphanumberic, '-', and '_' characters.
* This name should not be changed after use (e.g. with journaling).
* Note that the name is *not* used in actual container names.
* - wikiId : Prefix to container names that is unique to this backend.
* It should only consist of alphanumberic, '-', and '_' characters.
* This ID is what avoids collisions if multiple logical backends
* use the same storage system, so this should be set carefully.
* - lockManager : LockManager object to use for any file locking.
* If not provided, then no file locking will be enforced.
* - fileJournal : FileJournal object to use for logging changes to files.
* If not provided, then change journaling will be disabled.
* - readOnly : Write operations are disallowed if this is a non-empty string.
* It should be an explanation for the backend being read-only.
* - parallelize : When to do file operations in parallel (when possible).
* Allowed values are "implicit", "explicit" and "off".
* - concurrency : How many file operations can be done in parallel.
* @throws FileBackendException
*/
public function __construct(array $config)
{
$this->name = $config['name'];
if (!preg_match('!^[a-zA-Z0-9-_]{1,255}$!', $this->name)) {
throw new FileBackendException("Backend name `{$this->name}` is invalid.");
}
if (!isset($config['wikiId'])) {
$config['wikiId'] = wfWikiID();
wfDeprecated(__METHOD__ . ' called without "wikiID".', '1.23');
}
if (isset($config['lockManager']) && !is_object($config['lockManager'])) {
$config['lockManager'] = LockManagerGroup::singleton($config['wikiId'])->get($config['lockManager']);
wfDeprecated(__METHOD__ . ' called with non-object "lockManager".', '1.23');
}
$this->wikiId = $config['wikiId'];
// e.g. "my_wiki-en_"
$this->lockManager = isset($config['lockManager']) ? $config['lockManager'] : new NullLockManager(array());
$this->fileJournal = isset($config['fileJournal']) ? $config['fileJournal'] : FileJournal::factory(array('class' => 'NullFileJournal'), $this->name);
$this->readOnly = isset($config['readOnly']) ? (string) $config['readOnly'] : '';
$this->parallelize = isset($config['parallelize']) ? (string) $config['parallelize'] : 'off';
$this->concurrency = isset($config['concurrency']) ? (int) $config['concurrency'] : 50;
}
/**
* Method enable or disable domain
*
* @param bool $enable if true, function enable domain, otherwise disable it
* @return bool return TRUE on success, FALSE on failure
*/
function enable_domain($enable)
{
global $data;
$data->add_method('enable_domain');
$data->add_method('reload_domains');
$errors = array();
FileJournal::clear();
if ($enable) {
$opt['did'] = $this->did;
$opt['disable'] = false;
if (false === $this->create_or_remove_all_symlinks(true, $errors)) {
FileJournal::rollback();
ErrorHandler::add_error($errors);
return false;
}
} else {
$opt['did'] = $this->did;
$opt['disable'] = true;
if (false === $this->create_or_remove_all_symlinks(false, $errors)) {
FileJournal::rollback();
ErrorHandler::add_error($errors);
return false;
}
}
if (false === $data->enable_domain($opt)) {
FileJournal::rollback();
return false;
}
/* notify SER to reload domains */
if (false === $data->reload_domains(null, $errors)) {
ErrorHandler::add_error($errors);
return false;
}
return true;
}
/**
* Create a new backend instance from configuration.
* This should only be called from within FileBackendGroup.
*
* @param array $config Parameters include:
* - name : The unique name of this backend.
* This should consist of alphanumberic, '-', and '_' characters.
* This name should not be changed after use (e.g. with journaling).
* Note that the name is *not* used in actual container names.
* - wikiId : Prefix to container names that is unique to this backend.
* It should only consist of alphanumberic, '-', and '_' characters.
* This ID is what avoids collisions if multiple logical backends
* use the same storage system, so this should be set carefully.
* - lockManager : LockManager object to use for any file locking.
* If not provided, then no file locking will be enforced.
* - fileJournal : FileJournal object to use for logging changes to files.
* If not provided, then change journaling will be disabled.
* - readOnly : Write operations are disallowed if this is a non-empty string.
* It should be an explanation for the backend being read-only.
* - parallelize : When to do file operations in parallel (when possible).
* Allowed values are "implicit", "explicit" and "off".
* - concurrency : How many file operations can be done in parallel.
* @throws FileBackendException
*/
public function __construct(array $config)
{
$this->name = $config['name'];
$this->wikiId = $config['wikiId'];
// e.g. "my_wiki-en_"
if (!preg_match('!^[a-zA-Z0-9-_]{1,255}$!', $this->name)) {
throw new FileBackendException("Backend name '{$this->name}' is invalid.");
} elseif (!is_string($this->wikiId)) {
throw new FileBackendException("Backend wiki ID not provided for '{$this->name}'.");
}
$this->lockManager = isset($config['lockManager']) ? $config['lockManager'] : new NullLockManager(array());
$this->fileJournal = isset($config['fileJournal']) ? $config['fileJournal'] : FileJournal::factory(array('class' => 'NullFileJournal'), $this->name);
$this->readOnly = isset($config['readOnly']) ? (string) $config['readOnly'] : '';
$this->parallelize = isset($config['parallelize']) ? (string) $config['parallelize'] : 'off';
$this->concurrency = isset($config['concurrency']) ? (int) $config['concurrency'] : 50;
}
/**
* Get the config array for a backend object with a given name
*
* @param string $name
* @return array Parameters to FileBackend::__construct()
* @throws InvalidArgumentException
*/
public function config($name)
{
if (!isset($this->backends[$name])) {
throw new InvalidArgumentException("No backend defined with the name `{$name}`.");
}
$class = $this->backends[$name]['class'];
$config = $this->backends[$name]['config'];
$config['class'] = $class;
$config += ['wikiId' => wfWikiID(), 'mimeCallback' => [$this, 'guessMimeInternal'], 'obResetFunc' => 'wfResetOutputBuffers', 'streamMimeFunc' => ['StreamFile', 'contentTypeFromPath'], 'tmpDirectory' => wfTempDir(), 'statusWrapper' => ['Status', 'wrap'], 'wanCache' => MediaWikiServices::getInstance()->getMainWANObjectCache(), 'srvCache' => ObjectCache::getLocalServerInstance('hash'), 'logger' => LoggerFactory::getInstance('FileOperation'), 'profiler' => Profiler::instance()];
$config['lockManager'] = LockManagerGroup::singleton($config['wikiId'])->get($config['lockManager']);
$config['fileJournal'] = isset($config['fileJournal']) ? FileJournal::factory($config['fileJournal'], $name) : FileJournal::factory(['class' => 'NullFileJournal'], $name);
return $config;
}
function update_domain($domainname, $customer, &$errors)
{
global $data;
$sem = new Shm_Semaphore(__FILE__, "s", 1, 0600);
if (false === $data->transaction_start()) {
return false;
}
FileJournal::clear();
/* set semaphore to be sure there will not be generated same domain id for two domains */
if (!$sem->acquire()) {
$data->transaction_rollback();
return false;
}
if (false === $this->generate_domain_id($domainname, $errors)) {
return false;
}
if (!empty($domainname)) {
if (false === DomainManipulator::add_alias($this->id, $domainname, null)) {
FileJournal::rollback();
$data->transaction_rollback();
$this->revert_domain_id();
$sem->release();
return false;
}
}
$a_vals = array("owner_id" => $customer, "alias" => $domainname);
if (false === DomainManipulator::update_domain_attrs($this->id, $a_vals)) {
FileJournal::rollback();
$data->transaction_rollback();
$this->revert_domain_id();
$sem->release();
return false;
}
if (false === $data->transaction_commit()) {
return false;
}
$sem->release();
return true;
}
/**
* Rollback changes
*
* This method for now only deleting created files - it don't rollback
* deleted files.
*
* This method may be called staticaly e.g.: FileJournal::rollback();
* or dynamicaly e.g. $e = &FileJournal::singleton(); $e->rollback();
*
* @return none
*/
function rollback()
{
if (isset($this) and is_a($this, 'FileJournal')) {
$in =& $this;
} else {
$in =& FileJournal::singleton();
}
foreach ($in->files_created as $file) {
sw_log("FileJournal::rollback() - deleting file: " . $file, PEAR_LOG_DEBUG);
if (false === rm($file)) {
sw_log("Can't rollback created files. Can't delete file: " . $file, PEAR_LOG_ERR);
}
}
}
/**
* Create a new backend instance from configuration.
* This should only be called from within FileBackendGroup.
*
* @param array $config Parameters include:
* - name : The unique name of this backend.
* This should consist of alphanumberic, '-', and '_' characters.
* This name should not be changed after use (e.g. with journaling).
* Note that the name is *not* used in actual container names.
* - domainId : Prefix to container names that is unique to this backend.
* It should only consist of alphanumberic, '-', and '_' characters.
* This ID is what avoids collisions if multiple logical backends
* use the same storage system, so this should be set carefully.
* - lockManager : LockManager object to use for any file locking.
* If not provided, then no file locking will be enforced.
* - fileJournal : FileJournal object to use for logging changes to files.
* If not provided, then change journaling will be disabled.
* - readOnly : Write operations are disallowed if this is a non-empty string.
* It should be an explanation for the backend being read-only.
* - parallelize : When to do file operations in parallel (when possible).
* Allowed values are "implicit", "explicit" and "off".
* - concurrency : How many file operations can be done in parallel.
* - tmpDirectory : Directory to use for temporary files. If this is not set or null,
* then the backend will try to discover a usable temporary directory.
* - obResetFunc : alternative callback to clear the output buffer
* - streamMimeFunc : alternative method to determine the content type from the path
* - logger : Optional PSR logger object.
* - profiler : Optional class name or object With profileIn/profileOut methods.
* @throws InvalidArgumentException
*/
public function __construct(array $config)
{
$this->name = $config['name'];
$this->domainId = isset($config['domainId']) ? $config['domainId'] : $config['wikiId'];
// b/c alias
if (!preg_match('!^[a-zA-Z0-9-_]{1,255}$!', $this->name)) {
throw new InvalidArgumentException("Backend name '{$this->name}' is invalid.");
} elseif (!is_string($this->domainId)) {
throw new InvalidArgumentException("Backend domain ID not provided for '{$this->name}'.");
}
$this->lockManager = isset($config['lockManager']) ? $config['lockManager'] : new NullLockManager([]);
$this->fileJournal = isset($config['fileJournal']) ? $config['fileJournal'] : FileJournal::factory(['class' => 'NullFileJournal'], $this->name);
$this->readOnly = isset($config['readOnly']) ? (string) $config['readOnly'] : '';
$this->parallelize = isset($config['parallelize']) ? (string) $config['parallelize'] : 'off';
$this->concurrency = isset($config['concurrency']) ? (int) $config['concurrency'] : 50;
$this->obResetFunc = isset($config['obResetFunc']) ? $config['obResetFunc'] : [$this, 'resetOutputBuffer'];
$this->streamMimeFunc = isset($config['streamMimeFunc']) ? $config['streamMimeFunc'] : null;
$this->statusWrapper = isset($config['statusWrapper']) ? $config['statusWrapper'] : null;
$this->profiler = isset($config['profiler']) ? $config['profiler'] : null;
$this->logger = isset($config['logger']) ? $config['logger'] : new \Psr\Log\NullLogger();
$this->statusWrapper = isset($config['statusWrapper']) ? $config['statusWrapper'] : null;
$this->tmpDirectory = isset($config['tmpDirectory']) ? $config['tmpDirectory'] : null;
}