protected function setUp() { global $wgFileBackends; parent::setUp(); $uniqueId = time() . '-' . mt_rand(); $tmpPrefix = wfTempDir() . '/filebackend-unittest-' . $uniqueId; if ($this->getCliArg('use-filebackend=')) { if (self::$backendToUse) { $this->singleBackend = self::$backendToUse; } else { $name = $this->getCliArg('use-filebackend='); $useConfig = array(); foreach ($wgFileBackends as $conf) { if ($conf['name'] == $name) { $useConfig = $conf; break; } } $useConfig['name'] = 'localtesting'; // swap name $useConfig['shardViaHashLevels'] = array('unittest-cont1' => array('levels' => 1, 'base' => 16, 'repeat' => 1)); if (isset($useConfig['fileJournal'])) { $useConfig['fileJournal'] = FileJournal::factory($useConfig['fileJournal'], $name); } $useConfig['lockManager'] = LockManagerGroup::singleton()->get($useConfig['lockManager']); $class = $useConfig['class']; self::$backendToUse = new $class($useConfig); $this->singleBackend = self::$backendToUse; } } else { $this->singleBackend = new FSFileBackend(array('name' => 'localtesting', 'lockManager' => LockManagerGroup::singleton()->get('fsLockManager'), 'wikiId' => wfWikiID(), 'containerPaths' => array('unittest-cont1' => "{$tmpPrefix}-localtesting-cont1", 'unittest-cont2' => "{$tmpPrefix}-localtesting-cont2"))); } $this->multiBackend = new FileBackendMultiWrite(array('name' => 'localtesting', 'lockManager' => LockManagerGroup::singleton()->get('fsLockManager'), 'parallelize' => 'implicit', 'wikiId' => wfWikiId() . $uniqueId, 'backends' => array(array('name' => 'localmultitesting1', 'class' => 'FSFileBackend', 'containerPaths' => array('unittest-cont1' => "{$tmpPrefix}-localtestingmulti1-cont1", 'unittest-cont2' => "{$tmpPrefix}-localtestingmulti1-cont2"), 'isMultiMaster' => false), array('name' => 'localmultitesting2', 'class' => 'FSFileBackend', 'containerPaths' => array('unittest-cont1' => "{$tmpPrefix}-localtestingmulti2-cont1", 'unittest-cont2' => "{$tmpPrefix}-localtestingmulti2-cont2"), 'isMultiMaster' => true)))); $this->filesToPrune = array(); }
protected function setUp() { global $wgFileBackends; parent::setUp(); $tmpDir = $this->getNewTempDirectory(); if ($this->getCliArg('use-filebackend')) { if (self::$backendToUse) { $this->singleBackend = self::$backendToUse; } else { $name = $this->getCliArg('use-filebackend'); $useConfig = []; foreach ($wgFileBackends as $conf) { if ($conf['name'] == $name) { $useConfig = $conf; break; } } $useConfig['name'] = 'localtesting'; // swap name $useConfig['shardViaHashLevels'] = ['unittest-cont1' => ['levels' => 1, 'base' => 16, 'repeat' => 1]]; if (isset($useConfig['fileJournal'])) { $useConfig['fileJournal'] = FileJournal::factory($useConfig['fileJournal'], $name); } $useConfig['lockManager'] = LockManagerGroup::singleton()->get($useConfig['lockManager']); $class = $useConfig['class']; self::$backendToUse = new $class($useConfig); $this->singleBackend = self::$backendToUse; } } else { $this->singleBackend = new FSFileBackend(['name' => 'localtesting', 'lockManager' => LockManagerGroup::singleton()->get('fsLockManager'), 'wikiId' => wfWikiID(), 'containerPaths' => ['unittest-cont1' => "{$tmpDir}/localtesting-cont1", 'unittest-cont2' => "{$tmpDir}/localtesting-cont2"]]); } $this->multiBackend = new FileBackendMultiWrite(['name' => 'localtesting', 'lockManager' => LockManagerGroup::singleton()->get('fsLockManager'), 'parallelize' => 'implicit', 'wikiId' => wfWikiID() . wfRandomString(), 'backends' => [['name' => 'localmultitesting1', 'class' => 'FSFileBackend', 'containerPaths' => ['unittest-cont1' => "{$tmpDir}/localtestingmulti1-cont1", 'unittest-cont2' => "{$tmpDir}/localtestingmulti1-cont2"], 'isMultiMaster' => false], ['name' => 'localmultitesting2', 'class' => 'FSFileBackend', 'containerPaths' => ['unittest-cont1' => "{$tmpDir}/localtestingmulti2-cont1", 'unittest-cont2' => "{$tmpDir}/localtestingmulti2-cont2"], 'isMultiMaster' => true]]]); }
/** * Get the backend object with a given name * * @param string $name * @return FileBackend * @throws FileBackendException */ public function get($name) { if (!isset($this->backends[$name])) { throw new FileBackendException("No backend defined with the name `{$name}`."); } // Lazy-load the actual backend instance if (!isset($this->backends[$name]['instance'])) { $class = $this->backends[$name]['class']; $config = $this->backends[$name]['config']; $config['wikiId'] = isset($config['wikiId']) ? $config['wikiId'] : wfWikiID(); // e.g. "my_wiki-en_" $config['lockManager'] = LockManagerGroup::singleton($config['wikiId'])->get($config['lockManager']); $config['fileJournal'] = isset($config['fileJournal']) ? FileJournal::factory($config['fileJournal'], $name) : FileJournal::factory(array('class' => 'NullFileJournal'), $name); $config['wanCache'] = ObjectCache::getMainWANInstance(); $this->backends[$name]['instance'] = new $class($config); } return $this->backends[$name]['instance']; }
/** * Create a new backend instance from configuration. * This should only be called from within FileBackendGroup. * * $config includes: * - name : The unique name of this backend. * This should consist of alphanumberic, '-', and '_' characters. * This name should not be changed after use (e.g. with journaling). * Note that the name is *not* used in actual container names. * - wikiId : Prefix to container names that is unique to this backend. * If not provided, this defaults to the current wiki ID. * It should only consist of alphanumberic, '-', and '_' characters. * This ID is what avoids collisions if multiple logical backends * use the same storage system, so this should be set carefully. * - lockManager : Registered name of a file lock manager to use. * - fileJournal : File journal configuration; see FileJournal::factory(). * Journals simply log changes to files stored in the backend. * - readOnly : Write operations are disallowed if this is a non-empty string. * It should be an explanation for the backend being read-only. * - parallelize : When to do file operations in parallel (when possible). * Allowed values are "implicit", "explicit" and "off". * - concurrency : How many file operations can be done in parallel. * * @param array $config * @throws MWException */ public function __construct( array $config ) { $this->name = $config['name']; if ( !preg_match( '!^[a-zA-Z0-9-_]{1,255}$!', $this->name ) ) { throw new MWException( "Backend name `{$this->name}` is invalid." ); } $this->wikiId = isset( $config['wikiId'] ) ? $config['wikiId'] : wfWikiID(); // e.g. "my_wiki-en_" $this->lockManager = ( $config['lockManager'] instanceof LockManager ) ? $config['lockManager'] : LockManagerGroup::singleton( $this->wikiId )->get( $config['lockManager'] ); $this->fileJournal = isset( $config['fileJournal'] ) ? ( ( $config['fileJournal'] instanceof FileJournal ) ? $config['fileJournal'] : FileJournal::factory( $config['fileJournal'], $this->name ) ) : FileJournal::factory( array( 'class' => 'NullFileJournal' ), $this->name ); $this->readOnly = isset( $config['readOnly'] ) ? (string)$config['readOnly'] : ''; $this->parallelize = isset( $config['parallelize'] ) ? (string)$config['parallelize'] : 'off'; $this->concurrency = isset( $config['concurrency'] ) ? (int)$config['concurrency'] : 50; }
/** * Construct a new instance from configuration. * $config includes: * 'wiki' : wiki name to use for LoadBalancer * * @param $config Array */ protected function __construct(array $config) { parent::__construct($config); $this->wiki = $config['wiki']; }
/** * Attempt to perform a series of file operations. * Callers are responsible for handling file locking. * * $opts is an array of options, including: * - force : Errors that would normally cause a rollback do not. * The remaining operations are still attempted if any fail. * - nonJournaled : Don't log this operation batch in the file journal. * - concurrency : Try to do this many operations in parallel when possible. * * The resulting Status will be "OK" unless: * - a) unexpected operation errors occurred (network partitions, disk full...) * - b) significant operation errors occurred and 'force' was not set * * @param array $performOps List of FileOp operations * @param array $opts Batch operation options * @param FileJournal $journal Journal to log operations to * @return Status */ public static function attempt( array $performOps, array $opts, FileJournal $journal ) { wfProfileIn( __METHOD__ ); $status = Status::newGood(); $n = count( $performOps ); if ( $n > self::MAX_BATCH_SIZE ) { $status->fatal( 'backend-fail-batchsize', $n, self::MAX_BATCH_SIZE ); wfProfileOut( __METHOD__ ); return $status; } $batchId = $journal->getTimestampedUUID(); $ignoreErrors = !empty( $opts['force'] ); $journaled = empty( $opts['nonJournaled'] ); $maxConcurrency = isset( $opts['concurrency'] ) ? $opts['concurrency'] : 1; $entries = array(); // file journal entry list $predicates = FileOp::newPredicates(); // account for previous ops in prechecks $curBatch = array(); // concurrent FileOp sub-batch accumulation $curBatchDeps = FileOp::newDependencies(); // paths used in FileOp sub-batch $pPerformOps = array(); // ordered list of concurrent FileOp sub-batches $lastBackend = null; // last op backend name // Do pre-checks for each operation; abort on failure... foreach ( $performOps as $index => $fileOp ) { $backendName = $fileOp->getBackend()->getName(); $fileOp->setBatchId( $batchId ); // transaction ID // Decide if this op can be done concurrently within this sub-batch // or if a new concurrent sub-batch must be started after this one... if ( $fileOp->dependsOn( $curBatchDeps ) || count( $curBatch ) >= $maxConcurrency || ( $backendName !== $lastBackend && count( $curBatch ) ) ) { $pPerformOps[] = $curBatch; // push this batch $curBatch = array(); // start a new sub-batch $curBatchDeps = FileOp::newDependencies(); } $lastBackend = $backendName; $curBatch[$index] = $fileOp; // keep index // Update list of affected paths in this batch $curBatchDeps = $fileOp->applyDependencies( $curBatchDeps ); // Simulate performing the operation... $oldPredicates = $predicates; $subStatus = $fileOp->precheck( $predicates ); // updates $predicates $status->merge( $subStatus ); if ( $subStatus->isOK() ) { if ( $journaled ) { // journal log entries $entries = array_merge( $entries, $fileOp->getJournalEntries( $oldPredicates, $predicates ) ); } } else { // operation failed? $status->success[$index] = false; ++$status->failCount; if ( !$ignoreErrors ) { wfProfileOut( __METHOD__ ); return $status; // abort } } } // Push the last sub-batch if ( count( $curBatch ) ) { $pPerformOps[] = $curBatch; } // Log the operations in the file journal... if ( count( $entries ) ) { $subStatus = $journal->logChangeBatch( $entries, $batchId ); if ( !$subStatus->isOK() ) { wfProfileOut( __METHOD__ ); return $subStatus; // abort } } if ( $ignoreErrors ) { // treat precheck() fatals as mere warnings $status->setResult( true, $status->value ); } // Attempt each operation (in parallel if allowed and possible)... self::runParallelBatches( $pPerformOps, $status ); wfProfileOut( __METHOD__ ); return $status; }
/** * Create a new backend instance from configuration. * This should only be called from within FileBackendGroup. * * @param array $config Parameters include: * - name : The unique name of this backend. * This should consist of alphanumberic, '-', and '_' characters. * This name should not be changed after use (e.g. with journaling). * Note that the name is *not* used in actual container names. * - wikiId : Prefix to container names that is unique to this backend. * It should only consist of alphanumberic, '-', and '_' characters. * This ID is what avoids collisions if multiple logical backends * use the same storage system, so this should be set carefully. * - lockManager : LockManager object to use for any file locking. * If not provided, then no file locking will be enforced. * - fileJournal : FileJournal object to use for logging changes to files. * If not provided, then change journaling will be disabled. * - readOnly : Write operations are disallowed if this is a non-empty string. * It should be an explanation for the backend being read-only. * - parallelize : When to do file operations in parallel (when possible). * Allowed values are "implicit", "explicit" and "off". * - concurrency : How many file operations can be done in parallel. * @throws FileBackendException */ public function __construct(array $config) { $this->name = $config['name']; if (!preg_match('!^[a-zA-Z0-9-_]{1,255}$!', $this->name)) { throw new FileBackendException("Backend name `{$this->name}` is invalid."); } if (!isset($config['wikiId'])) { $config['wikiId'] = wfWikiID(); wfDeprecated(__METHOD__ . ' called without "wikiID".', '1.23'); } if (isset($config['lockManager']) && !is_object($config['lockManager'])) { $config['lockManager'] = LockManagerGroup::singleton($config['wikiId'])->get($config['lockManager']); wfDeprecated(__METHOD__ . ' called with non-object "lockManager".', '1.23'); } $this->wikiId = $config['wikiId']; // e.g. "my_wiki-en_" $this->lockManager = isset($config['lockManager']) ? $config['lockManager'] : new NullLockManager(array()); $this->fileJournal = isset($config['fileJournal']) ? $config['fileJournal'] : FileJournal::factory(array('class' => 'NullFileJournal'), $this->name); $this->readOnly = isset($config['readOnly']) ? (string) $config['readOnly'] : ''; $this->parallelize = isset($config['parallelize']) ? (string) $config['parallelize'] : 'off'; $this->concurrency = isset($config['concurrency']) ? (int) $config['concurrency'] : 50; }
/** * Method enable or disable domain * * @param bool $enable if true, function enable domain, otherwise disable it * @return bool return TRUE on success, FALSE on failure */ function enable_domain($enable) { global $data; $data->add_method('enable_domain'); $data->add_method('reload_domains'); $errors = array(); FileJournal::clear(); if ($enable) { $opt['did'] = $this->did; $opt['disable'] = false; if (false === $this->create_or_remove_all_symlinks(true, $errors)) { FileJournal::rollback(); ErrorHandler::add_error($errors); return false; } } else { $opt['did'] = $this->did; $opt['disable'] = true; if (false === $this->create_or_remove_all_symlinks(false, $errors)) { FileJournal::rollback(); ErrorHandler::add_error($errors); return false; } } if (false === $data->enable_domain($opt)) { FileJournal::rollback(); return false; } /* notify SER to reload domains */ if (false === $data->reload_domains(null, $errors)) { ErrorHandler::add_error($errors); return false; } return true; }
/** * Create a new backend instance from configuration. * This should only be called from within FileBackendGroup. * * @param array $config Parameters include: * - name : The unique name of this backend. * This should consist of alphanumberic, '-', and '_' characters. * This name should not be changed after use (e.g. with journaling). * Note that the name is *not* used in actual container names. * - wikiId : Prefix to container names that is unique to this backend. * It should only consist of alphanumberic, '-', and '_' characters. * This ID is what avoids collisions if multiple logical backends * use the same storage system, so this should be set carefully. * - lockManager : LockManager object to use for any file locking. * If not provided, then no file locking will be enforced. * - fileJournal : FileJournal object to use for logging changes to files. * If not provided, then change journaling will be disabled. * - readOnly : Write operations are disallowed if this is a non-empty string. * It should be an explanation for the backend being read-only. * - parallelize : When to do file operations in parallel (when possible). * Allowed values are "implicit", "explicit" and "off". * - concurrency : How many file operations can be done in parallel. * @throws FileBackendException */ public function __construct(array $config) { $this->name = $config['name']; $this->wikiId = $config['wikiId']; // e.g. "my_wiki-en_" if (!preg_match('!^[a-zA-Z0-9-_]{1,255}$!', $this->name)) { throw new FileBackendException("Backend name '{$this->name}' is invalid."); } elseif (!is_string($this->wikiId)) { throw new FileBackendException("Backend wiki ID not provided for '{$this->name}'."); } $this->lockManager = isset($config['lockManager']) ? $config['lockManager'] : new NullLockManager(array()); $this->fileJournal = isset($config['fileJournal']) ? $config['fileJournal'] : FileJournal::factory(array('class' => 'NullFileJournal'), $this->name); $this->readOnly = isset($config['readOnly']) ? (string) $config['readOnly'] : ''; $this->parallelize = isset($config['parallelize']) ? (string) $config['parallelize'] : 'off'; $this->concurrency = isset($config['concurrency']) ? (int) $config['concurrency'] : 50; }
/** * Get the config array for a backend object with a given name * * @param string $name * @return array Parameters to FileBackend::__construct() * @throws InvalidArgumentException */ public function config($name) { if (!isset($this->backends[$name])) { throw new InvalidArgumentException("No backend defined with the name `{$name}`."); } $class = $this->backends[$name]['class']; $config = $this->backends[$name]['config']; $config['class'] = $class; $config += ['wikiId' => wfWikiID(), 'mimeCallback' => [$this, 'guessMimeInternal'], 'obResetFunc' => 'wfResetOutputBuffers', 'streamMimeFunc' => ['StreamFile', 'contentTypeFromPath'], 'tmpDirectory' => wfTempDir(), 'statusWrapper' => ['Status', 'wrap'], 'wanCache' => MediaWikiServices::getInstance()->getMainWANObjectCache(), 'srvCache' => ObjectCache::getLocalServerInstance('hash'), 'logger' => LoggerFactory::getInstance('FileOperation'), 'profiler' => Profiler::instance()]; $config['lockManager'] = LockManagerGroup::singleton($config['wikiId'])->get($config['lockManager']); $config['fileJournal'] = isset($config['fileJournal']) ? FileJournal::factory($config['fileJournal'], $name) : FileJournal::factory(['class' => 'NullFileJournal'], $name); return $config; }
function update_domain($domainname, $customer, &$errors) { global $data; $sem = new Shm_Semaphore(__FILE__, "s", 1, 0600); if (false === $data->transaction_start()) { return false; } FileJournal::clear(); /* set semaphore to be sure there will not be generated same domain id for two domains */ if (!$sem->acquire()) { $data->transaction_rollback(); return false; } if (false === $this->generate_domain_id($domainname, $errors)) { return false; } if (!empty($domainname)) { if (false === DomainManipulator::add_alias($this->id, $domainname, null)) { FileJournal::rollback(); $data->transaction_rollback(); $this->revert_domain_id(); $sem->release(); return false; } } $a_vals = array("owner_id" => $customer, "alias" => $domainname); if (false === DomainManipulator::update_domain_attrs($this->id, $a_vals)) { FileJournal::rollback(); $data->transaction_rollback(); $this->revert_domain_id(); $sem->release(); return false; } if (false === $data->transaction_commit()) { return false; } $sem->release(); return true; }
/** * Rollback changes * * This method for now only deleting created files - it don't rollback * deleted files. * * This method may be called staticaly e.g.: FileJournal::rollback(); * or dynamicaly e.g. $e = &FileJournal::singleton(); $e->rollback(); * * @return none */ function rollback() { if (isset($this) and is_a($this, 'FileJournal')) { $in =& $this; } else { $in =& FileJournal::singleton(); } foreach ($in->files_created as $file) { sw_log("FileJournal::rollback() - deleting file: " . $file, PEAR_LOG_DEBUG); if (false === rm($file)) { sw_log("Can't rollback created files. Can't delete file: " . $file, PEAR_LOG_ERR); } } }
/** * Create a new backend instance from configuration. * This should only be called from within FileBackendGroup. * * @param array $config Parameters include: * - name : The unique name of this backend. * This should consist of alphanumberic, '-', and '_' characters. * This name should not be changed after use (e.g. with journaling). * Note that the name is *not* used in actual container names. * - domainId : Prefix to container names that is unique to this backend. * It should only consist of alphanumberic, '-', and '_' characters. * This ID is what avoids collisions if multiple logical backends * use the same storage system, so this should be set carefully. * - lockManager : LockManager object to use for any file locking. * If not provided, then no file locking will be enforced. * - fileJournal : FileJournal object to use for logging changes to files. * If not provided, then change journaling will be disabled. * - readOnly : Write operations are disallowed if this is a non-empty string. * It should be an explanation for the backend being read-only. * - parallelize : When to do file operations in parallel (when possible). * Allowed values are "implicit", "explicit" and "off". * - concurrency : How many file operations can be done in parallel. * - tmpDirectory : Directory to use for temporary files. If this is not set or null, * then the backend will try to discover a usable temporary directory. * - obResetFunc : alternative callback to clear the output buffer * - streamMimeFunc : alternative method to determine the content type from the path * - logger : Optional PSR logger object. * - profiler : Optional class name or object With profileIn/profileOut methods. * @throws InvalidArgumentException */ public function __construct(array $config) { $this->name = $config['name']; $this->domainId = isset($config['domainId']) ? $config['domainId'] : $config['wikiId']; // b/c alias if (!preg_match('!^[a-zA-Z0-9-_]{1,255}$!', $this->name)) { throw new InvalidArgumentException("Backend name '{$this->name}' is invalid."); } elseif (!is_string($this->domainId)) { throw new InvalidArgumentException("Backend domain ID not provided for '{$this->name}'."); } $this->lockManager = isset($config['lockManager']) ? $config['lockManager'] : new NullLockManager([]); $this->fileJournal = isset($config['fileJournal']) ? $config['fileJournal'] : FileJournal::factory(['class' => 'NullFileJournal'], $this->name); $this->readOnly = isset($config['readOnly']) ? (string) $config['readOnly'] : ''; $this->parallelize = isset($config['parallelize']) ? (string) $config['parallelize'] : 'off'; $this->concurrency = isset($config['concurrency']) ? (int) $config['concurrency'] : 50; $this->obResetFunc = isset($config['obResetFunc']) ? $config['obResetFunc'] : [$this, 'resetOutputBuffer']; $this->streamMimeFunc = isset($config['streamMimeFunc']) ? $config['streamMimeFunc'] : null; $this->statusWrapper = isset($config['statusWrapper']) ? $config['statusWrapper'] : null; $this->profiler = isset($config['profiler']) ? $config['profiler'] : null; $this->logger = isset($config['logger']) ? $config['logger'] : new \Psr\Log\NullLogger(); $this->statusWrapper = isset($config['statusWrapper']) ? $config['statusWrapper'] : null; $this->tmpDirectory = isset($config['tmpDirectory']) ? $config['tmpDirectory'] : null; }