public function execute() { $backend = FileBackendGroup::singleton()->get($this->getOption('b1')); $this->doPerfTest($backend); if ($this->getOption('b2')) { $backend = FileBackendGroup::singleton()->get($this->getOption('b2')); $this->doPerfTest($backend); } }
public function execute() { $backend = FileBackendGroup::singleton()->get($this->getOption('b1')); $this->doPerfTest($backend); if ($this->getOption('b2')) { $backend = FileBackendGroup::singleton()->get($this->getOption('b2')); $this->doPerfTest($backend); } $profiler = Profiler::instance(); $profiler->setTemplated(true); //NOTE: as of MW1.21, $profiler->logData() is called implicitly by doMaintenance.php. }
public function execute() { $backend = FileBackendGroup::singleton()->get($this->getOption('b1')); $this->doPerfTest($backend); if ($this->getOption('b2')) { $backend = FileBackendGroup::singleton()->get($this->getOption('b2')); $this->doPerfTest($backend); } $profiler = Profiler::instance(); $profiler->setTemplated(true); $profiler->logData(); // prints }
/** * @return FileBackend */ public function getBackend() { global $wgCaptchaFileBackend, $wgCaptchaDirectory; if ($wgCaptchaFileBackend) { return FileBackendGroup::singleton()->get($wgCaptchaFileBackend); } else { static $backend = null; if (!$backend) { $backend = new FSFileBackend(array('name' => 'captcha-backend', 'wikiId' => wfWikiId(), 'lockManager' => new NullLockManager(array()), 'containerPaths' => array('captcha-render' => $wgCaptchaDirectory), 'fileMode' => 777)); } return $backend; } }
/** * @param $info array|null * @throws MWException */ public function __construct(array $info = null) { // Verify required settings presence if ($info === null || !array_key_exists('name', $info) || !array_key_exists('backend', $info)) { throw new MWException(__CLASS__ . " requires an array of options having both 'name' and 'backend' keys.\n"); } // Required settings $this->name = $info['name']; if ($info['backend'] instanceof FileBackend) { $this->backend = $info['backend']; // useful for testing } else { $this->backend = FileBackendGroup::singleton()->get($info['backend']); } // Optional settings that can have no value $optionalSettings = array('descBaseUrl', 'scriptDirUrl', 'articleUrl', 'fetchDescription', 'thumbScriptUrl', 'pathDisclosureProtection', 'descriptionCacheExpiry', 'scriptExtension'); foreach ($optionalSettings as $var) { if (isset($info[$var])) { $this->{$var} = $info[$var]; } } // Optional settings that have a default $this->initialCapital = isset($info['initialCapital']) ? $info['initialCapital'] : MWNamespace::isCapitalized(NS_FILE); $this->url = isset($info['url']) ? $info['url'] : false; // a subclass may set the URL (e.g. ForeignAPIRepo) if (isset($info['thumbUrl'])) { $this->thumbUrl = $info['thumbUrl']; } else { $this->thumbUrl = $this->url ? "{$this->url}/thumb" : false; } $this->hashLevels = isset($info['hashLevels']) ? $info['hashLevels'] : 2; $this->deletedHashLevels = isset($info['deletedHashLevels']) ? $info['deletedHashLevels'] : $this->hashLevels; $this->transformVia404 = !empty($info['transformVia404']); $this->abbrvThreshold = isset($info['abbrvThreshold']) ? $info['abbrvThreshold'] : 255; $this->isPrivate = !empty($info['isPrivate']); // Give defaults for the basic zones... $this->zones = isset($info['zones']) ? $info['zones'] : array(); foreach (array('public', 'thumb', 'transcoded', 'temp', 'deleted') as $zone) { if (!isset($this->zones[$zone]['container'])) { $this->zones[$zone]['container'] = "{$this->name}-{$zone}"; } if (!isset($this->zones[$zone]['directory'])) { $this->zones[$zone]['directory'] = ''; } if (!isset($this->zones[$zone]['urlsByExt'])) { $this->zones[$zone]['urlsByExt'] = array(); } } }
public function execute() { $src = FileBackendGroup::singleton()->get($this->getOption('src')); $dst = FileBackendGroup::singleton()->get($this->getOption('dst')); $posDir = $this->getOption('posdir'); $posFile = $posDir ? $posDir . '/' . wfWikiID() : false; $start = $this->getOption('start', 0); if (!$start && $posFile && is_dir($posDir)) { $start = is_file($posFile) ? (int) trim(file_get_contents($posFile)) : 0; ++$start; // we already did this ID, start with the next one $startFromPosFile = true; } else { $startFromPosFile = false; } $end = $this->getOption('end', INF); $this->output("Synchronizing backend '{$dst->getName()}' to '{$src->getName()}'...\n"); $this->output("Starting journal position is {$start}.\n"); if (is_finite($end)) { $this->output("Ending journal position is {$end}.\n"); } // Actually sync the dest backend with the reference backend $lastOKPos = $this->syncBackends($src, $dst, $start, $end); // Update the sync position file if ($startFromPosFile && $lastOKPos >= $start) { // successfully advanced if (file_put_contents($posFile, $lastOKPos, LOCK_EX) !== false) { $this->output("Updated journal position file.\n"); } else { $this->output("Could not update journal position file.\n"); } } if ($lastOKPos === false) { if (!$start) { $this->output("No journal entries found.\n"); } else { $this->output("No new journal entries found.\n"); } } else { $this->output("Stopped synchronization at journal position {$lastOKPos}.\n"); } if ($this->isQuiet()) { print $lastOKPos; // give a single machine-readable number } }
/** * @see ExternalStoreMedium::store() */ public function store($backend, $data) { $be = FileBackendGroup::singleton()->get($backend); if ($be instanceof FileBackend) { // Get three random base 36 characters to act as shard directories $rand = wfBaseConvert(mt_rand(0, 46655), 10, 36, 3); // Make sure ID is roughly lexicographically increasing for performance $id = str_pad(UIDGenerator::newTimestampedUID128(32), 26, '0', STR_PAD_LEFT); // Segregate items by wiki ID for the sake of bookkeeping $wiki = isset($this->params['wiki']) ? $this->params['wiki'] : wfWikiID(); $url = $be->getContainerStoragePath('data') . '/' . rawurlencode($wiki) . "/{$rand[0]}/{$rand[1]}/{$rand[2]}/{$id}"; $be->prepare(array('dir' => dirname($url), 'noAccess' => 1, 'noListing' => 1)); if ($be->create(array('dst' => $url, 'content' => $data))->isOK()) { return $url; } } return false; }
function wfImageAuthMain() { global $wgImgAuthUrlPathMap; $request = RequestContext::getMain()->getRequest(); $publicWiki = in_array('read', User::getGroupPermissions(array('*')), true); // Get the requested file path (source file or thumbnail) $matches = WebRequest::getPathInfo(); if (!isset($matches['title'])) { wfForbidden('img-auth-accessdenied', 'img-auth-nopathinfo'); return; } $path = $matches['title']; if ($path && $path[0] !== '/') { // Make sure $path has a leading / $path = "/" . $path; } // Check for bug 28235: QUERY_STRING overriding the correct extension $whitelist = array(); $extension = FileBackend::extensionFromPath($path, 'rawcase'); if ($extension != '') { $whitelist[] = $extension; } if (!$request->checkUrlExtension($whitelist)) { return; } // Various extensions may have their own backends that need access. // Check if there is a special backend and storage base path for this file. foreach ($wgImgAuthUrlPathMap as $prefix => $storageDir) { $prefix = rtrim($prefix, '/') . '/'; // implicit trailing slash if (strpos($path, $prefix) === 0) { $be = FileBackendGroup::singleton()->backendFromPath($storageDir); $filename = $storageDir . substr($path, strlen($prefix)); // strip prefix // Check basic user authorization if (!RequestContext::getMain()->getUser()->isAllowed('read')) { wfForbidden('img-auth-accessdenied', 'img-auth-noread', $path); return; } if ($be->fileExists(array('src' => $filename))) { wfDebugLog('img_auth', "Streaming `" . $filename . "`."); $be->streamFile(array('src' => $filename), array('Cache-Control: private', 'Vary: Cookie')); } else { wfForbidden('img-auth-accessdenied', 'img-auth-nofile', $path); } return; } } // Get the local file repository $repo = RepoGroup::singleton()->getRepo('local'); $zone = strstr(ltrim($path, '/'), '/', true); // Get the full file storage path and extract the source file name. // (e.g. 120px-Foo.png => Foo.png or page2-120px-Foo.png => Foo.png). // This only applies to thumbnails/transcoded, and each of them should // be under a folder that has the source file name. if ($zone === 'thumb' || $zone === 'transcoded') { $name = wfBaseName(dirname($path)); $filename = $repo->getZonePath($zone) . substr($path, strlen("/" . $zone)); // Check to see if the file exists if (!$repo->fileExists($filename)) { wfForbidden('img-auth-accessdenied', 'img-auth-nofile', $filename); return; } } else { $name = wfBaseName($path); // file is a source file $filename = $repo->getZonePath('public') . $path; // Check to see if the file exists and is not deleted $bits = explode('!', $name, 2); if (substr($path, 0, 9) === '/archive/' && count($bits) == 2) { $file = $repo->newFromArchiveName($bits[1], $name); } else { $file = $repo->newFile($name); } if (!$file->exists() || $file->isDeleted(File::DELETED_FILE)) { wfForbidden('img-auth-accessdenied', 'img-auth-nofile', $filename); return; } } $headers = array(); // extra HTTP headers to send if (!$publicWiki) { // For private wikis, run extra auth checks and set cache control headers $headers[] = 'Cache-Control: private'; $headers[] = 'Vary: Cookie'; $title = Title::makeTitleSafe(NS_FILE, $name); if (!$title instanceof Title) { // files have valid titles wfForbidden('img-auth-accessdenied', 'img-auth-badtitle', $name); return; } // Run hook for extension authorization plugins /** @var $result array */ $result = null; if (!wfRunHooks('ImgAuthBeforeStream', array(&$title, &$path, &$name, &$result))) { wfForbidden($result[0], $result[1], array_slice($result, 2)); return; } // Check user authorization for this title // Checks Whitelist too if (!$title->userCan('read')) { wfForbidden('img-auth-accessdenied', 'img-auth-noread', $name); return; } } if ($request->getCheck('download')) { $headers[] = 'Content-Disposition: attachment'; } // Stream the requested file wfDebugLog('img_auth', "Streaming `" . $filename . "`."); $repo->streamFile($filename, $headers); }
/** * Construct a proxy backend that consists of several internal backends. * Locking, journaling, and read-only checks are handled by the proxy backend. * * Additional $config params include: * - backends : Array of backend config and multi-backend settings. * Each value is the config used in the constructor of a * FileBackendStore class, but with these additional settings: * - class : The name of the backend class * - isMultiMaster : This must be set for one backend. * - template: : If given a backend name, this will use * the config of that backend as a template. * Values specified here take precedence. * - syncChecks : Integer bitfield of internal backend sync checks to perform. * Possible bits include the FileBackendMultiWrite::CHECK_* constants. * There are constants for SIZE, TIME, and SHA1. * The checks are done before allowing any file operations. * - autoResync : Automatically resync the clone backends to the master backend * when pre-operation sync checks fail. This should only be used * if the master backend is stable and not missing any files. * Use "conservative" to limit resyncing to copying newer master * backend files over older (or non-existing) clone backend files. * Cases that cannot be handled will result in operation abortion. * - noPushQuickOps : (hack) Only apply doQuickOperations() to the master backend. * - noPushDirConts : (hack) Only apply directory functions to the master backend. * * @param Array $config * @throws MWException */ public function __construct( array $config ) { parent::__construct( $config ); $this->syncChecks = isset( $config['syncChecks'] ) ? $config['syncChecks'] : self::CHECK_SIZE; $this->autoResync = isset( $config['autoResync'] ) ? $config['autoResync'] : false; $this->noPushQuickOps = isset( $config['noPushQuickOps'] ) ? $config['noPushQuickOps'] : false; $this->noPushDirConts = isset( $config['noPushDirConts'] ) ? $config['noPushDirConts'] : array(); // Construct backends here rather than via registration // to keep these backends hidden from outside the proxy. $namesUsed = array(); foreach ( $config['backends'] as $index => $config ) { if ( isset( $config['template'] ) ) { // Config is just a modified version of a registered backend's. // This should only be used when that config is used only by this backend. $config = $config + FileBackendGroup::singleton()->config( $config['template'] ); } $name = $config['name']; if ( isset( $namesUsed[$name] ) ) { // don't break FileOp predicates throw new MWException( "Two or more backends defined with the name $name." ); } $namesUsed[$name] = 1; // Alter certain sub-backend settings for sanity unset( $config['readOnly'] ); // use proxy backend setting unset( $config['fileJournal'] ); // use proxy backend journal $config['wikiId'] = $this->wikiId; // use the proxy backend wiki ID $config['lockManager'] = 'nullLockManager'; // lock under proxy backend if ( !empty( $config['isMultiMaster'] ) ) { if ( $this->masterIndex >= 0 ) { throw new MWException( 'More than one master backend defined.' ); } $this->masterIndex = $index; // this is the "master" $config['fileJournal'] = $this->fileJournal; // log under proxy backend } // Create sub-backend object if ( !isset( $config['class'] ) ) { throw new MWException( 'No class given for a backend config.' ); } $class = $config['class']; $this->backends[$index] = new $class( $config ); } if ( $this->masterIndex < 0 ) { // need backends and must have a master throw new MWException( 'No master backend defined.' ); } }
public function execute() { $src = FileBackendGroup::singleton()->get($this->getOption('src')); $posDir = $this->getOption('posdir'); $posFile = $posDir ? $posDir . '/' . wfWikiID() : false; if ($this->hasOption('posdump')) { // Just dump the current position into the specified position dir if (!$this->hasOption('posdir')) { $this->error("Param posdir required!", 1); } if ($this->hasOption('postime')) { $id = (int) $src->getJournal()->getPositionAtTime($this->getOption('postime')); $this->output("Requested journal position is {$id}.\n"); } else { $id = (int) $src->getJournal()->getCurrentPosition(); $this->output("Current journal position is {$id}.\n"); } if (file_put_contents($posFile, $id, LOCK_EX) !== false) { $this->output("Saved journal position file.\n"); } else { $this->output("Could not save journal position file.\n"); } if ($this->isQuiet()) { print $id; // give a single machine-readable number } return; } if (!$this->hasOption('dst')) { $this->error("Param dst required!", 1); } $dst = FileBackendGroup::singleton()->get($this->getOption('dst')); $start = $this->getOption('start', 0); if (!$start && $posFile && is_dir($posDir)) { $start = is_file($posFile) ? (int) trim(file_get_contents($posFile)) : 0; ++$start; // we already did this ID, start with the next one $startFromPosFile = true; } else { $startFromPosFile = false; } if ($this->hasOption('backoff')) { $time = time() - $this->getOption('backoff', 0); $end = (int) $src->getJournal()->getPositionAtTime($time); } else { $end = $this->getOption('end', INF); } $this->output("Synchronizing backend '{$dst->getName()}' to '{$src->getName()}'...\n"); $this->output("Starting journal position is {$start}.\n"); if (is_finite($end)) { $this->output("Ending journal position is {$end}.\n"); } // Periodically update the position file $callback = function ($pos) use($startFromPosFile, $posFile, $start) { if ($startFromPosFile && $pos >= $start) { // successfully advanced file_put_contents($posFile, $pos, LOCK_EX); } }; // Actually sync the dest backend with the reference backend $lastOKPos = $this->syncBackends($src, $dst, $start, $end, $callback); // Update the sync position file if ($startFromPosFile && $lastOKPos >= $start) { // successfully advanced if (file_put_contents($posFile, $lastOKPos, LOCK_EX) !== false) { $this->output("Updated journal position file.\n"); } else { $this->output("Could not update journal position file.\n"); } } if ($lastOKPos === false) { if (!$start) { $this->output("No journal entries found.\n"); } else { $this->output("No new journal entries found.\n"); } } else { $this->output("Stopped synchronization at journal position {$lastOKPos}.\n"); } if ($this->isQuiet()) { print $lastOKPos; // give a single machine-readable number } }
public function execute() { $src = FileBackendGroup::singleton()->get( $this->getOption( 'src' ) ); $dst = FileBackendGroup::singleton()->get( $this->getOption( 'dst' ) ); $containers = explode( '|', $this->getOption( 'containers' ) ); $subDir = rtrim( $this->getOption( 'subdir', '' ), '/' ); $rateFile = $this->getOption( 'ratefile' ); if ( $this->hasOption( 'utf8only' ) && !extension_loaded( 'mbstring' ) ) { $this->error( "Cannot check for UTF-8, mbstring extension missing.", 1 ); // die } foreach ( $containers as $container ) { if ( $subDir != '' ) { $backendRel = "$container/$subDir"; $this->output( "Doing container '$container', directory '$subDir'...\n" ); } else { $backendRel = $container; $this->output( "Doing container '$container'...\n" ); } if ( $this->hasOption( 'missingonly' ) ) { $this->output( "\tBuilding list of missing files..." ); $srcPathsRel = $this->getListingDiffRel( $src, $dst, $backendRel ); $this->output( count( $srcPathsRel ) . " file(s) need to be copied.\n" ); } else { $srcPathsRel = $src->getFileList( array( 'dir' => $src->getRootStoragePath() . "/$backendRel", 'adviseStat' => true // avoid HEADs ) ); if ( $srcPathsRel === null ) { $this->error( "Could not list files in $container.", 1 ); // die } } if ( $this->getOption( 'prestat' ) && !$this->hasOption( 'missingonly' ) ) { // Build the stat cache for the destination files $this->output( "\tBuilding destination stat cache..." ); $dstPathsRel = $dst->getFileList( array( 'dir' => $dst->getRootStoragePath() . "/$backendRel", 'adviseStat' => true // avoid HEADs ) ); if ( $dstPathsRel === null ) { $this->error( "Could not list files in $container.", 1 ); // die } $this->statCache = array(); // clear foreach ( $dstPathsRel as $dstPathRel ) { $path = $dst->getRootStoragePath() . "/$backendRel/$dstPathRel"; $this->statCache[sha1( $path )] = $dst->getFileStat( array( 'src' => $path ) ); } $this->output( "done [" . count( $this->statCache ) . " file(s)]\n" ); } $this->output( "\tCopying file(s)...\n" ); $count = 0; $batchPaths = array(); foreach ( $srcPathsRel as $srcPathRel ) { // Check up on the rate file periodically to adjust the concurrency if ( $rateFile && ( !$count || ( $count % 500 ) == 0 ) ) { $this->mBatchSize = max( 1, (int)file_get_contents( $rateFile ) ); $this->output( "\tBatch size is now {$this->mBatchSize}.\n" ); } $batchPaths[$srcPathRel] = 1; // remove duplicates if ( count( $batchPaths ) >= $this->mBatchSize ) { $this->copyFileBatch( array_keys( $batchPaths ), $backendRel, $src, $dst ); $batchPaths = array(); // done } ++$count; } if ( count( $batchPaths ) ) { // left-overs $this->copyFileBatch( array_keys( $batchPaths ), $backendRel, $src, $dst ); $batchPaths = array(); // done } $this->output( "\tCopied $count file(s).\n" ); if ( $this->hasOption( 'syncviadelete' ) ) { $this->output( "\tBuilding list of excess destination files..." ); $delPathsRel = $this->getListingDiffRel( $dst, $src, $backendRel ); $this->output( count( $delPathsRel ) . " file(s) need to be deleted.\n" ); $this->output( "\tDeleting file(s)...\n" ); $count = 0; $batchPaths = array(); foreach ( $delPathsRel as $delPathRel ) { // Check up on the rate file periodically to adjust the concurrency if ( $rateFile && ( !$count || ( $count % 500 ) == 0 ) ) { $this->mBatchSize = max( 1, (int)file_get_contents( $rateFile ) ); $this->output( "\tBatch size is now {$this->mBatchSize}.\n" ); } $batchPaths[$delPathRel] = 1; // remove duplicates if ( count( $batchPaths ) >= $this->mBatchSize ) { $this->delFileBatch( array_keys( $batchPaths ), $backendRel, $dst ); $batchPaths = array(); // done } ++$count; } if ( count( $batchPaths ) ) { // left-overs $this->delFileBatch( array_keys( $batchPaths ), $backendRel, $dst ); $batchPaths = array(); // done } $this->output( "\tDeleted $count file(s).\n" ); } if ( $subDir != '' ) { $this->output( "Finished container '$container', directory '$subDir'.\n" ); } else { $this->output( "Finished container '$container'.\n" ); } } $this->output( "Done.\n" ); }
/** * Modify timeline extension to use Swift storage (BAC-893) * * @param FileBackend $backend * @param string $fname mwstore abstract path * @param string $hash file hash * @return bool true - it's a hook */ static function onBeforeRenderTimeline(&$backend, &$fname, $hash) { global $wgEnableSwiftFileBackend, $wgFSSwiftContainer; if (!empty($wgEnableSwiftFileBackend)) { $backend = FileBackendGroup::singleton()->get('swift-backend'); $fname = 'mwstore://' . $backend->getName() . "/{$wgFSSwiftContainer}/images/timeline/{$hash}"; } return true; }
function wfImageAuthMain() { global $wgImgAuthPublicTest, $wgImgAuthUrlPathMap, $wgRequest; // See if this is a public Wiki (no protections). if ($wgImgAuthPublicTest && in_array('read', User::getGroupPermissions(array('*')), true)) { // This is a public wiki, so disable this script (for private wikis only) wfForbidden('img-auth-accessdenied', 'img-auth-public'); return; } // Get the requested file path (source file or thumbnail) $matches = WebRequest::getPathInfo(); if (!isset($matches['title'])) { wfForbidden('img-auth-accessdenied', 'img-auth-nopathinfo'); return; } $path = $matches['title']; if ($path && $path[0] !== '/') { // Make sure $path has a leading / $path = "/" . $path; } // Check for bug 28235: QUERY_STRING overriding the correct extension $whitelist = array(); $extension = FileBackend::extensionFromPath($path, 'rawcase'); if ($extension != '') { $whitelist[] = $extension; } if (!$wgRequest->checkUrlExtension($whitelist)) { return; } // Various extensions may have their own backends that need access. // Check if there is a special backend and storage base path for this file. foreach ($wgImgAuthUrlPathMap as $prefix => $storageDir) { $prefix = rtrim($prefix, '/') . '/'; // implicit trailing slash if (strpos($path, $prefix) === 0) { $be = FileBackendGroup::singleton()->backendFromPath($storageDir); $filename = $storageDir . substr($path, strlen($prefix)); // strip prefix // Check basic user authorization if (!RequestContext::getMain()->getUser()->isAllowed('read')) { wfForbidden('img-auth-accessdenied', 'img-auth-noread', $path); return; } if ($be->fileExists(array('src' => $filename))) { wfDebugLog('img_auth', "Streaming `" . $filename . "`."); $be->streamFile(array('src' => $filename), array('Cache-Control: private', 'Vary: Cookie')); } else { wfForbidden('img-auth-accessdenied', 'img-auth-nofile', $path); } return; } } // Get the local file repository $repo = RepoGroup::singleton()->getRepo('local'); // Get the full file storage path and extract the source file name. // (e.g. 120px-Foo.png => Foo.png or page2-120px-Foo.png => Foo.png). // This only applies to thumbnails, and all thumbnails should // be under a folder that has the source file name. if (strpos($path, '/thumb/') === 0) { $name = wfBaseName(dirname($path)); // file is a thumbnail $filename = $repo->getZonePath('thumb') . substr($path, 6); // strip "/thumb" } else { $name = wfBaseName($path); // file is a source file $filename = $repo->getZonePath('public') . $path; } // Check to see if the file exists if (!$repo->fileExists($filename)) { wfForbidden('img-auth-accessdenied', 'img-auth-nofile', $filename); return; } $title = Title::makeTitleSafe(NS_FILE, $name); if (!$title instanceof Title) { // files have valid titles wfForbidden('img-auth-accessdenied', 'img-auth-badtitle', $name); return; } // Run hook for extension authorization plugins /** @var $result array */ $result = null; if (!wfRunHooks('ImgAuthBeforeStream', array(&$title, &$path, &$name, &$result))) { wfForbidden($result[0], $result[1], array_slice($result, 2)); return; } // Check user authorization for this title // Checks Whitelist too if (!$title->userCan('read')) { wfForbidden('img-auth-accessdenied', 'img-auth-noread', $name); return; } if ($wgRequest->getCheck('download')) { header('Content-Disposition: attachment'); } // Stream the requested file wfDebugLog('img_auth', "Streaming `" . $filename . "`."); $repo->streamFile($filename, array('Cache-Control: private', 'Vary: Cookie')); }
public function execute() { $src = FileBackendGroup::singleton()->get($this->getOption('src')); $dst = FileBackendGroup::singleton()->get($this->getOption('dst')); $containers = explode('|', $this->getOption('containers')); $subDir = $this->getOption(rtrim('subdir', '/'), ''); $rateFile = $this->getOption('ratefile'); if ($this->hasOption('utf8only') && !extension_loaded('mbstring')) { $this->error("Cannot check for UTF-8, mbstring extension missing.", 1); // die } $count = 0; foreach ($containers as $container) { if ($subDir != '') { $backendRel = "{$container}/{$subDir}"; $this->output("Doing container '{$container}', directory '{$subDir}'...\n"); } else { $backendRel = $container; $this->output("Doing container '{$container}'...\n"); } $srcPathsRel = $src->getFileList(array('dir' => $src->getRootStoragePath() . "/{$backendRel}")); if ($srcPathsRel === null) { $this->error("Could not list files in {$container}.", 1); // die } // Do a listing comparison if specified if ($this->hasOption('missingonly')) { $relFilesSrc = array(); $relFilesDst = array(); foreach ($srcPathsRel as $srcPathRel) { $relFilesSrc[] = $srcPathRel; } $dstPathsRel = $dst->getFileList(array('dir' => $dst->getRootStoragePath() . "/{$backendRel}")); if ($dstPathsRel === null) { $this->error("Could not list files in {$container}.", 1); // die } foreach ($dstPathsRel as $dstPathRel) { $relFilesDst[] = $dstPathRel; } // Only copy the missing files over in the next loop $srcPathsRel = array_diff($relFilesSrc, $relFilesDst); $this->output(count($srcPathsRel) . " file(s) need to be copied.\n"); unset($relFilesSrc); unset($relFilesDst); } $batchPaths = array(); foreach ($srcPathsRel as $srcPathRel) { // Check up on the rate file periodically to adjust the concurrency if ($rateFile && (!$count || $count % 500 == 0)) { $this->mBatchSize = max(1, (int) file_get_contents($rateFile)); $this->output("Batch size is now {$this->mBatchSize}.\n"); } $batchPaths[$srcPathRel] = 1; // remove duplicates if (count($batchPaths) >= $this->mBatchSize) { $this->copyFileBatch(array_keys($batchPaths), $backendRel, $src, $dst); $batchPaths = array(); // done } ++$count; } if (count($batchPaths)) { // left-overs $this->copyFileBatch(array_keys($batchPaths), $backendRel, $src, $dst); $batchPaths = array(); // done } if ($subDir != '') { $this->output("Finished container '{$container}', directory '{$subDir}'.\n"); } else { $this->output("Finished container '{$container}'.\n"); } } $this->output("Done [{$count} file(s)].\n"); }
/** * @param $timelinesrc string * @return string */ function wfRenderTimeline($timelinesrc) { global $wgUploadDirectory, $wgUploadPath, $wgArticlePath, $wgTmpDirectory, $wgRenderHashAppend; global $wgTimelineSettings; // Get the backend to store plot data and pngs if ($wgTimelineSettings->fileBackend != '') { $backend = FileBackendGroup::singleton()->get($wgTimelineSettings->fileBackend); } else { $backend = new FSFileBackend(array('name' => 'timeline-backend', 'lockManager' => 'nullLockManager', 'containerPaths' => array('timeline-render' => "{$wgUploadDirectory}/timeline"), 'fileMode' => 777)); } // Get a hash of the plot data $hash = md5($timelinesrc); if ($wgRenderHashAppend != '') { $hash = md5($hash . $wgRenderHashAppend); } // Storage destination path (excluding file extension) $fname = 'mwstore://' . $backend->getName() . "/timeline-render/{$hash}"; // Wikia change - begin wfRunHooks('BeforeRenderTimeline', [&$backend, &$fname, $hash]); // Wikia change - end $previouslyFailed = $backend->fileExists(array('src' => "{$fname}.err")); $previouslyRendered = $backend->fileExists(array('src' => "{$fname}.png")); if ($previouslyRendered) { $timestamp = $backend->getFileTimestamp(array('src' => "{$fname}.png")); $expired = $timestamp < $wgTimelineSettings->epochTimestamp; } else { $expired = false; } // Create a new .map, .png (or .gif), and .err file as needed... if ($expired || !$previouslyRendered && !$previouslyFailed) { if (!is_dir($wgTmpDirectory)) { mkdir($wgTmpDirectory, 0777); } $tmpFile = TempFSFile::factory('timeline_'); if ($tmpFile) { $tmpPath = $tmpFile->getPath(); file_put_contents($tmpPath, $timelinesrc); // store plot data to file // Get command for ploticus to read the user input and output an error, // map, and rendering (png or gif) file under the same dir as the temp file. $cmdline = wfEscapeShellArg($wgTimelineSettings->perlCommand, $wgTimelineSettings->timelineFile) . " -i " . wfEscapeShellArg($tmpPath) . " -m -P " . wfEscapeShellArg($wgTimelineSettings->ploticusCommand) . " -T " . wfEscapeShellArg($wgTmpDirectory) . " -A " . wfEscapeShellArg($wgArticlePath) . " -f " . wfEscapeShellArg($wgTimelineSettings->fontFile); // Actually run the command... wfDebug("Timeline cmd: {$cmdline}\n"); $retVal = null; $ret = wfShellExec($cmdline, $retVal); // Copy the output files into storage... // @TODO: store error files in another container or not at all? $opt = array('force' => 1, 'nonLocking' => 1, 'allowStale' => 1); // performance $backend->prepare(array('dir' => dirname($fname))); $backend->store(array('src' => "{$tmpPath}.map", 'dst' => "{$fname}.map"), $opt); $backend->store(array('src' => "{$tmpPath}.png", 'dst' => "{$fname}.png"), $opt); $backend->store(array('src' => "{$tmpPath}.err", 'dst' => "{$fname}.err"), $opt); } else { return "<div id=\"toc\" dir=\"ltr\"><tt>Timeline error. " . "Could not create temp file</tt></div>"; // ugh } if ($ret == "" || $retVal > 0) { // Message not localized, only relevant during install return "<div id=\"toc\" dir=\"ltr\"><tt>Timeline error. " . "Command line was: " . htmlspecialchars($cmdline) . "</tt></div>"; } } // Wikia change - begin if ($backend->fileExists(array('src' => "{$fname}.err", 'latest' => true))) { $err = $backend->getFileContents(array('src' => "{$fname}.err")); } else { $err = ''; } // Wikia change - end if ($err != "") { // Convert the error from poorly-sanitized HTML to plain text $err = strtr($err, array('</p><p>' => "\n\n", '<p>' => '', '</p>' => '', '<b>' => '', '</b>' => '', '<br>' => "\n")); $err = Sanitizer::decodeCharReferences($err); // Now convert back to HTML again $encErr = nl2br(htmlspecialchars($err)); $txt = "<div id=\"toc\" dir=\"ltr\"><tt>{$encErr}</tt></div>"; } else { // Wikia change - begin if ($backend->fileExists(array('src' => "{$fname}.map", 'latest' => true))) { $map = $backend->getFileContents(array('src' => "{$fname}.map")); } else { $map = ''; } // Wikia change - end $map = str_replace(' >', ' />', $map); $map = "<map name=\"timeline_" . htmlspecialchars($hash) . "\">{$map}</map>"; $map = easyTimelineFixMap($map); $url = "{$wgUploadPath}/timeline/{$hash}.png"; // Wikia change - begin $url = wfReplaceImageServer($url); // Wikia change - end $txt = $map . "<img usemap=\"#timeline_" . htmlspecialchars($hash) . "\" " . "src=\"" . htmlspecialchars($url) . "\">"; if ($expired) { // Replacing an older file, we may need to purge the old one. global $wgUseSquid; if ($wgUseSquid) { $u = new SquidUpdate(array($url)); $u->doUpdate(); } } } return $txt; }
public function execute() { $src = FileBackendGroup::singleton()->get($this->getOption('src')); $dst = FileBackendGroup::singleton()->get($this->getOption('dst')); $containers = explode('|', $this->getOption('containers')); $subDir = $this->getOption(rtrim('subdir', '/'), ''); $rateFile = $this->getOption('ratefile'); if ($this->hasOption('utf8only') && !extension_loaded('mbstring')) { $this->error("Cannot check for UTF-8, mbstring extension missing.", 1); // die } $count = 0; foreach ($containers as $container) { if ($subDir != '') { $backendRel = "{$container}/{$subDir}"; $this->output("Doing container '{$container}', directory '{$subDir}'...\n"); } else { $backendRel = $container; $this->output("Doing container '{$container}'...\n"); } $srcPathsRel = $src->getFileList(array('dir' => $src->getRootStoragePath() . "/{$backendRel}", 'adviseStat' => !$this->hasOption('missingonly'))); if ($srcPathsRel === null) { $this->error("Could not list files in {$container}.", 1); // die } if ($this->hasOption('missingonly')) { $dstPathsRel = $dst->getFileList(array('dir' => $dst->getRootStoragePath() . "/{$backendRel}")); if ($dstPathsRel === null) { $this->error("Could not list files in {$container}.", 1); // die } // Get the list of destination files $relFilesDstSha1 = array(); foreach ($dstPathsRel as $dstPathRel) { $relFilesDstSha1[sha1($dstPathRel)] = 1; } unset($dstPathsRel); // free // Get the list of missing files $missingPathsRel = array(); foreach ($srcPathsRel as $srcPathRel) { if (!isset($relFilesDstSha1[sha1($srcPathRel)])) { $missingPathsRel[] = $srcPathRel; } } unset($srcPathsRel); // free // Only copy the missing files over in the next loop $srcPathsRel = $missingPathsRel; $this->output(count($srcPathsRel) . " file(s) need to be copied.\n"); } elseif ($this->getOption('prestat')) { // Build the stat cache for the destination files $this->output("Building destination stat cache..."); $dstPathsRel = $dst->getFileList(array('dir' => $dst->getRootStoragePath() . "/{$backendRel}", 'adviseStat' => true)); if ($dstPathsRel === null) { $this->error("Could not list files in {$container}.", 1); // die } $this->statCache = array(); // clear foreach ($dstPathsRel as $dstPathRel) { $path = $dst->getRootStoragePath() . "/{$backendRel}/{$dstPathRel}"; $this->statCache[sha1($path)] = $dst->getFileStat(array('src' => $path)); } $this->output("done [" . count($this->statCache) . " file(s)]\n"); } $batchPaths = array(); foreach ($srcPathsRel as $srcPathRel) { // Check up on the rate file periodically to adjust the concurrency if ($rateFile && (!$count || $count % 500 == 0)) { $this->mBatchSize = max(1, (int) file_get_contents($rateFile)); $this->output("Batch size is now {$this->mBatchSize}.\n"); } $batchPaths[$srcPathRel] = 1; // remove duplicates if (count($batchPaths) >= $this->mBatchSize) { $this->copyFileBatch(array_keys($batchPaths), $backendRel, $src, $dst); $batchPaths = array(); // done } ++$count; } if (count($batchPaths)) { // left-overs $this->copyFileBatch(array_keys($batchPaths), $backendRel, $src, $dst); $batchPaths = array(); // done } if ($subDir != '') { $this->output("Finished container '{$container}', directory '{$subDir}'.\n"); } else { $this->output("Finished container '{$container}'.\n"); } } $this->output("Done [{$count} file(s)].\n"); }
/** * @brief remove thumbnails for avatar by cleaning up whole folder * * @author Krzysztof Krzyżaniak (eloy) <*****@*****.**> * @access private * * @return boolean status of operation */ private function purgeThumbnails() { global $wgAvatarsUseSwiftStorage, $wgBlogAvatarPath, $wgBlogAvatarSwiftContainer, $wgBlogAvatarSwiftPathPrefix; // get path to thumbnail folder wfProfileIn(__METHOD__); // dirty hack, should work in this case if (!empty($wgAvatarsUseSwiftStorage)) { $swift = $this->getSwiftStorage(); $backend = FileBackendGroup::singleton()->get('swift-backend'); $dir = sprintf('mwstore://swift-backend/%s%s%s', $wgBlogAvatarSwiftContainer, $wgBlogAvatarSwiftPathPrefix, $this->getLocalPath()); $dir = $this->getThumbPath($dir); $avatarRemotePath = sprintf("thumb%s", $this->getLocalPath()); $urls = []; $files = []; $iterator = $backend->getFileList(array('dir' => $dir)); foreach ($iterator as $file) { $files[] = sprintf("%s/%s", $avatarRemotePath, $file); } // deleting files on file system and creating an array of URLs to purge if (!empty($files)) { foreach ($files as $file) { $status = $swift->remove($file); if (!$status->isOk()) { wfDebugLog("avatar", __METHOD__ . ": {$file} exists but cannot be removed.\n", true); } else { $urls[] = wfReplaceImageServer($wgBlogAvatarPath) . "/{$file}"; wfDebugLog("avatar", __METHOD__ . ": {$file} removed.\n", true); } } } wfDebugLog("avatar", __METHOD__ . ": all thumbs removed.\n", true); } else { $dir = $this->getFullPath(); $dir = $this->getThumbPath($dir); if (is_dir($dir)) { $urls = []; $files = []; // copied from LocalFile->getThumbnails $handle = opendir($dir); if ($handle) { while (false !== ($file = readdir($handle))) { if ($file[0] != '.') { $files[] = $file; } } closedir($handle); } // partialy copied from LocalFile->purgeThumbnails() foreach ($files as $file) { // deleting files on file system @unlink("{$dir}/{$file}"); $urls[] = $this->getPurgeUrl('/thumb/') . "/{$file}"; wfDebugLog("avatar", __METHOD__ . ": removing {$dir}/{$file}\n", true); } } else { wfDebugLog("avatar", __METHOD__ . ": {$dir} exists but is not directory so not removed.\n", true); } wfDebugLog("avatar", __METHOD__ . ": all thumbs removed.\n", true); } // purging avatars urls SquidUpdate::purge($urls); wfProfileOut(__METHOD__); }
public static function provider_testGetContentType() { return [[null, false], [[FileBackendGroup::singleton(), 'guessMimeInternal'], true]]; }
public static function provider_testGetContentType() { return array(array(null, false), array(array(FileBackendGroup::singleton(), 'guessMimeInternal'), true)); }