public function execute()
{
$backend = FileBackendGroup::singleton()->get($this->getOption('b1'));
$this->doPerfTest($backend);
if ($this->getOption('b2')) {
$backend = FileBackendGroup::singleton()->get($this->getOption('b2'));
$this->doPerfTest($backend);
}
}
public function execute()
{
$backend = FileBackendGroup::singleton()->get($this->getOption('b1'));
$this->doPerfTest($backend);
if ($this->getOption('b2')) {
$backend = FileBackendGroup::singleton()->get($this->getOption('b2'));
$this->doPerfTest($backend);
}
$profiler = Profiler::instance();
$profiler->setTemplated(true);
//NOTE: as of MW1.21, $profiler->logData() is called implicitly by doMaintenance.php.
}
public function execute()
{
$backend = FileBackendGroup::singleton()->get($this->getOption('b1'));
$this->doPerfTest($backend);
if ($this->getOption('b2')) {
$backend = FileBackendGroup::singleton()->get($this->getOption('b2'));
$this->doPerfTest($backend);
}
$profiler = Profiler::instance();
$profiler->setTemplated(true);
$profiler->logData();
// prints
}
/**
* @return FileBackend
*/
public function getBackend()
{
global $wgCaptchaFileBackend, $wgCaptchaDirectory;
if ($wgCaptchaFileBackend) {
return FileBackendGroup::singleton()->get($wgCaptchaFileBackend);
} else {
static $backend = null;
if (!$backend) {
$backend = new FSFileBackend(array('name' => 'captcha-backend', 'wikiId' => wfWikiId(), 'lockManager' => new NullLockManager(array()), 'containerPaths' => array('captcha-render' => $wgCaptchaDirectory), 'fileMode' => 777));
}
return $backend;
}
}
/**
* @param $info array|null
* @throws MWException
*/
public function __construct(array $info = null)
{
// Verify required settings presence
if ($info === null || !array_key_exists('name', $info) || !array_key_exists('backend', $info)) {
throw new MWException(__CLASS__ . " requires an array of options having both 'name' and 'backend' keys.\n");
}
// Required settings
$this->name = $info['name'];
if ($info['backend'] instanceof FileBackend) {
$this->backend = $info['backend'];
// useful for testing
} else {
$this->backend = FileBackendGroup::singleton()->get($info['backend']);
}
// Optional settings that can have no value
$optionalSettings = array('descBaseUrl', 'scriptDirUrl', 'articleUrl', 'fetchDescription', 'thumbScriptUrl', 'pathDisclosureProtection', 'descriptionCacheExpiry', 'scriptExtension');
foreach ($optionalSettings as $var) {
if (isset($info[$var])) {
$this->{$var} = $info[$var];
}
}
// Optional settings that have a default
$this->initialCapital = isset($info['initialCapital']) ? $info['initialCapital'] : MWNamespace::isCapitalized(NS_FILE);
$this->url = isset($info['url']) ? $info['url'] : false;
// a subclass may set the URL (e.g. ForeignAPIRepo)
if (isset($info['thumbUrl'])) {
$this->thumbUrl = $info['thumbUrl'];
} else {
$this->thumbUrl = $this->url ? "{$this->url}/thumb" : false;
}
$this->hashLevels = isset($info['hashLevels']) ? $info['hashLevels'] : 2;
$this->deletedHashLevels = isset($info['deletedHashLevels']) ? $info['deletedHashLevels'] : $this->hashLevels;
$this->transformVia404 = !empty($info['transformVia404']);
$this->abbrvThreshold = isset($info['abbrvThreshold']) ? $info['abbrvThreshold'] : 255;
$this->isPrivate = !empty($info['isPrivate']);
// Give defaults for the basic zones...
$this->zones = isset($info['zones']) ? $info['zones'] : array();
foreach (array('public', 'thumb', 'transcoded', 'temp', 'deleted') as $zone) {
if (!isset($this->zones[$zone]['container'])) {
$this->zones[$zone]['container'] = "{$this->name}-{$zone}";
}
if (!isset($this->zones[$zone]['directory'])) {
$this->zones[$zone]['directory'] = '';
}
if (!isset($this->zones[$zone]['urlsByExt'])) {
$this->zones[$zone]['urlsByExt'] = array();
}
}
}
public function execute()
{
$src = FileBackendGroup::singleton()->get($this->getOption('src'));
$dst = FileBackendGroup::singleton()->get($this->getOption('dst'));
$posDir = $this->getOption('posdir');
$posFile = $posDir ? $posDir . '/' . wfWikiID() : false;
$start = $this->getOption('start', 0);
if (!$start && $posFile && is_dir($posDir)) {
$start = is_file($posFile) ? (int) trim(file_get_contents($posFile)) : 0;
++$start;
// we already did this ID, start with the next one
$startFromPosFile = true;
} else {
$startFromPosFile = false;
}
$end = $this->getOption('end', INF);
$this->output("Synchronizing backend '{$dst->getName()}' to '{$src->getName()}'...\n");
$this->output("Starting journal position is {$start}.\n");
if (is_finite($end)) {
$this->output("Ending journal position is {$end}.\n");
}
// Actually sync the dest backend with the reference backend
$lastOKPos = $this->syncBackends($src, $dst, $start, $end);
// Update the sync position file
if ($startFromPosFile && $lastOKPos >= $start) {
// successfully advanced
if (file_put_contents($posFile, $lastOKPos, LOCK_EX) !== false) {
$this->output("Updated journal position file.\n");
} else {
$this->output("Could not update journal position file.\n");
}
}
if ($lastOKPos === false) {
if (!$start) {
$this->output("No journal entries found.\n");
} else {
$this->output("No new journal entries found.\n");
}
} else {
$this->output("Stopped synchronization at journal position {$lastOKPos}.\n");
}
if ($this->isQuiet()) {
print $lastOKPos;
// give a single machine-readable number
}
}
/**
* @see ExternalStoreMedium::store()
*/
public function store($backend, $data)
{
$be = FileBackendGroup::singleton()->get($backend);
if ($be instanceof FileBackend) {
// Get three random base 36 characters to act as shard directories
$rand = wfBaseConvert(mt_rand(0, 46655), 10, 36, 3);
// Make sure ID is roughly lexicographically increasing for performance
$id = str_pad(UIDGenerator::newTimestampedUID128(32), 26, '0', STR_PAD_LEFT);
// Segregate items by wiki ID for the sake of bookkeeping
$wiki = isset($this->params['wiki']) ? $this->params['wiki'] : wfWikiID();
$url = $be->getContainerStoragePath('data') . '/' . rawurlencode($wiki) . "/{$rand[0]}/{$rand[1]}/{$rand[2]}/{$id}";
$be->prepare(array('dir' => dirname($url), 'noAccess' => 1, 'noListing' => 1));
if ($be->create(array('dst' => $url, 'content' => $data))->isOK()) {
return $url;
}
}
return false;
}
function wfImageAuthMain()
{
global $wgImgAuthUrlPathMap;
$request = RequestContext::getMain()->getRequest();
$publicWiki = in_array('read', User::getGroupPermissions(array('*')), true);
// Get the requested file path (source file or thumbnail)
$matches = WebRequest::getPathInfo();
if (!isset($matches['title'])) {
wfForbidden('img-auth-accessdenied', 'img-auth-nopathinfo');
return;
}
$path = $matches['title'];
if ($path && $path[0] !== '/') {
// Make sure $path has a leading /
$path = "/" . $path;
}
// Check for bug 28235: QUERY_STRING overriding the correct extension
$whitelist = array();
$extension = FileBackend::extensionFromPath($path, 'rawcase');
if ($extension != '') {
$whitelist[] = $extension;
}
if (!$request->checkUrlExtension($whitelist)) {
return;
}
// Various extensions may have their own backends that need access.
// Check if there is a special backend and storage base path for this file.
foreach ($wgImgAuthUrlPathMap as $prefix => $storageDir) {
$prefix = rtrim($prefix, '/') . '/';
// implicit trailing slash
if (strpos($path, $prefix) === 0) {
$be = FileBackendGroup::singleton()->backendFromPath($storageDir);
$filename = $storageDir . substr($path, strlen($prefix));
// strip prefix
// Check basic user authorization
if (!RequestContext::getMain()->getUser()->isAllowed('read')) {
wfForbidden('img-auth-accessdenied', 'img-auth-noread', $path);
return;
}
if ($be->fileExists(array('src' => $filename))) {
wfDebugLog('img_auth', "Streaming `" . $filename . "`.");
$be->streamFile(array('src' => $filename), array('Cache-Control: private', 'Vary: Cookie'));
} else {
wfForbidden('img-auth-accessdenied', 'img-auth-nofile', $path);
}
return;
}
}
// Get the local file repository
$repo = RepoGroup::singleton()->getRepo('local');
$zone = strstr(ltrim($path, '/'), '/', true);
// Get the full file storage path and extract the source file name.
// (e.g. 120px-Foo.png => Foo.png or page2-120px-Foo.png => Foo.png).
// This only applies to thumbnails/transcoded, and each of them should
// be under a folder that has the source file name.
if ($zone === 'thumb' || $zone === 'transcoded') {
$name = wfBaseName(dirname($path));
$filename = $repo->getZonePath($zone) . substr($path, strlen("/" . $zone));
// Check to see if the file exists
if (!$repo->fileExists($filename)) {
wfForbidden('img-auth-accessdenied', 'img-auth-nofile', $filename);
return;
}
} else {
$name = wfBaseName($path);
// file is a source file
$filename = $repo->getZonePath('public') . $path;
// Check to see if the file exists and is not deleted
$bits = explode('!', $name, 2);
if (substr($path, 0, 9) === '/archive/' && count($bits) == 2) {
$file = $repo->newFromArchiveName($bits[1], $name);
} else {
$file = $repo->newFile($name);
}
if (!$file->exists() || $file->isDeleted(File::DELETED_FILE)) {
wfForbidden('img-auth-accessdenied', 'img-auth-nofile', $filename);
return;
}
}
$headers = array();
// extra HTTP headers to send
if (!$publicWiki) {
// For private wikis, run extra auth checks and set cache control headers
$headers[] = 'Cache-Control: private';
$headers[] = 'Vary: Cookie';
$title = Title::makeTitleSafe(NS_FILE, $name);
if (!$title instanceof Title) {
// files have valid titles
wfForbidden('img-auth-accessdenied', 'img-auth-badtitle', $name);
return;
}
// Run hook for extension authorization plugins
/** @var $result array */
$result = null;
if (!wfRunHooks('ImgAuthBeforeStream', array(&$title, &$path, &$name, &$result))) {
wfForbidden($result[0], $result[1], array_slice($result, 2));
return;
}
// Check user authorization for this title
// Checks Whitelist too
if (!$title->userCan('read')) {
wfForbidden('img-auth-accessdenied', 'img-auth-noread', $name);
return;
}
}
if ($request->getCheck('download')) {
$headers[] = 'Content-Disposition: attachment';
}
// Stream the requested file
wfDebugLog('img_auth', "Streaming `" . $filename . "`.");
$repo->streamFile($filename, $headers);
}
/**
* Construct a proxy backend that consists of several internal backends.
* Locking, journaling, and read-only checks are handled by the proxy backend.
*
* Additional $config params include:
* - backends : Array of backend config and multi-backend settings.
* Each value is the config used in the constructor of a
* FileBackendStore class, but with these additional settings:
* - class : The name of the backend class
* - isMultiMaster : This must be set for one backend.
* - template: : If given a backend name, this will use
* the config of that backend as a template.
* Values specified here take precedence.
* - syncChecks : Integer bitfield of internal backend sync checks to perform.
* Possible bits include the FileBackendMultiWrite::CHECK_* constants.
* There are constants for SIZE, TIME, and SHA1.
* The checks are done before allowing any file operations.
* - autoResync : Automatically resync the clone backends to the master backend
* when pre-operation sync checks fail. This should only be used
* if the master backend is stable and not missing any files.
* Use "conservative" to limit resyncing to copying newer master
* backend files over older (or non-existing) clone backend files.
* Cases that cannot be handled will result in operation abortion.
* - noPushQuickOps : (hack) Only apply doQuickOperations() to the master backend.
* - noPushDirConts : (hack) Only apply directory functions to the master backend.
*
* @param Array $config
* @throws MWException
*/
public function __construct( array $config ) {
parent::__construct( $config );
$this->syncChecks = isset( $config['syncChecks'] )
? $config['syncChecks']
: self::CHECK_SIZE;
$this->autoResync = isset( $config['autoResync'] )
? $config['autoResync']
: false;
$this->noPushQuickOps = isset( $config['noPushQuickOps'] )
? $config['noPushQuickOps']
: false;
$this->noPushDirConts = isset( $config['noPushDirConts'] )
? $config['noPushDirConts']
: array();
// Construct backends here rather than via registration
// to keep these backends hidden from outside the proxy.
$namesUsed = array();
foreach ( $config['backends'] as $index => $config ) {
if ( isset( $config['template'] ) ) {
// Config is just a modified version of a registered backend's.
// This should only be used when that config is used only by this backend.
$config = $config + FileBackendGroup::singleton()->config( $config['template'] );
}
$name = $config['name'];
if ( isset( $namesUsed[$name] ) ) { // don't break FileOp predicates
throw new MWException( "Two or more backends defined with the name $name." );
}
$namesUsed[$name] = 1;
// Alter certain sub-backend settings for sanity
unset( $config['readOnly'] ); // use proxy backend setting
unset( $config['fileJournal'] ); // use proxy backend journal
$config['wikiId'] = $this->wikiId; // use the proxy backend wiki ID
$config['lockManager'] = 'nullLockManager'; // lock under proxy backend
if ( !empty( $config['isMultiMaster'] ) ) {
if ( $this->masterIndex >= 0 ) {
throw new MWException( 'More than one master backend defined.' );
}
$this->masterIndex = $index; // this is the "master"
$config['fileJournal'] = $this->fileJournal; // log under proxy backend
}
// Create sub-backend object
if ( !isset( $config['class'] ) ) {
throw new MWException( 'No class given for a backend config.' );
}
$class = $config['class'];
$this->backends[$index] = new $class( $config );
}
if ( $this->masterIndex < 0 ) { // need backends and must have a master
throw new MWException( 'No master backend defined.' );
}
}
public function execute()
{
$src = FileBackendGroup::singleton()->get($this->getOption('src'));
$posDir = $this->getOption('posdir');
$posFile = $posDir ? $posDir . '/' . wfWikiID() : false;
if ($this->hasOption('posdump')) {
// Just dump the current position into the specified position dir
if (!$this->hasOption('posdir')) {
$this->error("Param posdir required!", 1);
}
if ($this->hasOption('postime')) {
$id = (int) $src->getJournal()->getPositionAtTime($this->getOption('postime'));
$this->output("Requested journal position is {$id}.\n");
} else {
$id = (int) $src->getJournal()->getCurrentPosition();
$this->output("Current journal position is {$id}.\n");
}
if (file_put_contents($posFile, $id, LOCK_EX) !== false) {
$this->output("Saved journal position file.\n");
} else {
$this->output("Could not save journal position file.\n");
}
if ($this->isQuiet()) {
print $id;
// give a single machine-readable number
}
return;
}
if (!$this->hasOption('dst')) {
$this->error("Param dst required!", 1);
}
$dst = FileBackendGroup::singleton()->get($this->getOption('dst'));
$start = $this->getOption('start', 0);
if (!$start && $posFile && is_dir($posDir)) {
$start = is_file($posFile) ? (int) trim(file_get_contents($posFile)) : 0;
++$start;
// we already did this ID, start with the next one
$startFromPosFile = true;
} else {
$startFromPosFile = false;
}
if ($this->hasOption('backoff')) {
$time = time() - $this->getOption('backoff', 0);
$end = (int) $src->getJournal()->getPositionAtTime($time);
} else {
$end = $this->getOption('end', INF);
}
$this->output("Synchronizing backend '{$dst->getName()}' to '{$src->getName()}'...\n");
$this->output("Starting journal position is {$start}.\n");
if (is_finite($end)) {
$this->output("Ending journal position is {$end}.\n");
}
// Periodically update the position file
$callback = function ($pos) use($startFromPosFile, $posFile, $start) {
if ($startFromPosFile && $pos >= $start) {
// successfully advanced
file_put_contents($posFile, $pos, LOCK_EX);
}
};
// Actually sync the dest backend with the reference backend
$lastOKPos = $this->syncBackends($src, $dst, $start, $end, $callback);
// Update the sync position file
if ($startFromPosFile && $lastOKPos >= $start) {
// successfully advanced
if (file_put_contents($posFile, $lastOKPos, LOCK_EX) !== false) {
$this->output("Updated journal position file.\n");
} else {
$this->output("Could not update journal position file.\n");
}
}
if ($lastOKPos === false) {
if (!$start) {
$this->output("No journal entries found.\n");
} else {
$this->output("No new journal entries found.\n");
}
} else {
$this->output("Stopped synchronization at journal position {$lastOKPos}.\n");
}
if ($this->isQuiet()) {
print $lastOKPos;
// give a single machine-readable number
}
}
public function execute() {
$src = FileBackendGroup::singleton()->get( $this->getOption( 'src' ) );
$dst = FileBackendGroup::singleton()->get( $this->getOption( 'dst' ) );
$containers = explode( '|', $this->getOption( 'containers' ) );
$subDir = rtrim( $this->getOption( 'subdir', '' ), '/' );
$rateFile = $this->getOption( 'ratefile' );
if ( $this->hasOption( 'utf8only' ) && !extension_loaded( 'mbstring' ) ) {
$this->error( "Cannot check for UTF-8, mbstring extension missing.", 1 ); // die
}
foreach ( $containers as $container ) {
if ( $subDir != '' ) {
$backendRel = "$container/$subDir";
$this->output( "Doing container '$container', directory '$subDir'...\n" );
} else {
$backendRel = $container;
$this->output( "Doing container '$container'...\n" );
}
if ( $this->hasOption( 'missingonly' ) ) {
$this->output( "\tBuilding list of missing files..." );
$srcPathsRel = $this->getListingDiffRel( $src, $dst, $backendRel );
$this->output( count( $srcPathsRel ) . " file(s) need to be copied.\n" );
} else {
$srcPathsRel = $src->getFileList( array(
'dir' => $src->getRootStoragePath() . "/$backendRel",
'adviseStat' => true // avoid HEADs
) );
if ( $srcPathsRel === null ) {
$this->error( "Could not list files in $container.", 1 ); // die
}
}
if ( $this->getOption( 'prestat' ) && !$this->hasOption( 'missingonly' ) ) {
// Build the stat cache for the destination files
$this->output( "\tBuilding destination stat cache..." );
$dstPathsRel = $dst->getFileList( array(
'dir' => $dst->getRootStoragePath() . "/$backendRel",
'adviseStat' => true // avoid HEADs
) );
if ( $dstPathsRel === null ) {
$this->error( "Could not list files in $container.", 1 ); // die
}
$this->statCache = array(); // clear
foreach ( $dstPathsRel as $dstPathRel ) {
$path = $dst->getRootStoragePath() . "/$backendRel/$dstPathRel";
$this->statCache[sha1( $path )] = $dst->getFileStat( array( 'src' => $path ) );
}
$this->output( "done [" . count( $this->statCache ) . " file(s)]\n" );
}
$this->output( "\tCopying file(s)...\n" );
$count = 0;
$batchPaths = array();
foreach ( $srcPathsRel as $srcPathRel ) {
// Check up on the rate file periodically to adjust the concurrency
if ( $rateFile && ( !$count || ( $count % 500 ) == 0 ) ) {
$this->mBatchSize = max( 1, (int)file_get_contents( $rateFile ) );
$this->output( "\tBatch size is now {$this->mBatchSize}.\n" );
}
$batchPaths[$srcPathRel] = 1; // remove duplicates
if ( count( $batchPaths ) >= $this->mBatchSize ) {
$this->copyFileBatch( array_keys( $batchPaths ), $backendRel, $src, $dst );
$batchPaths = array(); // done
}
++$count;
}
if ( count( $batchPaths ) ) { // left-overs
$this->copyFileBatch( array_keys( $batchPaths ), $backendRel, $src, $dst );
$batchPaths = array(); // done
}
$this->output( "\tCopied $count file(s).\n" );
if ( $this->hasOption( 'syncviadelete' ) ) {
$this->output( "\tBuilding list of excess destination files..." );
$delPathsRel = $this->getListingDiffRel( $dst, $src, $backendRel );
$this->output( count( $delPathsRel ) . " file(s) need to be deleted.\n" );
$this->output( "\tDeleting file(s)...\n" );
$count = 0;
$batchPaths = array();
foreach ( $delPathsRel as $delPathRel ) {
// Check up on the rate file periodically to adjust the concurrency
if ( $rateFile && ( !$count || ( $count % 500 ) == 0 ) ) {
$this->mBatchSize = max( 1, (int)file_get_contents( $rateFile ) );
$this->output( "\tBatch size is now {$this->mBatchSize}.\n" );
}
$batchPaths[$delPathRel] = 1; // remove duplicates
if ( count( $batchPaths ) >= $this->mBatchSize ) {
$this->delFileBatch( array_keys( $batchPaths ), $backendRel, $dst );
$batchPaths = array(); // done
}
++$count;
}
if ( count( $batchPaths ) ) { // left-overs
$this->delFileBatch( array_keys( $batchPaths ), $backendRel, $dst );
$batchPaths = array(); // done
}
$this->output( "\tDeleted $count file(s).\n" );
}
if ( $subDir != '' ) {
$this->output( "Finished container '$container', directory '$subDir'.\n" );
} else {
$this->output( "Finished container '$container'.\n" );
}
}
$this->output( "Done.\n" );
}
/**
* Modify timeline extension to use Swift storage (BAC-893)
*
* @param FileBackend $backend
* @param string $fname mwstore abstract path
* @param string $hash file hash
* @return bool true - it's a hook
*/
static function onBeforeRenderTimeline(&$backend, &$fname, $hash)
{
global $wgEnableSwiftFileBackend, $wgFSSwiftContainer;
if (!empty($wgEnableSwiftFileBackend)) {
$backend = FileBackendGroup::singleton()->get('swift-backend');
$fname = 'mwstore://' . $backend->getName() . "/{$wgFSSwiftContainer}/images/timeline/{$hash}";
}
return true;
}
function wfImageAuthMain()
{
global $wgImgAuthPublicTest, $wgImgAuthUrlPathMap, $wgRequest;
// See if this is a public Wiki (no protections).
if ($wgImgAuthPublicTest && in_array('read', User::getGroupPermissions(array('*')), true)) {
// This is a public wiki, so disable this script (for private wikis only)
wfForbidden('img-auth-accessdenied', 'img-auth-public');
return;
}
// Get the requested file path (source file or thumbnail)
$matches = WebRequest::getPathInfo();
if (!isset($matches['title'])) {
wfForbidden('img-auth-accessdenied', 'img-auth-nopathinfo');
return;
}
$path = $matches['title'];
if ($path && $path[0] !== '/') {
// Make sure $path has a leading /
$path = "/" . $path;
}
// Check for bug 28235: QUERY_STRING overriding the correct extension
$whitelist = array();
$extension = FileBackend::extensionFromPath($path, 'rawcase');
if ($extension != '') {
$whitelist[] = $extension;
}
if (!$wgRequest->checkUrlExtension($whitelist)) {
return;
}
// Various extensions may have their own backends that need access.
// Check if there is a special backend and storage base path for this file.
foreach ($wgImgAuthUrlPathMap as $prefix => $storageDir) {
$prefix = rtrim($prefix, '/') . '/';
// implicit trailing slash
if (strpos($path, $prefix) === 0) {
$be = FileBackendGroup::singleton()->backendFromPath($storageDir);
$filename = $storageDir . substr($path, strlen($prefix));
// strip prefix
// Check basic user authorization
if (!RequestContext::getMain()->getUser()->isAllowed('read')) {
wfForbidden('img-auth-accessdenied', 'img-auth-noread', $path);
return;
}
if ($be->fileExists(array('src' => $filename))) {
wfDebugLog('img_auth', "Streaming `" . $filename . "`.");
$be->streamFile(array('src' => $filename), array('Cache-Control: private', 'Vary: Cookie'));
} else {
wfForbidden('img-auth-accessdenied', 'img-auth-nofile', $path);
}
return;
}
}
// Get the local file repository
$repo = RepoGroup::singleton()->getRepo('local');
// Get the full file storage path and extract the source file name.
// (e.g. 120px-Foo.png => Foo.png or page2-120px-Foo.png => Foo.png).
// This only applies to thumbnails, and all thumbnails should
// be under a folder that has the source file name.
if (strpos($path, '/thumb/') === 0) {
$name = wfBaseName(dirname($path));
// file is a thumbnail
$filename = $repo->getZonePath('thumb') . substr($path, 6);
// strip "/thumb"
} else {
$name = wfBaseName($path);
// file is a source file
$filename = $repo->getZonePath('public') . $path;
}
// Check to see if the file exists
if (!$repo->fileExists($filename)) {
wfForbidden('img-auth-accessdenied', 'img-auth-nofile', $filename);
return;
}
$title = Title::makeTitleSafe(NS_FILE, $name);
if (!$title instanceof Title) {
// files have valid titles
wfForbidden('img-auth-accessdenied', 'img-auth-badtitle', $name);
return;
}
// Run hook for extension authorization plugins
/** @var $result array */
$result = null;
if (!wfRunHooks('ImgAuthBeforeStream', array(&$title, &$path, &$name, &$result))) {
wfForbidden($result[0], $result[1], array_slice($result, 2));
return;
}
// Check user authorization for this title
// Checks Whitelist too
if (!$title->userCan('read')) {
wfForbidden('img-auth-accessdenied', 'img-auth-noread', $name);
return;
}
if ($wgRequest->getCheck('download')) {
header('Content-Disposition: attachment');
}
// Stream the requested file
wfDebugLog('img_auth', "Streaming `" . $filename . "`.");
$repo->streamFile($filename, array('Cache-Control: private', 'Vary: Cookie'));
}
public function execute()
{
$src = FileBackendGroup::singleton()->get($this->getOption('src'));
$dst = FileBackendGroup::singleton()->get($this->getOption('dst'));
$containers = explode('|', $this->getOption('containers'));
$subDir = $this->getOption(rtrim('subdir', '/'), '');
$rateFile = $this->getOption('ratefile');
if ($this->hasOption('utf8only') && !extension_loaded('mbstring')) {
$this->error("Cannot check for UTF-8, mbstring extension missing.", 1);
// die
}
$count = 0;
foreach ($containers as $container) {
if ($subDir != '') {
$backendRel = "{$container}/{$subDir}";
$this->output("Doing container '{$container}', directory '{$subDir}'...\n");
} else {
$backendRel = $container;
$this->output("Doing container '{$container}'...\n");
}
$srcPathsRel = $src->getFileList(array('dir' => $src->getRootStoragePath() . "/{$backendRel}"));
if ($srcPathsRel === null) {
$this->error("Could not list files in {$container}.", 1);
// die
}
// Do a listing comparison if specified
if ($this->hasOption('missingonly')) {
$relFilesSrc = array();
$relFilesDst = array();
foreach ($srcPathsRel as $srcPathRel) {
$relFilesSrc[] = $srcPathRel;
}
$dstPathsRel = $dst->getFileList(array('dir' => $dst->getRootStoragePath() . "/{$backendRel}"));
if ($dstPathsRel === null) {
$this->error("Could not list files in {$container}.", 1);
// die
}
foreach ($dstPathsRel as $dstPathRel) {
$relFilesDst[] = $dstPathRel;
}
// Only copy the missing files over in the next loop
$srcPathsRel = array_diff($relFilesSrc, $relFilesDst);
$this->output(count($srcPathsRel) . " file(s) need to be copied.\n");
unset($relFilesSrc);
unset($relFilesDst);
}
$batchPaths = array();
foreach ($srcPathsRel as $srcPathRel) {
// Check up on the rate file periodically to adjust the concurrency
if ($rateFile && (!$count || $count % 500 == 0)) {
$this->mBatchSize = max(1, (int) file_get_contents($rateFile));
$this->output("Batch size is now {$this->mBatchSize}.\n");
}
$batchPaths[$srcPathRel] = 1;
// remove duplicates
if (count($batchPaths) >= $this->mBatchSize) {
$this->copyFileBatch(array_keys($batchPaths), $backendRel, $src, $dst);
$batchPaths = array();
// done
}
++$count;
}
if (count($batchPaths)) {
// left-overs
$this->copyFileBatch(array_keys($batchPaths), $backendRel, $src, $dst);
$batchPaths = array();
// done
}
if ($subDir != '') {
$this->output("Finished container '{$container}', directory '{$subDir}'.\n");
} else {
$this->output("Finished container '{$container}'.\n");
}
}
$this->output("Done [{$count} file(s)].\n");
}
/**
* @param $timelinesrc string
* @return string
*/
function wfRenderTimeline($timelinesrc)
{
global $wgUploadDirectory, $wgUploadPath, $wgArticlePath, $wgTmpDirectory, $wgRenderHashAppend;
global $wgTimelineSettings;
// Get the backend to store plot data and pngs
if ($wgTimelineSettings->fileBackend != '') {
$backend = FileBackendGroup::singleton()->get($wgTimelineSettings->fileBackend);
} else {
$backend = new FSFileBackend(array('name' => 'timeline-backend', 'lockManager' => 'nullLockManager', 'containerPaths' => array('timeline-render' => "{$wgUploadDirectory}/timeline"), 'fileMode' => 777));
}
// Get a hash of the plot data
$hash = md5($timelinesrc);
if ($wgRenderHashAppend != '') {
$hash = md5($hash . $wgRenderHashAppend);
}
// Storage destination path (excluding file extension)
$fname = 'mwstore://' . $backend->getName() . "/timeline-render/{$hash}";
// Wikia change - begin
wfRunHooks('BeforeRenderTimeline', [&$backend, &$fname, $hash]);
// Wikia change - end
$previouslyFailed = $backend->fileExists(array('src' => "{$fname}.err"));
$previouslyRendered = $backend->fileExists(array('src' => "{$fname}.png"));
if ($previouslyRendered) {
$timestamp = $backend->getFileTimestamp(array('src' => "{$fname}.png"));
$expired = $timestamp < $wgTimelineSettings->epochTimestamp;
} else {
$expired = false;
}
// Create a new .map, .png (or .gif), and .err file as needed...
if ($expired || !$previouslyRendered && !$previouslyFailed) {
if (!is_dir($wgTmpDirectory)) {
mkdir($wgTmpDirectory, 0777);
}
$tmpFile = TempFSFile::factory('timeline_');
if ($tmpFile) {
$tmpPath = $tmpFile->getPath();
file_put_contents($tmpPath, $timelinesrc);
// store plot data to file
// Get command for ploticus to read the user input and output an error,
// map, and rendering (png or gif) file under the same dir as the temp file.
$cmdline = wfEscapeShellArg($wgTimelineSettings->perlCommand, $wgTimelineSettings->timelineFile) . " -i " . wfEscapeShellArg($tmpPath) . " -m -P " . wfEscapeShellArg($wgTimelineSettings->ploticusCommand) . " -T " . wfEscapeShellArg($wgTmpDirectory) . " -A " . wfEscapeShellArg($wgArticlePath) . " -f " . wfEscapeShellArg($wgTimelineSettings->fontFile);
// Actually run the command...
wfDebug("Timeline cmd: {$cmdline}\n");
$retVal = null;
$ret = wfShellExec($cmdline, $retVal);
// Copy the output files into storage...
// @TODO: store error files in another container or not at all?
$opt = array('force' => 1, 'nonLocking' => 1, 'allowStale' => 1);
// performance
$backend->prepare(array('dir' => dirname($fname)));
$backend->store(array('src' => "{$tmpPath}.map", 'dst' => "{$fname}.map"), $opt);
$backend->store(array('src' => "{$tmpPath}.png", 'dst' => "{$fname}.png"), $opt);
$backend->store(array('src' => "{$tmpPath}.err", 'dst' => "{$fname}.err"), $opt);
} else {
return "<div id=\"toc\" dir=\"ltr\"><tt>Timeline error. " . "Could not create temp file</tt></div>";
// ugh
}
if ($ret == "" || $retVal > 0) {
// Message not localized, only relevant during install
return "<div id=\"toc\" dir=\"ltr\"><tt>Timeline error. " . "Command line was: " . htmlspecialchars($cmdline) . "</tt></div>";
}
}
// Wikia change - begin
if ($backend->fileExists(array('src' => "{$fname}.err", 'latest' => true))) {
$err = $backend->getFileContents(array('src' => "{$fname}.err"));
} else {
$err = '';
}
// Wikia change - end
if ($err != "") {
// Convert the error from poorly-sanitized HTML to plain text
$err = strtr($err, array('</p><p>' => "\n\n", '<p>' => '', '</p>' => '', '<b>' => '', '</b>' => '', '<br>' => "\n"));
$err = Sanitizer::decodeCharReferences($err);
// Now convert back to HTML again
$encErr = nl2br(htmlspecialchars($err));
$txt = "<div id=\"toc\" dir=\"ltr\"><tt>{$encErr}</tt></div>";
} else {
// Wikia change - begin
if ($backend->fileExists(array('src' => "{$fname}.map", 'latest' => true))) {
$map = $backend->getFileContents(array('src' => "{$fname}.map"));
} else {
$map = '';
}
// Wikia change - end
$map = str_replace(' >', ' />', $map);
$map = "<map name=\"timeline_" . htmlspecialchars($hash) . "\">{$map}</map>";
$map = easyTimelineFixMap($map);
$url = "{$wgUploadPath}/timeline/{$hash}.png";
// Wikia change - begin
$url = wfReplaceImageServer($url);
// Wikia change - end
$txt = $map . "<img usemap=\"#timeline_" . htmlspecialchars($hash) . "\" " . "src=\"" . htmlspecialchars($url) . "\">";
if ($expired) {
// Replacing an older file, we may need to purge the old one.
global $wgUseSquid;
if ($wgUseSquid) {
$u = new SquidUpdate(array($url));
$u->doUpdate();
}
}
}
return $txt;
}
public function execute()
{
$src = FileBackendGroup::singleton()->get($this->getOption('src'));
$dst = FileBackendGroup::singleton()->get($this->getOption('dst'));
$containers = explode('|', $this->getOption('containers'));
$subDir = $this->getOption(rtrim('subdir', '/'), '');
$rateFile = $this->getOption('ratefile');
if ($this->hasOption('utf8only') && !extension_loaded('mbstring')) {
$this->error("Cannot check for UTF-8, mbstring extension missing.", 1);
// die
}
$count = 0;
foreach ($containers as $container) {
if ($subDir != '') {
$backendRel = "{$container}/{$subDir}";
$this->output("Doing container '{$container}', directory '{$subDir}'...\n");
} else {
$backendRel = $container;
$this->output("Doing container '{$container}'...\n");
}
$srcPathsRel = $src->getFileList(array('dir' => $src->getRootStoragePath() . "/{$backendRel}", 'adviseStat' => !$this->hasOption('missingonly')));
if ($srcPathsRel === null) {
$this->error("Could not list files in {$container}.", 1);
// die
}
if ($this->hasOption('missingonly')) {
$dstPathsRel = $dst->getFileList(array('dir' => $dst->getRootStoragePath() . "/{$backendRel}"));
if ($dstPathsRel === null) {
$this->error("Could not list files in {$container}.", 1);
// die
}
// Get the list of destination files
$relFilesDstSha1 = array();
foreach ($dstPathsRel as $dstPathRel) {
$relFilesDstSha1[sha1($dstPathRel)] = 1;
}
unset($dstPathsRel);
// free
// Get the list of missing files
$missingPathsRel = array();
foreach ($srcPathsRel as $srcPathRel) {
if (!isset($relFilesDstSha1[sha1($srcPathRel)])) {
$missingPathsRel[] = $srcPathRel;
}
}
unset($srcPathsRel);
// free
// Only copy the missing files over in the next loop
$srcPathsRel = $missingPathsRel;
$this->output(count($srcPathsRel) . " file(s) need to be copied.\n");
} elseif ($this->getOption('prestat')) {
// Build the stat cache for the destination files
$this->output("Building destination stat cache...");
$dstPathsRel = $dst->getFileList(array('dir' => $dst->getRootStoragePath() . "/{$backendRel}", 'adviseStat' => true));
if ($dstPathsRel === null) {
$this->error("Could not list files in {$container}.", 1);
// die
}
$this->statCache = array();
// clear
foreach ($dstPathsRel as $dstPathRel) {
$path = $dst->getRootStoragePath() . "/{$backendRel}/{$dstPathRel}";
$this->statCache[sha1($path)] = $dst->getFileStat(array('src' => $path));
}
$this->output("done [" . count($this->statCache) . " file(s)]\n");
}
$batchPaths = array();
foreach ($srcPathsRel as $srcPathRel) {
// Check up on the rate file periodically to adjust the concurrency
if ($rateFile && (!$count || $count % 500 == 0)) {
$this->mBatchSize = max(1, (int) file_get_contents($rateFile));
$this->output("Batch size is now {$this->mBatchSize}.\n");
}
$batchPaths[$srcPathRel] = 1;
// remove duplicates
if (count($batchPaths) >= $this->mBatchSize) {
$this->copyFileBatch(array_keys($batchPaths), $backendRel, $src, $dst);
$batchPaths = array();
// done
}
++$count;
}
if (count($batchPaths)) {
// left-overs
$this->copyFileBatch(array_keys($batchPaths), $backendRel, $src, $dst);
$batchPaths = array();
// done
}
if ($subDir != '') {
$this->output("Finished container '{$container}', directory '{$subDir}'.\n");
} else {
$this->output("Finished container '{$container}'.\n");
}
}
$this->output("Done [{$count} file(s)].\n");
}
/**
* @brief remove thumbnails for avatar by cleaning up whole folder
*
* @author Krzysztof Krzyżaniak (eloy) <*****@*****.**>
* @access private
*
* @return boolean status of operation
*/
private function purgeThumbnails()
{
global $wgAvatarsUseSwiftStorage, $wgBlogAvatarPath, $wgBlogAvatarSwiftContainer, $wgBlogAvatarSwiftPathPrefix;
// get path to thumbnail folder
wfProfileIn(__METHOD__);
// dirty hack, should work in this case
if (!empty($wgAvatarsUseSwiftStorage)) {
$swift = $this->getSwiftStorage();
$backend = FileBackendGroup::singleton()->get('swift-backend');
$dir = sprintf('mwstore://swift-backend/%s%s%s', $wgBlogAvatarSwiftContainer, $wgBlogAvatarSwiftPathPrefix, $this->getLocalPath());
$dir = $this->getThumbPath($dir);
$avatarRemotePath = sprintf("thumb%s", $this->getLocalPath());
$urls = [];
$files = [];
$iterator = $backend->getFileList(array('dir' => $dir));
foreach ($iterator as $file) {
$files[] = sprintf("%s/%s", $avatarRemotePath, $file);
}
// deleting files on file system and creating an array of URLs to purge
if (!empty($files)) {
foreach ($files as $file) {
$status = $swift->remove($file);
if (!$status->isOk()) {
wfDebugLog("avatar", __METHOD__ . ": {$file} exists but cannot be removed.\n", true);
} else {
$urls[] = wfReplaceImageServer($wgBlogAvatarPath) . "/{$file}";
wfDebugLog("avatar", __METHOD__ . ": {$file} removed.\n", true);
}
}
}
wfDebugLog("avatar", __METHOD__ . ": all thumbs removed.\n", true);
} else {
$dir = $this->getFullPath();
$dir = $this->getThumbPath($dir);
if (is_dir($dir)) {
$urls = [];
$files = [];
// copied from LocalFile->getThumbnails
$handle = opendir($dir);
if ($handle) {
while (false !== ($file = readdir($handle))) {
if ($file[0] != '.') {
$files[] = $file;
}
}
closedir($handle);
}
// partialy copied from LocalFile->purgeThumbnails()
foreach ($files as $file) {
// deleting files on file system
@unlink("{$dir}/{$file}");
$urls[] = $this->getPurgeUrl('/thumb/') . "/{$file}";
wfDebugLog("avatar", __METHOD__ . ": removing {$dir}/{$file}\n", true);
}
} else {
wfDebugLog("avatar", __METHOD__ . ": {$dir} exists but is not directory so not removed.\n", true);
}
wfDebugLog("avatar", __METHOD__ . ": all thumbs removed.\n", true);
}
// purging avatars urls
SquidUpdate::purge($urls);
wfProfileOut(__METHOD__);
}
public static function provider_testGetContentType()
{
return [[null, false], [[FileBackendGroup::singleton(), 'guessMimeInternal'], true]];
}
public static function provider_testGetContentType()
{
return array(array(null, false), array(array(FileBackendGroup::singleton(), 'guessMimeInternal'), true));
}
