/** * Stream a file to the browser, adding all the headings and fun stuff. * Headers sent include: Content-type, Content-Length, Last-Modified, * and Content-Disposition. * * @param string $fname Full name and path of the file to stream * @param array $headers Any additional headers to send * @param bool $sendErrors Send error messages if errors occur (like 404) * @throws MWException * @return bool Success */ public static function stream($fname, $headers = array(), $sendErrors = true) { wfProfileIn(__METHOD__); if (FileBackend::isStoragePath($fname)) { // sanity wfProfileOut(__METHOD__); throw new MWException(__FUNCTION__ . " given storage path '{$fname}'."); } wfSuppressWarnings(); $stat = stat($fname); wfRestoreWarnings(); $res = self::prepareForStream($fname, $stat, $headers, $sendErrors); if ($res == self::NOT_MODIFIED) { $ok = true; // use client cache } elseif ($res == self::READY_STREAM) { wfProfileIn(__METHOD__ . '-send'); $ok = readfile($fname); wfProfileOut(__METHOD__ . '-send'); } else { $ok = false; // failed } wfProfileOut(__METHOD__); return $ok; }
function assertBackendPathsConsistent(array $paths) { if ($this->backend instanceof FileBackendMultiWrite) { $status = $this->backend->consistencyCheck($paths); $this->assertGoodStatus($status, "Files synced: " . implode(',', $paths)); } }
/** * Construct a proxy backend that consists of several internal backends. * Additional $config params include: * 'backends' : Array of backend config and multi-backend settings. * Each value is the config used in the constructor of a * FileBackendStore class, but with these additional settings: * 'class' : The name of the backend class * 'isMultiMaster' : This must be set for one backend. * 'syncChecks' : Integer bitfield of internal backend sync checks to perform. * Possible bits include self::CHECK_SIZE and self::CHECK_TIME. * The checks are done before allowing any file operations. * @param $config Array */ public function __construct(array $config) { parent::__construct($config); $namesUsed = array(); // Construct backends here rather than via registration // to keep these backends hidden from outside the proxy. foreach ($config['backends'] as $index => $config) { $name = $config['name']; if (isset($namesUsed[$name])) { // don't break FileOp predicates throw new MWException("Two or more backends defined with the name {$name}."); } $namesUsed[$name] = 1; if (!isset($config['class'])) { throw new MWException('No class given for a backend config.'); } $class = $config['class']; $this->backends[$index] = new $class($config); if (!empty($config['isMultiMaster'])) { if ($this->masterIndex >= 0) { throw new MWException('More than one master backend defined.'); } $this->masterIndex = $index; } } if ($this->masterIndex < 0) { // need backends and must have a master throw new MWException('No master backend defined.'); } $this->syncChecks = isset($config['syncChecks']) ? $config['syncChecks'] : self::CHECK_SIZE; }
/** * Sets up the file object * * @param $path string Path to temporary file on local disk * @throws MWException */ public function __construct($path) { if (FileBackend::isStoragePath($path)) { throw new MWException(__METHOD__ . " given storage path `{$path}`."); } $this->path = $path; }
/** * Get an associative array containing information about * a file with the given storage path. * * Resulting array fields include: * - fileExists * - size (filesize in bytes) * - mime (as major/minor) * - media_type (value to be used with the MEDIATYPE_xxx constants) * - metadata (handler specific) * - sha1 (in base 36) * - width * - height * - bits (bitrate) * - file-mime * - major_mime * - minor_mime * * @param string $path Filesystem path to a file * @param string|bool $ext The file extension, or true to extract it from the filename. * Set it to false to ignore the extension. * @return array * @since 1.28 */ public function getPropsFromPath($path, $ext) { $fsFile = new FSFile($path); $info = $this->newPlaceholderProps(); $info['fileExists'] = $fsFile->exists(); if ($info['fileExists']) { $info['size'] = $fsFile->getSize(); // bytes $info['sha1'] = $fsFile->getSha1Base36(); # MIME type according to file contents $info['file-mime'] = $this->magic->guessMimeType($path, false); # Logical MIME type $ext = $ext === true ? FileBackend::extensionFromPath($path) : $ext; $info['mime'] = $this->magic->improveTypeFromExtension($info['file-mime'], $ext); list($info['major_mime'], $info['minor_mime']) = File::splitMime($info['mime']); $info['media_type'] = $this->magic->getMediaType($path, $info['mime']); # Height, width and metadata $handler = MediaHandler::getHandler($info['mime']); if ($handler) { $info['metadata'] = $handler->getMetadata($fsFile, $path); /** @noinspection PhpMethodParametersCountMismatchInspection */ $gis = $handler->getImageSize($fsFile, $path, $info['metadata']); if (is_array($gis)) { $info = $this->extractImageSizeInfo($gis) + $info; } } } return $info; }
/** * Get the ultimate original storage path for a file * * Use this when putting a new file into the system * * @param string $sha1 File SHA-1 base36 * @return string */ public function getPathForSHA1($sha1) { if (strlen($sha1) < 3) { throw new MWException("Invalid file SHA-1."); } return $this->backend->getContainerStoragePath("{$this->repoName}-original") . "/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}"; }
/** * @see FileBackend::__construct() * * @param $config Array */ public function __construct(array $config) { parent::__construct($config); $this->memCache = new EmptyBagOStuff(); // disabled by default $this->cheapCache = new ProcessCacheLRU(300); $this->expensiveCache = new ProcessCacheLRU(5); }
/** * Stream a file to the browser, adding all the headings and fun stuff. * Headers sent include: Content-type, Content-Length, Last-Modified, * and Content-Disposition. * * @param string $fname Full name and path of the file to stream * @param array $headers Any additional headers to send if the file exists * @param bool $sendErrors Send error messages if errors occur (like 404) * @param array $optHeaders HTTP request header map (e.g. "range") (use lowercase keys) * @param integer $flags Bitfield of STREAM_* constants * @throws MWException * @return bool Success */ public static function stream($fname, $headers = [], $sendErrors = true, $optHeaders = [], $flags = 0) { if (FileBackend::isStoragePath($fname)) { // sanity throw new InvalidArgumentException(__FUNCTION__ . " given storage path '{$fname}'."); } $streamer = new HTTPFileStreamer($fname, ['obResetFunc' => 'wfResetOutputBuffers', 'streamMimeFunc' => [__CLASS__, 'contentTypeFromPath']]); return $streamer->stream($headers, $sendErrors, $optHeaders, $flags); }
protected function tearDown() { $this->repo->cleanupBatch($this->createdFiles); // delete files foreach ($this->createdFiles as $tmp) { // delete dirs $tmp = $this->repo->resolveVirtualUrl($tmp); while ($tmp = FileBackend::parentStoragePath($tmp)) { $this->repo->getBackend()->clean(array('dir' => $tmp)); } } parent::tearDown(); }
/** * Call a callback function for every public file in the repository. * May use either the database or the filesystem. * * @param $callback Array|string * @return void */ protected function enumFilesInStorage($callback) { $publicRoot = $this->getZonePath('public'); $numDirs = 1 << $this->hashLevels * 4; // Use a priori assumptions about directory structure // to reduce the tree height of the scanning process. for ($flatIndex = 0; $flatIndex < $numDirs; $flatIndex++) { $hexString = sprintf("%0{$this->hashLevels}x", $flatIndex); $path = $publicRoot; for ($hexPos = 0; $hexPos < $this->hashLevels; $hexPos++) { $path .= '/' . substr($hexString, 0, $hexPos + 1); } $iterator = $this->backend->getFileList(array('dir' => $path)); foreach ($iterator as $name) { // Each item returned is a public file call_user_func($callback, "{$path}/{$name}"); } } }
/** * Stream a file to the browser, adding all the headings and fun stuff. * Headers sent include: Content-type, Content-Length, Last-Modified, * and Content-Disposition. * * @param string $fname Full name and path of the file to stream * @param array $headers Any additional headers to send * @param bool $sendErrors Send error messages if errors occur (like 404) * @throws MWException * @return bool Success */ public static function stream($fname, $headers = [], $sendErrors = true) { if (FileBackend::isStoragePath($fname)) { // sanity throw new MWException(__FUNCTION__ . " given storage path '{$fname}'."); } MediaWiki\suppressWarnings(); $stat = stat($fname); MediaWiki\restoreWarnings(); $res = self::prepareForStream($fname, $stat, $headers, $sendErrors); if ($res == self::NOT_MODIFIED) { $ok = true; // use client cache } elseif ($res == self::READY_STREAM) { $ok = readfile($fname); } else { $ok = false; // failed } return $ok; }
public function execute() { $out = $this->mSpecial->getOutput(); $dbr = wfGetDB(DB_SLAVE); $row = $dbr->selectRow('moderation', array('mod_user AS user', 'mod_user_text AS user_text', 'mod_title AS title', 'mod_stash_key AS stash_key'), array('mod_id' => $this->id), __METHOD__); if (!$row) { throw new ModerationError('moderation-edit-not-found'); } $user = $row->user ? User::newFromId($row->user) : User::newFromName($row->user_text, false); $stash = RepoGroup::singleton()->getLocalRepo()->getUploadStash($user); try { $file = $stash->getFile($row->stash_key); } catch (MWException $e) { return $this->send404ImageNotFound(); } $is_thumb = $this->mSpecial->getRequest()->getVal('thumb'); if ($is_thumb) { $thumb = $file->transform(array('width' => self::THUMB_WIDTH), File::RENDER_NOW); if ($thumb) { if ($thumb->fileIsSource()) { $is_thumb = false; } else { $file = new UnregisteredLocalFile(false, $stash->repo, $thumb->getStoragePath(), false); } } } if (!$file) { return $this->send404ImageNotFound(); } $thumb_filename = ''; if ($is_thumb) { $thumb_filename .= $file->getWidth() . 'px-'; } $thumb_filename .= $row->title; $headers = array(); $headers[] = "Content-Disposition: " . FileBackend::makeContentDisposition('inline', $thumb_filename); $out->disable(); # No HTML output (image only) $file->getRepo()->streamFile($file->getPath(), $headers); }
/** * @param array $files * @return array */ function fileExistsBatch(array $files) { $results = []; foreach ($files as $k => $f) { if (isset($this->mFileExists[$f])) { $results[$k] = $this->mFileExists[$f]; unset($files[$k]); } elseif (self::isVirtualUrl($f)) { # @todo FIXME: We need to be able to handle virtual # URLs better, at least when we know they refer to the # same repo. $results[$k] = false; unset($files[$k]); } elseif (FileBackend::isStoragePath($f)) { $results[$k] = false; unset($files[$k]); wfWarn("Got mwstore:// path '{$f}'."); } } $data = $this->fetchImageQuery(['titles' => implode($files, '|'), 'prop' => 'imageinfo']); if (isset($data['query']['pages'])) { # First, get results from the query. Note we only care whether the image exists, # not whether it has a description page. foreach ($data['query']['pages'] as $p) { $this->mFileExists[$p['title']] = $p['imagerepository'] !== ''; } # Second, copy the results to any redirects that were queried if (isset($data['query']['redirects'])) { foreach ($data['query']['redirects'] as $r) { $this->mFileExists[$r['from']] = $this->mFileExists[$r['to']]; } } # Third, copy the results to any non-normalized titles that were queried if (isset($data['query']['normalized'])) { foreach ($data['query']['normalized'] as $n) { $this->mFileExists[$n['from']] = $this->mFileExists[$n['to']]; } } # Finally, copy the results to the output foreach ($files as $key => $file) { $results[$key] = $this->mFileExists[$file]; } } return $results; }
/** * Upload a file and record it in the DB * @param string $srcPath Source storage path, virtual URL, or filesystem path * @param string $comment Upload description * @param string $pageText Text to use for the new description page, * if a new description page is created * @param int|bool $flags Flags for publish() * @param array|bool $props File properties, if known. This can be used to * reduce the upload time when uploading virtual URLs for which the file * info is already known * @param string|bool $timestamp Timestamp for img_timestamp, or false to use the * current time * @param User|null $user User object or null to use $wgUser * @param string[] $tags Change tags to add to the log entry and page revision. * (This doesn't check $user's permissions.) * @return FileRepoStatus On success, the value member contains the * archive name, or an empty string if it was a new file. */ function upload($srcPath, $comment, $pageText, $flags = 0, $props = false, $timestamp = false, $user = null, $tags = array()) { global $wgContLang; if ($this->getRepo()->getReadOnlyReason() !== false) { return $this->readOnlyFatalStatus(); } if (!$props) { if ($this->repo->isVirtualUrl($srcPath) || FileBackend::isStoragePath($srcPath)) { $props = $this->repo->getFileProps($srcPath); } else { $props = FSFile::getPropsFromPath($srcPath); } } $options = array(); $handler = MediaHandler::getHandler($props['mime']); if ($handler) { $options['headers'] = $handler->getStreamHeaders($props['metadata']); } else { $options['headers'] = array(); } // Trim spaces on user supplied text $comment = trim($comment); // Truncate nicely or the DB will do it for us // non-nicely (dangling multi-byte chars, non-truncated version in cache). $comment = $wgContLang->truncate($comment, 255); $this->lock(); // begin $status = $this->publish($srcPath, $flags, $options); if ($status->successCount >= 2) { // There will be a copy+(one of move,copy,store). // The first succeeding does not commit us to updating the DB // since it simply copied the current version to a timestamped file name. // It is only *preferable* to avoid leaving such files orphaned. // Once the second operation goes through, then the current version was // updated and we must therefore update the DB too. $oldver = $status->value; if (!$this->recordUpload2($oldver, $comment, $pageText, $props, $timestamp, $user, $tags)) { $status->fatal('filenotfound', $srcPath); } } $this->unlock(); // done return $status; }
/** * Make directory, and make all parent directories if they don't exist * * @param string $dir Full path to directory to create * @param int $mode Chmod value to use, default is $wgDirectoryMode * @param string $caller Optional caller param for debugging. * @throws MWException * @return bool */ function wfMkdirParents($dir, $mode = null, $caller = null) { global $wgDirectoryMode; if (FileBackend::isStoragePath($dir)) { // sanity throw new MWException(__FUNCTION__ . " given storage path '{$dir}'."); } if (!is_null($caller)) { wfDebug("{$caller}: called wfMkdirParents({$dir})\n"); } if (strval($dir) === '' || is_dir($dir)) { return true; } $dir = str_replace(array('\\', '/'), DIRECTORY_SEPARATOR, $dir); if (is_null($mode)) { $mode = $wgDirectoryMode; } // Turn off the normal warning, we're doing our own below MediaWiki\suppressWarnings(); $ok = mkdir($dir, $mode, true); // PHP5 <3 MediaWiki\restoreWarnings(); if (!$ok) { //directory may have been created on another request since we last checked if (is_dir($dir)) { return true; } // PHP doesn't report the path in its warning message, so add our own to aid in diagnosis. wfLogWarning(sprintf("failed to mkdir \"%s\" mode 0%o", $dir, $mode)); } return $ok; }
/** * @dataProvider provider_testExtensionFromPath */ public function testExtensionFromPath($path, $res) { $this->assertEquals($res, FileBackend::extensionFromPath($path), "FileBackend::extensionFromPath on path '{$path}'"); }
protected function filesAreSame(FileBackend $src, FileBackend $dst, $sPath, $dPath) { return $src->getFileSize(['src' => $sPath]) === $dst->getFileSize(['src' => $dPath]) && $src->getFileSha1Base36(['src' => $sPath]) === $dst->getFileSha1Base36(['src' => $dPath]); }
/** * Upload a file and record it in the DB * @param string $srcPath Source storage path, virtual URL, or filesystem path * @param string $comment Upload description * @param string $pageText Text to use for the new description page, * if a new description page is created * @param int|bool $flags Flags for publish() * @param array|bool $props File properties, if known. This can be used to * reduce the upload time when uploading virtual URLs for which the file * info is already known * @param string|bool $timestamp Timestamp for img_timestamp, or false to use the * current time * @param User|null $user User object or null to use $wgUser * * @return FileRepoStatus object. On success, the value member contains the * archive name, or an empty string if it was a new file. */ function upload($srcPath, $comment, $pageText, $flags = 0, $props = false, $timestamp = false, $user = null) { global $wgContLang; if ($this->getRepo()->getReadOnlyReason() !== false) { return $this->readOnlyFatalStatus(); } if (!$props) { wfProfileIn(__METHOD__ . '-getProps'); if ($this->repo->isVirtualUrl($srcPath) || FileBackend::isStoragePath($srcPath)) { $props = $this->repo->getFileProps($srcPath); } else { $props = FSFile::getPropsFromPath($srcPath); } wfProfileOut(__METHOD__ . '-getProps'); } $options = array(); $handler = MediaHandler::getHandler($props['mime']); if ($handler) { $options['headers'] = $handler->getStreamHeaders($props['metadata']); } else { $options['headers'] = array(); } // Trim spaces on user supplied text $comment = trim($comment); // truncate nicely or the DB will do it for us // non-nicely (dangling multi-byte chars, non-truncated version in cache). $comment = $wgContLang->truncate($comment, 255); $this->lock(); // begin $status = $this->publish($srcPath, $flags, $options); if ($status->successCount > 0) { # Essentially we are displacing any existing current file and saving # a new current file at the old location. If just the first succeeded, # we still need to displace the current DB entry and put in a new one. if (!$this->recordUpload2($status->value, $comment, $pageText, $props, $timestamp, $user)) { $status->fatal('filenotfound', $srcPath); } } $this->unlock(); // done return $status; }
/** * Normalize a string if it is a valid storage path * * @param $path string * @return string */ protected static function normalizeIfValidStoragePath($path) { if (FileBackend::isStoragePath($path)) { $res = FileBackend::normalizeStoragePath($path); return $res !== null ? $res : $path; } return $path; }
/** * Get an appropriate backend object from a storage path * * @param string $storagePath * @return FileBackend|null Backend or null on failure */ public function backendFromPath($storagePath) { list($backend, , ) = FileBackend::splitStoragePath($storagePath); if ($backend !== null && isset($this->backends[$backend])) { return $this->get($backend); } return null; }
/** * @see FileBackendStore::doGetLocalCopyMulti() * @return null|TempFSFile */ protected function doGetLocalCopyMulti(array $params) { $tmpFiles = array(); $ep = array_diff_key($params, array('srcs' => 1)); // for error logging // Blindly create tmp files and stream to them, catching any exception if the file does // not exist. Doing a stat here is useless causes infinite loops in addMissingMetadata(). foreach (array_chunk($params['srcs'], $params['concurrency']) as $pathBatch) { $cfOps = array(); // (path => CF_Async_Op) foreach ($pathBatch as $path) { // each path in this concurrent batch list($srcCont, $srcRel) = $this->resolveStoragePathReal($path); if ($srcRel === null) { $tmpFiles[$path] = null; continue; } $tmpFile = null; try { $sContObj = $this->getContainer($srcCont); $obj = new CF_Object($sContObj, $srcRel, false, false); // skip HEAD // Get source file extension $ext = FileBackend::extensionFromPath($path); // Create a new temporary file... $tmpFile = TempFSFile::factory('localcopy_', $ext); if ($tmpFile) { $handle = fopen($tmpFile->getPath(), 'wb'); if ($handle) { $headers = $this->headersFromParams($params); if (count($pathBatch) > 1) { $cfOps[$path] = $obj->stream_async($handle, $headers); $cfOps[$path]->_file_handle = $handle; // close this later } else { $obj->stream($handle, $headers); fclose($handle); } } else { $tmpFile = null; } } } catch (NoSuchContainerException $e) { $tmpFile = null; } catch (NoSuchObjectException $e) { $tmpFile = null; } catch (CloudFilesException $e) { // some other exception? $tmpFile = null; $this->handleException($e, null, __METHOD__, array('src' => $path) + $ep); } $tmpFiles[$path] = $tmpFile; } $batch = new CF_Async_Op_Batch($cfOps); $cfOps = $batch->execute(); foreach ($cfOps as $path => $cfOp) { try { $cfOp->getLastResponse(); } catch (NoSuchContainerException $e) { $tmpFiles[$path] = null; } catch (NoSuchObjectException $e) { $tmpFiles[$path] = null; } catch (CloudFilesException $e) { // some other exception? $tmpFiles[$path] = null; $this->handleException($e, null, __METHOD__, array('src' => $path) + $ep); } fclose($cfOp->_file_handle); // close open handle } } return $tmpFiles; }
protected function doGetLocalCopyMulti(array $params) { $tmpFiles = array(); $auth = $this->getAuthentication(); $ep = array_diff_key($params, array('srcs' => 1)); // for error logging // Blindly create tmp files and stream to them, catching any exception if the file does // not exist. Doing a stat here is useless causes infinite loops in addMissingMetadata(). $reqs = array(); // (path => op) foreach ($params['srcs'] as $path) { // each path in this concurrent batch list($srcCont, $srcRel) = $this->resolveStoragePathReal($path); if ($srcRel === null || !$auth) { $tmpFiles[$path] = null; continue; } // Get source file extension $ext = FileBackend::extensionFromPath($path); // Create a new temporary file... $tmpFile = TempFSFile::factory('localcopy_', $ext); if ($tmpFile) { $handle = fopen($tmpFile->getPath(), 'wb'); if ($handle) { $reqs[$path] = array('method' => 'GET', 'url' => $this->storageUrl($auth, $srcCont, $srcRel), 'headers' => $this->authTokenHeaders($auth) + $this->headersFromParams($params), 'stream' => $handle); } else { $tmpFile = null; } } $tmpFiles[$path] = $tmpFile; } $isLatest = $this->isRGW || !empty($params['latest']); $opts = array('maxConnsPerHost' => $params['concurrency']); $reqs = $this->http->runMulti($reqs, $opts); foreach ($reqs as $path => $op) { list($rcode, $rdesc, $rhdrs, $rbody, $rerr) = $op['response']; fclose($op['stream']); // close open handle if ($rcode >= 200 && $rcode <= 299) { $size = $tmpFiles[$path] ? $tmpFiles[$path]->getSize() : 0; // Double check that the disk is not full/broken if ($size != $rhdrs['content-length']) { $tmpFiles[$path] = null; $rerr = "Got {$size}/{$rhdrs['content-length']} bytes"; $this->onError(null, __METHOD__, array('src' => $path) + $ep, $rerr, $rcode, $rdesc); } // Set the file stat process cache in passing $stat = $this->getStatFromHeaders($rhdrs); $stat['latest'] = $isLatest; $this->cheapCache->set($path, 'stat', $stat); } elseif ($rcode === 404) { $tmpFiles[$path] = false; } else { $tmpFiles[$path] = null; $this->onError(null, __METHOD__, array('src' => $path) + $ep, $rerr, $rcode, $rdesc); } } return $tmpFiles; }
/** * @see FileBackendStore::getLocalCopy() */ public function getLocalCopy(array $params) { list($srcCont, $srcRel) = $this->resolveStoragePathReal($params['src']); if ($srcRel === null) { return null; } /*if ( !$this->fileExists( $params ) ) { return null; }*/ $tmpFile = null; try { $sContObj = $this->getContainer($srcCont); $obj = new CF_Object($sContObj, $srcRel, false, false); // skip HEAD // Get source file extension $ext = FileBackend::extensionFromPath($srcRel); // Create a new temporary file... $tmpFile = TempFSFile::factory(wfBaseName($srcRel) . '_', $ext); if ($tmpFile) { $handle = fopen($tmpFile->getPath(), 'wb'); if ($handle) { $obj->stream($handle, $this->headersFromParams($params)); fclose($handle); } else { $tmpFile = null; // couldn't open temp file } } } catch (NoSuchContainerException $e) { $tmpFile = null; $this->logException($e, __METHOD__, $params); } catch (NoSuchObjectException $e) { $tmpFile = null; $this->logException($e, __METHOD__, $params); } catch (InvalidResponseException $e) { $tmpFile = null; $this->logException($e, __METHOD__, $params); } catch (Exception $e) { // some other exception? $tmpFile = null; $this->logException($e, __METHOD__, $params); } return $tmpFile; }
/** * Append the final chunk and ready file for parent::performUpload() * @return FileRepoStatus */ public function concatenateChunks() { wfDebug(__METHOD__ . " concatenate {$this->mChunkIndex} chunks:" . $this->getOffset() . ' inx:' . $this->getChunkIndex() . "\n"); // Concatenate all the chunks to mVirtualTempPath $fileList = array(); // The first chunk is stored at the mVirtualTempPath path so we start on "chunk 1" for ($i = 0; $i <= $this->getChunkIndex(); $i++) { $fileList[] = $this->getVirtualChunkLocation($i); } // Get the file extension from the last chunk $ext = FileBackend::extensionFromPath($this->mVirtualTempPath); // Get a 0-byte temp file to perform the concatenation at $tmpFile = TempFSFile::factory('chunkedupload_', $ext); $tmpPath = $tmpFile ? $tmpFile->getPath() : false; // fail in concatenate() // Concatenate the chunks at the temp file $status = $this->repo->concatenate($fileList, $tmpPath, FileRepo::DELETE_SOURCE); if (!$status->isOk()) { return $status; } // Update the mTempPath and mLocalFile // ( for FileUpload or normal Stash to take over ) $this->mTempPath = $tmpPath; // file system path $this->mLocalFile = parent::stashFile(); return $status; }
/** * Do a batch lookup from cache for file stats for all paths * used in a list of storage paths or FileOp objects. * This loads the persistent cache values into the process cache. * * @param array $items List of storage paths */ protected final function primeFileCache(array $items) { $ps = $this->scopedProfileSection(__METHOD__ . "-{$this->name}"); $paths = []; // list of storage paths $pathNames = []; // (cache key => storage path) // Get all the paths/containers from the items... foreach ($items as $item) { if (self::isStoragePath($item)) { $paths[] = FileBackend::normalizeStoragePath($item); } } // Get rid of any paths that failed normalization... $paths = array_filter($paths, 'strlen'); // remove nulls // Get all the corresponding cache keys for paths... foreach ($paths as $path) { list(, $rel, ) = $this->resolveStoragePath($path); if ($rel !== null) { // valid path for this backend $pathNames[$this->fileCacheKey($path)] = $path; } } // Get all cache entries for these file cache keys... $values = $this->memCache->getMulti(array_keys($pathNames)); foreach ($values as $cacheKey => $val) { $path = $pathNames[$cacheKey]; if (is_array($val)) { $val['latest'] = false; // never completely trust cache $this->cheapCache->set($path, 'stat', $val); if (isset($val['sha1'])) { // some backends store SHA-1 as metadata $this->cheapCache->set($path, 'sha1', ['hash' => $val['sha1'], 'latest' => false]); } if (isset($val['xattr'])) { // some backends store headers/metadata $val['xattr'] = self::normalizeXAttributes($val['xattr']); $this->cheapCache->set($path, 'xattr', ['map' => $val['xattr'], 'latest' => false]); } } } }
/** * Initialize the path information * @param string $name the desired destination name * @param string $tempPath the temporary path * @param int $fileSize the file size * @param bool $removeTempFile (false) remove the temporary file? * @throws MWException */ public function initializePathInfo($name, $tempPath, $fileSize, $removeTempFile = false) { $this->mDesiredDestName = $name; if (FileBackend::isStoragePath($tempPath)) { throw new MWException(__METHOD__ . " given storage path `{$tempPath}`."); } $this->mTempPath = $tempPath; $this->mFileSize = $fileSize; $this->mRemoveTempFile = $removeTempFile; }
/** * @param string $thumbName Thumbnail name * @return string Content-Disposition header value */ function getThumbDisposition($thumbName) { $fileName = $this->name; // file name to suggest $thumbExt = FileBackend::extensionFromPath($thumbName); if ($thumbExt != '' && $thumbExt !== $this->getExtension()) { $fileName .= ".{$thumbExt}"; } return FileBackend::makeContentDisposition('inline', $fileName); }
/** * Send response headers (using the header() function) that are necessary to correctly serve the * image data for this image, as returned by getImageData(). * * Note that the headers are independent of the language or image variant. * * @param ResourceLoaderContext $context Image context */ public function sendResponseHeaders(ResourceLoaderContext $context) { $format = $context->getFormat(); $mime = $this->getMimeType($format); $filename = $this->getName() . '.' . $this->getExtension($format); header('Content-Type: ' . $mime); header('Content-Disposition: ' . FileBackend::makeContentDisposition('inline', $filename)); }
/** * Append the final chunk and ready file for parent::performUpload() * @return FileRepoStatus */ public function concatenateChunks() { $chunkIndex = $this->getChunkIndex(); wfDebug(__METHOD__ . " concatenate {$this->mChunkIndex} chunks:" . $this->getOffset() . ' inx:' . $chunkIndex . "\n"); // Concatenate all the chunks to mVirtualTempPath $fileList = array(); // The first chunk is stored at the mVirtualTempPath path so we start on "chunk 1" for ($i = 0; $i <= $chunkIndex; $i++) { $fileList[] = $this->getVirtualChunkLocation($i); } // Get the file extension from the last chunk $ext = FileBackend::extensionFromPath($this->mVirtualTempPath); // Get a 0-byte temp file to perform the concatenation at $tmpFile = TempFSFile::factory('chunkedupload_', $ext); $tmpPath = false; // fail in concatenate() if ($tmpFile) { // keep alive with $this $tmpPath = $tmpFile->bind($this)->getPath(); } // Concatenate the chunks at the temp file $tStart = microtime(true); $status = $this->repo->concatenate($fileList, $tmpPath, FileRepo::DELETE_SOURCE); $tAmount = microtime(true) - $tStart; if (!$status->isOk()) { return $status; } wfDebugLog('fileconcatenate', "Combined {$i} chunks in {$tAmount} seconds."); // File system path $this->mTempPath = $tmpPath; // Since this was set for the last chunk previously $this->mFileSize = filesize($this->mTempPath); $ret = $this->verifyUpload(); if ($ret['status'] !== UploadBase::OK) { wfDebugLog('fileconcatenate', "Verification failed for chunked upload"); $status->fatal($this->getVerificationErrorCode($ret['status'])); return $status; } // Update the mTempPath and mLocalFile // (for FileUpload or normal Stash to take over) $tStart = microtime(true); $this->mLocalFile = parent::stashFile($this->user); $tAmount = microtime(true) - $tStart; $this->mLocalFile->setLocalReference($tmpFile); // reuse (e.g. for getImageInfo()) wfDebugLog('fileconcatenate', "Stashed combined file ({$i} chunks) in {$tAmount} seconds."); return $status; }
protected function doPerfTest(FileBackend $backend) { $ops1 = array(); $ops2 = array(); $ops3 = array(); $ops4 = array(); $ops5 = array(); $baseDir = 'mwstore://' . $backend->getName() . '/testing-cont1'; $backend->prepare(array('dir' => $baseDir)); $dirname = $this->getOption('srcdir'); $dir = opendir($dirname); if (!$dir) { return; } while ($dir && ($file = readdir($dir)) !== false) { if ($file[0] != '.') { $this->output("Using '{$dirname}/{$file}' in operations.\n"); $dst = $baseDir . '/' . wfBaseName($file); $ops1[] = array('op' => 'store', 'src' => "{$dirname}/{$file}", 'dst' => $dst, 'overwrite' => 1); $ops2[] = array('op' => 'copy', 'src' => "{$dst}", 'dst' => "{$dst}-1", 'overwrite' => 1); $ops3[] = array('op' => 'move', 'src' => $dst, 'dst' => "{$dst}-2", 'overwrite' => 1); $ops4[] = array('op' => 'delete', 'src' => "{$dst}-1"); $ops5[] = array('op' => 'delete', 'src' => "{$dst}-2"); } if (count($ops1) >= $this->getOption('maxfiles', 20)) { break; // enough } } closedir($dir); $this->output("\n"); $method = $this->hasOption('quick') ? 'doQuickOperations' : 'doOperations'; $opts = array('force' => 1); if ($this->hasOption('parallelize')) { $opts['parallelize'] = $this->getOption('parallelize') === 'true'; } $start = microtime(true); $status = $backend->{$method}($ops1, $opts); $e = (microtime(true) - $start) * 1000; if ($status->getErrorsArray()) { print_r($status->getErrorsArray()); exit(0); } $this->output($backend->getName() . ": Stored " . count($ops1) . " files in {$e} ms.\n"); $start = microtime(true); $backend->{$method}($ops2, $opts); $e = (microtime(true) - $start) * 1000; if ($status->getErrorsArray()) { print_r($status->getErrorsArray()); exit(0); } $this->output($backend->getName() . ": Copied " . count($ops2) . " files in {$e} ms.\n"); $start = microtime(true); $backend->{$method}($ops3, $opts); $e = (microtime(true) - $start) * 1000; if ($status->getErrorsArray()) { print_r($status->getErrorsArray()); exit(0); } $this->output($backend->getName() . ": Moved " . count($ops3) . " files in {$e} ms.\n"); $start = microtime(true); $backend->{$method}($ops4, $opts); $e = (microtime(true) - $start) * 1000; if ($status->getErrorsArray()) { print_r($status->getErrorsArray()); exit(0); } $this->output($backend->getName() . ": Deleted " . count($ops4) . " files in {$e} ms.\n"); $start = microtime(true); $backend->{$method}($ops5, $opts); $e = (microtime(true) - $start) * 1000; if ($status->getErrorsArray()) { print_r($status->getErrorsArray()); exit(0); } $this->output($backend->getName() . ": Deleted " . count($ops5) . " files in {$e} ms.\n"); }