/** * Get an instance of SwiftStorage associated with specified DC * * @param $dc * @return \Wikia\SwiftStorage */ private function getSwiftBackend($dc) { global $wgCityId; static $cache = array(); if (empty($cache[$dc])) { $cache[$dc] = \Wikia\SwiftStorage::newFromWiki($wgCityId, $dc); } return $cache[$dc]; }
/** * Create container and authenticate - for destination Ceph/Swift storage * * @return Wikia\SwiftStorage storage instance */ private function destConn() { $city_id = $this->imageSyncQueueItem->city_id; if (empty($this->dest_container[$city_id])) { if ($city_id == 0) { global $wgBlogAvatarSwiftContainer, $wgBlogAvatarSwiftPathPrefix; $this->dest_container[$city_id] = \Wikia\SwiftStorage::newFromContainer($wgBlogAvatarSwiftContainer, $wgBlogAvatarSwiftPathPrefix, $this->mDC_dst); } else { $this->dest_container[$city_id] = \Wikia\SwiftStorage::newFromWiki($city_id, $this->mDC_dst); } } return $this->dest_container[$city_id]; }
/** * Set up the config variables */ private function init() { global $wgUploadDirectory, $wgDBname, $wgCityId; $this->setupMd5Cache(); $dcs = explode(',', $this->getOption('dc', 'sjc,res')); foreach ($dcs as $dc) { // force a different bucket name via --bucket $swiftBackend = \Wikia\SwiftStorage::newFromContainer('common', '/avatars', $dc); $remotePath = $swiftBackend->getUrl(''); $this->output("Migrating avatars - <{$this->uploadDir}> -> <{$remotePath}> [dc: {$dc}]...\n"); $this->swiftBackends[$dc] = $swiftBackend; $this->timePerDC[$dc] = 0; } }
public function execute() { global $wgCityId; $this->storage = SwiftStorage::newFromWiki($wgCityId); $container = $this->storage->getContainer(); $this->isDryRun = $this->hasOption('dry-run'); // get the list of WebP thumbs $prefix = sprintf('%s/thumb', trim($this->storage->getPathPrefix(), '/')); $this->output("Looking for thumbnails in <{$prefix}>...\n\n"); // marker for getting the next batch of images from DFS $marker = null; // stats $scanned = 0; $removed = 0; // timestamps $timestampFrom = strtotime(self::TIMESTAMP_FROM); $timestampTo = strtotime(self::TIMESTAMP_TO); do { $this->output(" * Next batch..."); $thumbs = $container->get_objects(self::LIST_LIMIT, $marker, $prefix); $scanned += count($thumbs); // update the marker for getting the next batch $marker = end($thumbs)->name; $this->output(" ends with '{$marker}'\n"); foreach ($thumbs as $thumb) { $name = $thumb->name; if (endsWith($name, self::THUMB_SUFFIX)) { $name = substr($name, strlen($this->storage->getPathPrefix())); // remove path prefix from thumb path // check the timestamp $lastMod = strtotime($thumb->last_modified); if ($lastMod > $timestampFrom && $lastMod < $timestampTo) { $this->purgeThumb($name); $removed++; } else { $this->output("Skipping {$name} ({$thumb->last_modified})\n"); } } } } while (count($thumbs) === self::LIST_LIMIT); $this->output("\n{$scanned} thumbnails scanned, {$removed} purged, we're done!\n"); }
/** * Set up the config variables */ private function init() { global $wgUploadDirectory, $wgDBname, $wgCityId; $this->shortBucketNameFixed = $this->fixShortBucketName(); $bucketName = $this->getOption('bucket', false); $dcs = explode(',', $this->getOption('dc', 'sjc,res')); foreach ($dcs as $dc) { if (!is_string($bucketName)) { // use bucket name taken from wiki upload path $swiftBackend = \Wikia\SwiftStorage::newFromWiki($wgCityId, $dc); } else { // force a different bucket name via --bucket $swiftBackend = \Wikia\SwiftStorage::newFromContainer($bucketName, '/images', $dc); } $remotePath = $swiftBackend->getUrl(''); $this->output("Migrating images on {$wgDBname} - <{$wgUploadDirectory}> -> <{$remotePath}> [dc: {$dc}]...\n"); $this->swiftBackends[$dc] = $swiftBackend; $this->timePerDC[$dc] = 0; } }
public function testStoreAndRemove() { global $IP; $swift = \Wikia\SwiftStorage::newFromContainer(self::CONTAINER); // upload the file $localFile = "{$IP}/skins/shared/images/sprite.png"; $remoteFile = sprintf('Test_%s.png', time()); $this->assertFalse($swift->exists($remoteFile), 'File should not exist before the upload'); $res = $swift->store($localFile, $remoteFile, [], 'image/png'); $this->assertTrue($res->isOK(), 'Upload should be done'); // check the uploaded file $url = $swift->getUrl($remoteFile); $this->assertStringEndsWith('/' . self::CONTAINER . '/' . $remoteFile, $url); $this->assertTrue(Http::get($url, 'default', ['noProxy' => true]) !== false, 'Uploaded image should return HTTP 200 - ' . $url); $this->assertTrue($swift->exists($remoteFile), 'File should exist'); // npw remove the file $res = $swift->remove($remoteFile); $this->assertTrue($res->isOK(), 'Delete should be done'); $this->assertTrue(Http::get($url, 'default', ['noProxy' => true]) === false, 'Removed image should return HTTP 404 - ' . $url); $this->assertFalse($swift->exists($remoteFile), 'File should not exist after the delete'); }
/** * @param LocalFile $file file to check * @return int RESULT_* flag * @see BAC-731 */ private function processMissingFile(File $file) { global $wgUploadDirectory, $wgCityId; try { $exists = $this->repo->fileExists($file->getPath()); } catch (Exception $ex) { $exists = true; $this->error(sprintf("%s caught: %s", get_class($ex), $ex->getMessage())); } // file is fine, continue... if ($exists) { return self::RESULT_EXISTS; } $restored = false; $this->output(sprintf("'%s' doesn't exist (%s)\n", $file->getTitle(), $file->getUrlRel())); // let's assume that given file was moved from A // let's get all possible A's and try to find images for them $candidates = $this->getCandidates($file); if (empty($candidates) && !empty($this->otherLocation)) { # check other location - maybe this file is there :) $candidates = $this->checkOtherLocation($file); } if (!empty($candidates)) { $this->output(sprintf(" %d candidate(s) found...\n", count($candidates))); foreach ($candidates as $candidate) { $srcFile = LocalFile::newFromTitle($candidate, $this->repo); $srcPath = $wgUploadDirectory . '/' . $srcFile->getUrlRel(); // check on FS storage $foundOnFS = file_exists($srcPath); $this->output(sprintf(" '%s' -> <%s> [%s]\n", $srcFile->getName(), $srcPath, $foundOnFS ? 'found' : 'not found')); // check the next candidate (or if --dry-run) if (!$foundOnFS || $this->isDryRun) { continue; } // upload found image to Swift $swift = \Wikia\SwiftStorage::newFromWiki($wgCityId); $metadata = ['Sha1Base36' => $file->getSha1()]; $status = $swift->store($srcPath, $file->getUrlRel(), $metadata, $file->getMimeType()); if ($status->isOK()) { self::log('restored', $file->getName()); $restored = true; break; } } $this->output("\n"); } // remove an image if it can't be restored if (!$restored && !$this->isDryRun) { $file->delete(self::REASON); $this->output(sprintf(" Removed '%s'!\n", $file->getName())); self::log('removed', $file->getName()); } return $restored ? self::RESULT_RESTORED : self::RESULT_NOT_RESTORED; }
/** * Get Swift storage instance for avatars * * @return \Wikia\SwiftStorage storage instance */ public function getSwiftStorage() { global $wgBlogAvatarSwiftContainer, $wgBlogAvatarSwiftPathPrefix; return \Wikia\SwiftStorage::newFromContainer($wgBlogAvatarSwiftContainer, $wgBlogAvatarSwiftPathPrefix); }
/** * Check whether given file exists on Math storage (either NFS or Swift) * * @return bool true if rendered PNG exists */ function _recall() { global $wgMathDirectory, $wgMathCheckFiles; $this->md5 = md5($this->tex); $dbr = wfGetDB(DB_SLAVE); $rpage = $dbr->selectRow('math', array('math_outputhash', 'math_html_conservativeness', 'math_html', 'math_mathml'), array('math_inputhash' => $dbr->encodeBlob(pack("H32", $this->md5))), __METHOD__); if ($rpage !== false) { # Tailing 0x20s can get dropped by the database, add it back on if necessary: $xhash = unpack('H32md5', $dbr->decodeBlob($rpage->math_outputhash) . " "); $this->hash = $xhash['md5']; $this->conservativeness = $rpage->math_html_conservativeness; $this->html = $rpage->math_html; $this->mathml = $rpage->math_mathml; $filename = $this->_getHashPath() . "/{$this->hash}.png"; wfDebug(__METHOD__ . ": rendering to {$filename}\n"); // Wikia change if (!$wgMathCheckFiles) { // Short-circuit the file existence & migration checks return true; } // Wikia change - begin // use Swift storage instead of NFS mount (@author macbre) global $wgEnableSwiftFileBackend, $wgCityId; if (!empty($wgEnableSwiftFileBackend)) { $swift = \Wikia\SwiftStorage::newFromWiki($wgCityId); $remotePath = $this->getSwiftPath(); return $swift->exists($remotePath); } // Wikia change - end if (file_exists($filename)) { if (filesize($filename) == 0) { // Some horrible error corrupted stuff :( wfSuppressWarnings(); unlink($filename); wfRestoreWarnings(); } else { return true; } } if (file_exists($wgMathDirectory . "/{$this->hash}.png")) { $hashpath = $this->_getHashPath(); if (!file_exists($hashpath)) { wfSuppressWarnings(); $ret = wfMkdirParents($hashpath, 0755, __METHOD__); wfRestoreWarnings(); if (!$ret) { wfDebug(__METHOD__ . ": failed to create directory tree - {$hashpath}\n"); // Wikia change return false; } } elseif (!is_dir($hashpath) || !is_writable($hashpath)) { wfDebug(__METHOD__ . ": can't write to {$hashpath}\n"); // Wikia change return false; } if (function_exists('link')) { return link($wgMathDirectory . "/{$this->hash}.png", $hashpath . "/{$this->hash}.png"); } else { return rename($wgMathDirectory . "/{$this->hash}.png", $hashpath . "/{$this->hash}.png"); } } } # Missing from the database and/or the render cache return false; }