/**
* Get an instance of SwiftStorage associated with specified DC
*
* @param $dc
* @return \Wikia\SwiftStorage
*/
private function getSwiftBackend($dc)
{
global $wgCityId;
static $cache = array();
if (empty($cache[$dc])) {
$cache[$dc] = \Wikia\SwiftStorage::newFromWiki($wgCityId, $dc);
}
return $cache[$dc];
}
/**
* Create container and authenticate - for destination Ceph/Swift storage
*
* @return Wikia\SwiftStorage storage instance
*/
private function destConn()
{
$city_id = $this->imageSyncQueueItem->city_id;
if (empty($this->dest_container[$city_id])) {
if ($city_id == 0) {
global $wgBlogAvatarSwiftContainer, $wgBlogAvatarSwiftPathPrefix;
$this->dest_container[$city_id] = \Wikia\SwiftStorage::newFromContainer($wgBlogAvatarSwiftContainer, $wgBlogAvatarSwiftPathPrefix, $this->mDC_dst);
} else {
$this->dest_container[$city_id] = \Wikia\SwiftStorage::newFromWiki($city_id, $this->mDC_dst);
}
}
return $this->dest_container[$city_id];
}
/**
* Set up the config variables
*/
private function init()
{
global $wgUploadDirectory, $wgDBname, $wgCityId;
$this->setupMd5Cache();
$dcs = explode(',', $this->getOption('dc', 'sjc,res'));
foreach ($dcs as $dc) {
// force a different bucket name via --bucket
$swiftBackend = \Wikia\SwiftStorage::newFromContainer('common', '/avatars', $dc);
$remotePath = $swiftBackend->getUrl('');
$this->output("Migrating avatars - <{$this->uploadDir}> -> <{$remotePath}> [dc: {$dc}]...\n");
$this->swiftBackends[$dc] = $swiftBackend;
$this->timePerDC[$dc] = 0;
}
}
public function execute()
{
global $wgCityId;
$this->storage = SwiftStorage::newFromWiki($wgCityId);
$container = $this->storage->getContainer();
$this->isDryRun = $this->hasOption('dry-run');
// get the list of WebP thumbs
$prefix = sprintf('%s/thumb', trim($this->storage->getPathPrefix(), '/'));
$this->output("Looking for thumbnails in <{$prefix}>...\n\n");
// marker for getting the next batch of images from DFS
$marker = null;
// stats
$scanned = 0;
$removed = 0;
// timestamps
$timestampFrom = strtotime(self::TIMESTAMP_FROM);
$timestampTo = strtotime(self::TIMESTAMP_TO);
do {
$this->output(" * Next batch...");
$thumbs = $container->get_objects(self::LIST_LIMIT, $marker, $prefix);
$scanned += count($thumbs);
// update the marker for getting the next batch
$marker = end($thumbs)->name;
$this->output(" ends with '{$marker}'\n");
foreach ($thumbs as $thumb) {
$name = $thumb->name;
if (endsWith($name, self::THUMB_SUFFIX)) {
$name = substr($name, strlen($this->storage->getPathPrefix()));
// remove path prefix from thumb path
// check the timestamp
$lastMod = strtotime($thumb->last_modified);
if ($lastMod > $timestampFrom && $lastMod < $timestampTo) {
$this->purgeThumb($name);
$removed++;
} else {
$this->output("Skipping {$name} ({$thumb->last_modified})\n");
}
}
}
} while (count($thumbs) === self::LIST_LIMIT);
$this->output("\n{$scanned} thumbnails scanned, {$removed} purged, we're done!\n");
}
/**
* Set up the config variables
*/
private function init()
{
global $wgUploadDirectory, $wgDBname, $wgCityId;
$this->shortBucketNameFixed = $this->fixShortBucketName();
$bucketName = $this->getOption('bucket', false);
$dcs = explode(',', $this->getOption('dc', 'sjc,res'));
foreach ($dcs as $dc) {
if (!is_string($bucketName)) {
// use bucket name taken from wiki upload path
$swiftBackend = \Wikia\SwiftStorage::newFromWiki($wgCityId, $dc);
} else {
// force a different bucket name via --bucket
$swiftBackend = \Wikia\SwiftStorage::newFromContainer($bucketName, '/images', $dc);
}
$remotePath = $swiftBackend->getUrl('');
$this->output("Migrating images on {$wgDBname} - <{$wgUploadDirectory}> -> <{$remotePath}> [dc: {$dc}]...\n");
$this->swiftBackends[$dc] = $swiftBackend;
$this->timePerDC[$dc] = 0;
}
}
public function testStoreAndRemove()
{
global $IP;
$swift = \Wikia\SwiftStorage::newFromContainer(self::CONTAINER);
// upload the file
$localFile = "{$IP}/skins/shared/images/sprite.png";
$remoteFile = sprintf('Test_%s.png', time());
$this->assertFalse($swift->exists($remoteFile), 'File should not exist before the upload');
$res = $swift->store($localFile, $remoteFile, [], 'image/png');
$this->assertTrue($res->isOK(), 'Upload should be done');
// check the uploaded file
$url = $swift->getUrl($remoteFile);
$this->assertStringEndsWith('/' . self::CONTAINER . '/' . $remoteFile, $url);
$this->assertTrue(Http::get($url, 'default', ['noProxy' => true]) !== false, 'Uploaded image should return HTTP 200 - ' . $url);
$this->assertTrue($swift->exists($remoteFile), 'File should exist');
// npw remove the file
$res = $swift->remove($remoteFile);
$this->assertTrue($res->isOK(), 'Delete should be done');
$this->assertTrue(Http::get($url, 'default', ['noProxy' => true]) === false, 'Removed image should return HTTP 404 - ' . $url);
$this->assertFalse($swift->exists($remoteFile), 'File should not exist after the delete');
}
/**
* @param LocalFile $file file to check
* @return int RESULT_* flag
* @see BAC-731
*/
private function processMissingFile(File $file)
{
global $wgUploadDirectory, $wgCityId;
try {
$exists = $this->repo->fileExists($file->getPath());
} catch (Exception $ex) {
$exists = true;
$this->error(sprintf("%s caught: %s", get_class($ex), $ex->getMessage()));
}
// file is fine, continue...
if ($exists) {
return self::RESULT_EXISTS;
}
$restored = false;
$this->output(sprintf("'%s' doesn't exist (%s)\n", $file->getTitle(), $file->getUrlRel()));
// let's assume that given file was moved from A
// let's get all possible A's and try to find images for them
$candidates = $this->getCandidates($file);
if (empty($candidates) && !empty($this->otherLocation)) {
# check other location - maybe this file is there :)
$candidates = $this->checkOtherLocation($file);
}
if (!empty($candidates)) {
$this->output(sprintf(" %d candidate(s) found...\n", count($candidates)));
foreach ($candidates as $candidate) {
$srcFile = LocalFile::newFromTitle($candidate, $this->repo);
$srcPath = $wgUploadDirectory . '/' . $srcFile->getUrlRel();
// check on FS storage
$foundOnFS = file_exists($srcPath);
$this->output(sprintf(" '%s' -> <%s> [%s]\n", $srcFile->getName(), $srcPath, $foundOnFS ? 'found' : 'not found'));
// check the next candidate (or if --dry-run)
if (!$foundOnFS || $this->isDryRun) {
continue;
}
// upload found image to Swift
$swift = \Wikia\SwiftStorage::newFromWiki($wgCityId);
$metadata = ['Sha1Base36' => $file->getSha1()];
$status = $swift->store($srcPath, $file->getUrlRel(), $metadata, $file->getMimeType());
if ($status->isOK()) {
self::log('restored', $file->getName());
$restored = true;
break;
}
}
$this->output("\n");
}
// remove an image if it can't be restored
if (!$restored && !$this->isDryRun) {
$file->delete(self::REASON);
$this->output(sprintf(" Removed '%s'!\n", $file->getName()));
self::log('removed', $file->getName());
}
return $restored ? self::RESULT_RESTORED : self::RESULT_NOT_RESTORED;
}
/**
* Get Swift storage instance for avatars
*
* @return \Wikia\SwiftStorage storage instance
*/
public function getSwiftStorage()
{
global $wgBlogAvatarSwiftContainer, $wgBlogAvatarSwiftPathPrefix;
return \Wikia\SwiftStorage::newFromContainer($wgBlogAvatarSwiftContainer, $wgBlogAvatarSwiftPathPrefix);
}
/**
* Check whether given file exists on Math storage (either NFS or Swift)
*
* @return bool true if rendered PNG exists
*/
function _recall()
{
global $wgMathDirectory, $wgMathCheckFiles;
$this->md5 = md5($this->tex);
$dbr = wfGetDB(DB_SLAVE);
$rpage = $dbr->selectRow('math', array('math_outputhash', 'math_html_conservativeness', 'math_html', 'math_mathml'), array('math_inputhash' => $dbr->encodeBlob(pack("H32", $this->md5))), __METHOD__);
if ($rpage !== false) {
# Tailing 0x20s can get dropped by the database, add it back on if necessary:
$xhash = unpack('H32md5', $dbr->decodeBlob($rpage->math_outputhash) . " ");
$this->hash = $xhash['md5'];
$this->conservativeness = $rpage->math_html_conservativeness;
$this->html = $rpage->math_html;
$this->mathml = $rpage->math_mathml;
$filename = $this->_getHashPath() . "/{$this->hash}.png";
wfDebug(__METHOD__ . ": rendering to {$filename}\n");
// Wikia change
if (!$wgMathCheckFiles) {
// Short-circuit the file existence & migration checks
return true;
}
// Wikia change - begin
// use Swift storage instead of NFS mount (@author macbre)
global $wgEnableSwiftFileBackend, $wgCityId;
if (!empty($wgEnableSwiftFileBackend)) {
$swift = \Wikia\SwiftStorage::newFromWiki($wgCityId);
$remotePath = $this->getSwiftPath();
return $swift->exists($remotePath);
}
// Wikia change - end
if (file_exists($filename)) {
if (filesize($filename) == 0) {
// Some horrible error corrupted stuff :(
wfSuppressWarnings();
unlink($filename);
wfRestoreWarnings();
} else {
return true;
}
}
if (file_exists($wgMathDirectory . "/{$this->hash}.png")) {
$hashpath = $this->_getHashPath();
if (!file_exists($hashpath)) {
wfSuppressWarnings();
$ret = wfMkdirParents($hashpath, 0755, __METHOD__);
wfRestoreWarnings();
if (!$ret) {
wfDebug(__METHOD__ . ": failed to create directory tree - {$hashpath}\n");
// Wikia change
return false;
}
} elseif (!is_dir($hashpath) || !is_writable($hashpath)) {
wfDebug(__METHOD__ . ": can't write to {$hashpath}\n");
// Wikia change
return false;
}
if (function_exists('link')) {
return link($wgMathDirectory . "/{$this->hash}.png", $hashpath . "/{$this->hash}.png");
} else {
return rename($wgMathDirectory . "/{$this->hash}.png", $hashpath . "/{$this->hash}.png");
}
}
}
# Missing from the database and/or the render cache
return false;
}
