protected function setupMd5Cache()
{
global $wgDBname;
$md5Cache = \Wikia\Swift\File\Md5Cache::getInstance();
$md5CacheFile = sprintf("%s/%s/%s/%s.md5list", self::MD5_CACHE_DIRECTORY, substr($wgDBname, 0, 1), substr($wgDBname, 0, 2), $wgDBname);
$md5Cache->setCacheFile($md5CacheFile);
$this->md5Cache = $md5Cache;
}
public function loadMd5()
{
if ($this->md5 === null) {
$this->md5 = Md5Cache::getInstance()->get($this->localPath);
}
}
protected function processQueue()
{
global $wgFSSwiftDC;
$this->logDebug(sprintf("Found %d entries in md5 cache", $this->md5Cache->getCachedCount()));
$targets = array();
foreach ($this->swiftBackends as $dc => $swift) {
$hostnames = $wgFSSwiftDC[$dc]['servers'];
if ($this->hammer !== null) {
if (!in_array($this->hammer, $hostnames)) {
$this->logError("Skipping DC {$dc} due to --hammer argument");
continue;
} else {
$hostnames = array($this->hammer);
$this->logError("Using only {$this->hammer} for DC {$dc}");
}
}
$authConfig = $wgFSSwiftDC[$dc]['config'];
$cluster = new \Wikia\Swift\Net\Cluster($dc, $hostnames, $authConfig);
$container = new \Wikia\Swift\Entity\Container($swift->getContainerName());
$targets[] = new \Wikia\Swift\Wiki\Target($cluster, $container);
}
if (empty($targets)) {
$this->logError("No DC remaining after applying --hammer");
return;
}
$files = $this->allFiles;
if ($this->useDiff) {
$migration = new \Wikia\Swift\Wiki\DiffMigration($targets, $files);
$remoteFilter = new \Wikia\Swift\Wiki\WikiFilesFilter($this->pathPrefix);
$migration->setRemoteFilter($remoteFilter);
$migration->setUseDeletes($this->useDeletes);
} else {
$migration = new \Wikia\Swift\Wiki\SimpleMigration($targets, $files);
}
$migration->setThreads(400);
$migration->copyLogger($this);
$migration->run();
$invalid = $migration->getInvalid();
$completed = $migration->getCompleted();
$failed = $migration->getFailed();
$this->migratedImagesCnt = count($invalid);
foreach ($migration->getListNames() as $listName) {
$completedCount = array_key_exists($listName, $completed) ? count($completed[$listName]) : 0;
$failedCount = array_key_exists($listName, $failed) ? count($failed[$listName]) : 0;
$this->logError(sprintf("%s: completed=%d failed=%d invalid=%d", $listName, $completedCount, $failedCount, count($invalid)));
$this->migratedImagesCnt += $completedCount;
$this->migratedImagesCnt += $failedCount;
$this->migratedImagesFailedCnt += $failedCount;
$results = $failed[$listName];
/** @var \Wikia\Swift\Transaction\OperationResult $result */
foreach ($results as $result) {
$result->logError($result->getError());
// re-emit logs with increased severity
$logs = $result->getLogs();
foreach ($logs as $message) {
$result->logError($message);
}
}
}
$this->logInfo("Finished migration.");
}