protected function setupMd5Cache() { global $wgDBname; $md5Cache = \Wikia\Swift\File\Md5Cache::getInstance(); $md5CacheFile = sprintf("%s/%s/%s/%s.md5list", self::MD5_CACHE_DIRECTORY, substr($wgDBname, 0, 1), substr($wgDBname, 0, 2), $wgDBname); $md5Cache->setCacheFile($md5CacheFile); $this->md5Cache = $md5Cache; }
public function loadMd5() { if ($this->md5 === null) { $this->md5 = Md5Cache::getInstance()->get($this->localPath); } }
protected function processQueue() { global $wgFSSwiftDC; $this->logDebug(sprintf("Found %d entries in md5 cache", $this->md5Cache->getCachedCount())); $targets = array(); foreach ($this->swiftBackends as $dc => $swift) { $hostnames = $wgFSSwiftDC[$dc]['servers']; if ($this->hammer !== null) { if (!in_array($this->hammer, $hostnames)) { $this->logError("Skipping DC {$dc} due to --hammer argument"); continue; } else { $hostnames = array($this->hammer); $this->logError("Using only {$this->hammer} for DC {$dc}"); } } $authConfig = $wgFSSwiftDC[$dc]['config']; $cluster = new \Wikia\Swift\Net\Cluster($dc, $hostnames, $authConfig); $container = new \Wikia\Swift\Entity\Container($swift->getContainerName()); $targets[] = new \Wikia\Swift\Wiki\Target($cluster, $container); } if (empty($targets)) { $this->logError("No DC remaining after applying --hammer"); return; } $files = $this->allFiles; if ($this->useDiff) { $migration = new \Wikia\Swift\Wiki\DiffMigration($targets, $files); $remoteFilter = new \Wikia\Swift\Wiki\WikiFilesFilter($this->pathPrefix); $migration->setRemoteFilter($remoteFilter); $migration->setUseDeletes($this->useDeletes); } else { $migration = new \Wikia\Swift\Wiki\SimpleMigration($targets, $files); } $migration->setThreads(400); $migration->copyLogger($this); $migration->run(); $invalid = $migration->getInvalid(); $completed = $migration->getCompleted(); $failed = $migration->getFailed(); $this->migratedImagesCnt = count($invalid); foreach ($migration->getListNames() as $listName) { $completedCount = array_key_exists($listName, $completed) ? count($completed[$listName]) : 0; $failedCount = array_key_exists($listName, $failed) ? count($failed[$listName]) : 0; $this->logError(sprintf("%s: completed=%d failed=%d invalid=%d", $listName, $completedCount, $failedCount, count($invalid))); $this->migratedImagesCnt += $completedCount; $this->migratedImagesCnt += $failedCount; $this->migratedImagesFailedCnt += $failedCount; $results = $failed[$listName]; /** @var \Wikia\Swift\Transaction\OperationResult $result */ foreach ($results as $result) { $result->logError($result->getError()); // re-emit logs with increased severity $logs = $result->getLogs(); foreach ($logs as $message) { $result->logError($message); } } } $this->logInfo("Finished migration."); }