protected function processQueue() { global $wgFSSwiftDC; $this->logDebug(sprintf("Found %d entries in md5 cache", $this->md5Cache->getCachedCount())); $targets = array(); foreach ($this->swiftBackends as $dc => $swift) { $hostnames = $wgFSSwiftDC[$dc]['servers']; if ($this->hammer !== null) { if (!in_array($this->hammer, $hostnames)) { $this->logError("Skipping DC {$dc} due to --hammer argument"); continue; } else { $hostnames = array($this->hammer); $this->logError("Using only {$this->hammer} for DC {$dc}"); } } $authConfig = $wgFSSwiftDC[$dc]['config']; $cluster = new \Wikia\Swift\Net\Cluster($dc, $hostnames, $authConfig); $container = new \Wikia\Swift\Entity\Container($swift->getContainerName()); $targets[] = new \Wikia\Swift\Wiki\Target($cluster, $container); } if (empty($targets)) { $this->logError("No DC remaining after applying --hammer"); return; } $files = $this->allFiles; if ($this->useDiff) { $migration = new \Wikia\Swift\Wiki\DiffMigration($targets, $files); $remoteFilter = new \Wikia\Swift\Wiki\WikiFilesFilter($this->pathPrefix); $migration->setRemoteFilter($remoteFilter); $migration->setUseDeletes($this->useDeletes); } else { $migration = new \Wikia\Swift\Wiki\SimpleMigration($targets, $files); } $migration->setThreads(400); $migration->copyLogger($this); $migration->run(); $invalid = $migration->getInvalid(); $completed = $migration->getCompleted(); $failed = $migration->getFailed(); $this->migratedImagesCnt = count($invalid); foreach ($migration->getListNames() as $listName) { $completedCount = array_key_exists($listName, $completed) ? count($completed[$listName]) : 0; $failedCount = array_key_exists($listName, $failed) ? count($failed[$listName]) : 0; $this->logError(sprintf("%s: completed=%d failed=%d invalid=%d", $listName, $completedCount, $failedCount, count($invalid))); $this->migratedImagesCnt += $completedCount; $this->migratedImagesCnt += $failedCount; $this->migratedImagesFailedCnt += $failedCount; $results = $failed[$listName]; /** @var \Wikia\Swift\Transaction\OperationResult $result */ foreach ($results as $result) { $result->logError($result->getError()); // re-emit logs with increased severity $logs = $result->getLogs(); foreach ($logs as $message) { $result->logError($message); } } } $this->logInfo("Finished migration."); }