public function execute()
{
global $wgExternalSharedDB;
$this->output("md5 calculation started ... \n");
$this->time = time();
$migrated = 0;
$limit = $this->getOption('limit', -1);
$debug = $this->hasOption('debug');
$wikis = $this->getOption('wiki', '');
$this->db = $this->getDB(DB_SLAVE, array(), $wgExternalSharedDB);
$where = array();
if (!empty($wikis)) {
$where['city_id'] = explode(',', $wikis);
}
$order = $this->hasOption('reverse') ? " DESC" : "";
$res = $this->db->select(array('city_list'), array('city_id', 'city_dbname'), $where, __CLASS__, array_merge(array('ORDER BY' => "city_last_timestamp{$order}, city_id{$order}"), $limit > 0 ? array('LIMIT' => $limit) : array()));
$this->output("Building list of wiki IDs...\n");
$queue = array();
while ($row = $res->fetchObject()) {
$id = intval($row->city_id);
$dbname = $row->city_dbname;
$queue[$id] = $dbname;
}
$this->output(sprintf("Scheduling %d wikis for migration...\n", count($queue)));
$this->output("\nRun migrateImagesToSwift script \n");
$this->output("Building list of processes to run...\n");
$processes = array();
foreach ($queue as $id => $dbname) {
$processes[] = $this->getProcess($id);
}
$threads = $this->getOption('threads', self::THREADS_DEFAULT);
$this->output("Using {$threads} threads...\n");
$runner = new \Wikia\Swift\Process\Runner($processes, $threads);
$runner->run();
$this->output(sprintf("\nMigrated %d Wikis in %s\n", $migrated, Wikia::timeDuration(time() - $this->time)));
$this->output("\nDone!\n");
}
public function execute()
{
global $wgExternalSharedDB, $wgWikiaDatacenter;
$this->output("Wikis migration started ... \n");
$this->time = time();
$migrated = 0;
$limit = $this->getOption('limit', -1);
$debug = $this->hasOption('debug');
$force = $this->hasOption('force');
$wikis = $this->getOption('wiki', null);
$forceAll = $this->hasOption('all');
$this->dryRun = $this->hasOption('dry-run');
$this->noDeletes = $this->hasOption('no-deletes');
$this->calculateMd5 = $this->hasOption('md5');
$this->syncSecondary = $this->hasOption('sync');
if ($this->syncSecondary) {
$force = true;
$forceAll = true;
}
$this->dc = $this->getOption('dc', 'sjc,res');
# don't migrate top 200 Wikis
$top200Wikis = array();
if (!$forceAll) {
$top200Wikis = DataMartService::getWAM200Wikis();
if (count($top200Wikis) != 200) {
$this->output("Number of Top 200 Wikis is different than 200 !\n");
exit;
}
if ($debug) {
$this->output("Top 200 Wikis: " . implode(", ", $top200Wikis) . "\n");
}
}
# don't migrate video.wikia.com & corp.wikia.com
$this->disabled_wikis = array_merge($top200Wikis, $this->disabled_wikis);
foreach ($this->disabled_wikis as $k => $v) {
$this->disabled_wikis[$k] = intval($v);
}
if ($this->syncSecondary) {
$this->disabled_wikis = array();
}
$wikiIds = null;
if ($wikis !== null) {
$wikiIds = array();
foreach (explode(',', $wikis) as $id) {
if (is_numeric($id) && $id >= 0) {
$wikiIds[] = $id;
}
}
if (count($wikiIds) == 0) {
$wikiIds = null;
}
}
$this->db = $this->getDB(DB_SLAVE, array(), $wgExternalSharedDB);
$order = $this->hasOption('reverse') ? " DESC" : "";
$res = $this->db->select(array('city_list', 'city_variables'), array('city_id', 'city_dbname'), array_merge(array('city_public' => 1), !$force ? array('cv_value is null or cv_value != "b:1;"') : array(), is_array($wikiIds) ? array('city_id' => $wikiIds) : array()), __CLASS__, array_merge(array('ORDER BY' => "city_last_timestamp{$order}, city_id{$order}"), $limit > 0 ? array('LIMIT' => $limit) : array()), array('city_variables' => array('LEFT JOIN', 'city_list.city_id = city_variables.cv_city_id and cv_variable_id = 1334')));
$this->output(sprintf("Found %d wikis in database...\n", $res->numRows()));
$this->output("Building list of wiki IDs...\n");
$removedCount = 0;
$queue = array();
while ($row = $res->fetchObject()) {
$id = intval($row->city_id);
$dbname = $row->city_dbname;
if (!in_array($id, $this->disabled_wikis)) {
$queue[$id] = $dbname;
} else {
$removedCount++;
}
}
$this->output(sprintf("Skipped %d wikis that are on blacklist...\n", $removedCount));
$this->output(sprintf("Scheduling %d wikis for migration...\n", count($queue)));
if ($this->hasOption('stats-only')) {
return;
}
$this->output("\nRun migrateImagesToSwift script \n");
$this->output("Building list of processes to run...\n");
$processes = array();
foreach ($queue as $id => $dbname) {
if ($this->calculateMd5) {
$process = $this->getMd5Process($id, $dbname);
} elseif ($this->syncSecondary) {
$process = $this->getSyncProcess($id, $dbname);
} else {
$process = $this->getMigrationProcess($id, $dbname);
}
$processes[] = $process;
}
$threads = $this->getOption('threads', self::THREADS_DEFAULT);
$threads = intval($threads);
$threadDelay = $this->getOption('delay', self::DELAY_DEFAULT);
$threadDelay = intval($threadDelay);
$this->output("Using {$threads} threads...\n");
$runner = new \Wikia\Swift\Process\Runner($processes, $threads, $threadDelay);
$runner->run();
$this->output(sprintf("\nMigrated %d Wikis in %s\n", $migrated, Wikia::timeDuration(time() - $this->time)));
$this->output("\nDone!\n");
}
