public function execute() { global $wgExternalSharedDB; $this->output("md5 calculation started ... \n"); $this->time = time(); $migrated = 0; $limit = $this->getOption('limit', -1); $debug = $this->hasOption('debug'); $wikis = $this->getOption('wiki', ''); $this->db = $this->getDB(DB_SLAVE, array(), $wgExternalSharedDB); $where = array(); if (!empty($wikis)) { $where['city_id'] = explode(',', $wikis); } $order = $this->hasOption('reverse') ? " DESC" : ""; $res = $this->db->select(array('city_list'), array('city_id', 'city_dbname'), $where, __CLASS__, array_merge(array('ORDER BY' => "city_last_timestamp{$order}, city_id{$order}"), $limit > 0 ? array('LIMIT' => $limit) : array())); $this->output("Building list of wiki IDs...\n"); $queue = array(); while ($row = $res->fetchObject()) { $id = intval($row->city_id); $dbname = $row->city_dbname; $queue[$id] = $dbname; } $this->output(sprintf("Scheduling %d wikis for migration...\n", count($queue))); $this->output("\nRun migrateImagesToSwift script \n"); $this->output("Building list of processes to run...\n"); $processes = array(); foreach ($queue as $id => $dbname) { $processes[] = $this->getProcess($id); } $threads = $this->getOption('threads', self::THREADS_DEFAULT); $this->output("Using {$threads} threads...\n"); $runner = new \Wikia\Swift\Process\Runner($processes, $threads); $runner->run(); $this->output(sprintf("\nMigrated %d Wikis in %s\n", $migrated, Wikia::timeDuration(time() - $this->time))); $this->output("\nDone!\n"); }
public function execute() { global $wgExternalSharedDB, $wgWikiaDatacenter; $this->output("Wikis migration started ... \n"); $this->time = time(); $migrated = 0; $limit = $this->getOption('limit', -1); $debug = $this->hasOption('debug'); $force = $this->hasOption('force'); $wikis = $this->getOption('wiki', null); $forceAll = $this->hasOption('all'); $this->dryRun = $this->hasOption('dry-run'); $this->noDeletes = $this->hasOption('no-deletes'); $this->calculateMd5 = $this->hasOption('md5'); $this->syncSecondary = $this->hasOption('sync'); if ($this->syncSecondary) { $force = true; $forceAll = true; } $this->dc = $this->getOption('dc', 'sjc,res'); # don't migrate top 200 Wikis $top200Wikis = array(); if (!$forceAll) { $top200Wikis = DataMartService::getWAM200Wikis(); if (count($top200Wikis) != 200) { $this->output("Number of Top 200 Wikis is different than 200 !\n"); exit; } if ($debug) { $this->output("Top 200 Wikis: " . implode(", ", $top200Wikis) . "\n"); } } # don't migrate video.wikia.com & corp.wikia.com $this->disabled_wikis = array_merge($top200Wikis, $this->disabled_wikis); foreach ($this->disabled_wikis as $k => $v) { $this->disabled_wikis[$k] = intval($v); } if ($this->syncSecondary) { $this->disabled_wikis = array(); } $wikiIds = null; if ($wikis !== null) { $wikiIds = array(); foreach (explode(',', $wikis) as $id) { if (is_numeric($id) && $id >= 0) { $wikiIds[] = $id; } } if (count($wikiIds) == 0) { $wikiIds = null; } } $this->db = $this->getDB(DB_SLAVE, array(), $wgExternalSharedDB); $order = $this->hasOption('reverse') ? " DESC" : ""; $res = $this->db->select(array('city_list', 'city_variables'), array('city_id', 'city_dbname'), array_merge(array('city_public' => 1), !$force ? array('cv_value is null or cv_value != "b:1;"') : array(), is_array($wikiIds) ? array('city_id' => $wikiIds) : array()), __CLASS__, array_merge(array('ORDER BY' => "city_last_timestamp{$order}, city_id{$order}"), $limit > 0 ? array('LIMIT' => $limit) : array()), array('city_variables' => array('LEFT JOIN', 'city_list.city_id = city_variables.cv_city_id and cv_variable_id = 1334'))); $this->output(sprintf("Found %d wikis in database...\n", $res->numRows())); $this->output("Building list of wiki IDs...\n"); $removedCount = 0; $queue = array(); while ($row = $res->fetchObject()) { $id = intval($row->city_id); $dbname = $row->city_dbname; if (!in_array($id, $this->disabled_wikis)) { $queue[$id] = $dbname; } else { $removedCount++; } } $this->output(sprintf("Skipped %d wikis that are on blacklist...\n", $removedCount)); $this->output(sprintf("Scheduling %d wikis for migration...\n", count($queue))); if ($this->hasOption('stats-only')) { return; } $this->output("\nRun migrateImagesToSwift script \n"); $this->output("Building list of processes to run...\n"); $processes = array(); foreach ($queue as $id => $dbname) { if ($this->calculateMd5) { $process = $this->getMd5Process($id, $dbname); } elseif ($this->syncSecondary) { $process = $this->getSyncProcess($id, $dbname); } else { $process = $this->getMigrationProcess($id, $dbname); } $processes[] = $process; } $threads = $this->getOption('threads', self::THREADS_DEFAULT); $threads = intval($threads); $threadDelay = $this->getOption('delay', self::DELAY_DEFAULT); $threadDelay = intval($threadDelay); $this->output("Using {$threads} threads...\n"); $runner = new \Wikia\Swift\Process\Runner($processes, $threads, $threadDelay); $runner->run(); $this->output(sprintf("\nMigrated %d Wikis in %s\n", $migrated, Wikia::timeDuration(time() - $this->time))); $this->output("\nDone!\n"); }