/** * 1. go through all wikis which are marked for closing and check which one * want to have images packed. * * 2. pack images, send them via rsync to target server, * * 3. mark in city_list.city_flags that images are sent, * * 4. remove images * * @access public */ public function execute() { global $wgUploadDirectory, $wgDBname, $IP; $first = isset($this->mOptions["first"]) ? true : false; $sleep = isset($this->mOptions["sleep"]) ? $this->mOptions["sleep"] : 15; $condition = array("ORDER BY" => "city_id"); $this->info('start', ['first' => $first, 'limit' => $this->mOptions["limit"] ?: false]); /** * if $first is set skip limit checking */ if (!$first) { if (isset($this->mOptions["limit"]) && is_numeric($this->mOptions["limit"])) { $condition["LIMIT"] = $this->mOptions["limit"]; } } $timestamp = wfTimestamp(TS_DB, strtotime(sprintf("-%d days", self::CLOSE_WIKI_DELAY))); $dbr = WikiFactory::db(DB_SLAVE); $sth = $dbr->select(array("city_list"), array("city_id", "city_flags", "city_dbname", "city_url", "city_public"), array("city_public" => array(0, -1), "city_flags <> 0 && city_flags <> 32", "city_last_timestamp < '{$timestamp}'"), __METHOD__, $condition); $this->info('wikis to remove', ['wikis' => $sth->numRows()]); while ($row = $dbr->fetchObject($sth)) { /** * reasonable defaults for wikis and some presets */ $hide = false; $xdumpok = true; $newFlags = 0; $dbname = $row->city_dbname; $cityid = $row->city_id; $folder = WikiFactory::getVarValueByName("wgUploadDirectory", $cityid); $cluster = WikiFactory::getVarValueByName("wgDBcluster", $cityid); /** * safety check, if city_dbname is not unique die with message */ $check = $dbr->selectRow(array("city_list"), array("count(*) as count"), array("city_dbname" => $dbname), __METHOD__, array("GROUP BY" => "city_dbname")); if ($check->count > 1) { echo "{$dbname} is not unique. Check city_list and rerun script"; die(1); } $this->log("city_id={$row->city_id} city_url={$row->city_url} city_dbname={$dbname} city_flags={$row->city_flags} city_public={$row->city_public}"); /** * request for dump on remote server (now hardcoded for Iowa) */ if ($row->city_flags & WikiFactory::FLAG_HIDE_DB_IMAGES) { $hide = true; } if ($row->city_flags & WikiFactory::FLAG_CREATE_DB_DUMP) { $this->log("Dumping database on remote host"); list($remote) = explode(":", $this->mTarget, 2); $script = $hide ? "--script='../extensions/wikia/WikiFactory/Dumps/runBackups.php --both --id={$cityid} --tmp --s3'" : "--script='../extensions/wikia/WikiFactory/Dumps/runBackups.php --both --id={$cityid} --hide --tmp --s3'"; $cmd = array("/usr/wikia/backend/bin/run_maintenance", "--id=177", $script); $cmd = '/usr/wikia/backend/bin/run_maintenance --id=177 ' . wfEscapeShellArg($script); $this->log($cmd); $output = wfShellExec($cmd, $retval); $xdumpok = empty($retval) ? true : false; /** * reset flag */ $newFlags = $newFlags | WikiFactory::FLAG_CREATE_DB_DUMP | WikiFactory::FLAG_HIDE_DB_IMAGES; } if ($row->city_flags & WikiFactory::FLAG_CREATE_IMAGE_ARCHIVE) { if ($dbname && $folder) { $source = $this->tarFiles($folder, $dbname, $cityid); if ($source) { $retval = DumpsOnDemand::putToAmazonS3($source, !$hide, MimeMagic::singleton()->guessMimeType($source)); if ($retval > 0) { $this->log("putToAmazonS3 command failed."); echo "Can't copy images to remote host. Please, fix that and rerun"; die(1); } else { $this->log("{$source} copied to S3 Amazon"); unlink($source); $newFlags = $newFlags | WikiFactory::FLAG_CREATE_IMAGE_ARCHIVE | WikiFactory::FLAG_HIDE_DB_IMAGES; } } else { /** * actually it's better to die than remove * images later without backup */ echo "Can't copy images to remote host. Source {$source} is not defined"; } } } if ($row->city_flags & WikiFactory::FLAG_DELETE_DB_IMAGES || $row->city_flags & WikiFactory::FLAG_FREE_WIKI_URL) { /** * clear wikifactory tables, condition for city_public should * be always true there but better safe than sorry */ WikiFactory::copyToArchive($row->city_id); $dbw = WikiFactory::db(DB_MASTER); $dbw->delete("city_list", array("city_public" => array(0, -1), "city_id" => $row->city_id), __METHOD__); $this->log("{$row->city_id} removed from WikiFactory tables"); /** * remove records from dataware */ global $wgExternalDatawareDB; $datawareDB = wfGetDB(DB_MASTER, array(), $wgExternalDatawareDB); $datawareDB->delete("pages", array("page_wikia_id" => $row->city_id), __METHOD__); $this->log("{$row->city_id} removed from pages table"); /** * remove images from D.I.R.T. */ $datawareDB->delete("image_review", array("wiki_id" => $row->city_id), __METHOD__); $this->log("{$row->city_id} removed from image_review table"); $datawareDB->delete("image_review_stats", array("wiki_id" => $row->city_id), __METHOD__); $this->log("{$row->city_id} removed from image_review_stats table"); $datawareDB->delete("image_review_wikis", array("wiki_id" => $row->city_id), __METHOD__); $this->log("{$row->city_id} removed from image_review_wikis table"); $datawareDB->commit(); /** * drop database, get db handler for proper cluster */ global $wgDBadminuser, $wgDBadminpassword; $centralDB = empty($cluster) ? "wikicities" : "wikicities_{$cluster}"; /** * get connection but actually we only need info about host */ $local = wfGetDB(DB_MASTER, array(), $centralDB); $server = $local->getLBInfo('host'); try { $dbw = new DatabaseMysql($server, $wgDBadminuser, $wgDBadminpassword, $centralDB); $dbw->begin(); $dbw->query("DROP DATABASE `{$row->city_dbname}`"); $dbw->commit(); $this->log("{$row->city_dbname} dropped from cluster {$cluster}"); } catch (Exception $e) { $this->log("{$row->city_dbname} database drop failed! {$e->getMessage()}"); $this->info('drop database', ['cluster' => $cluster, 'dbname' => $row->city_dbname, 'exception' => $e, 'server' => $server]); } /** * update search index */ $indexer = new Wikia\Search\Indexer(); $indexer->deleteWikiDocs($row->city_id); $this->log("Wiki documents removed from index"); /** * there is nothing to set because row in city_list doesn't * exists */ $newFlags = false; } /** * reset flags, if database was dropped and data were removed from * WikiFactory tables it will return false anyway */ if ($newFlags) { WikiFactory::resetFlags($row->city_id, $newFlags); } $this->info('closed', ['cluster' => $cluster, 'city_id' => (int) $cityid, 'dbname' => $dbname]); /** * just one? */ if ($first) { break; } sleep($sleep); } }
/** * run backup for range of wikis */ function runBackups($from, $to, $full, $options) { global $IP, $wgWikiaLocalSettingsPath, $wgWikiaAdminSettingsPath, $wgMaxShellTime, $wgMaxShellFileSize, $wgDumpsDisabledWikis; $range = array(); /** * shortcut for full & current together */ $both = isset($options["both"]); /** * store backup in another folder, not available for users */ $hide = isset($options["hide"]); /** * store backup in the system tmp dir */ $use_temp = isset($options['tmp']); /** * send backup to Amazon S3 and delete the local copy */ $s3 = isset($options['s3']); /** * silly trick, if we have id defined we are defining $from & $to from it * if we have db param defined we first resolve which id is connected to this * database */ if (isset($options["db"]) && is_string($options["db"])) { $city_id = WikiFactory::DBtoID($options["db"]); if ($city_id) { $from = $to = $city_id; $to++; } } elseif (isset($options["id"]) && is_numeric($options["id"])) { $from = $to = $options["id"]; $to++; } elseif (isset($options["even"])) { $range[] = "city_id % 2 = 0"; $range[] = "city_public = 1"; } elseif (isset($options["odd"])) { $range[] = "city_id % 2 <> 0"; $range[] = "city_public = 1"; } else { /** * if all only for active */ $range[] = "city_public = 1"; } /** * exclude wikis with dumps disabled */ if (!empty($wgDumpsDisabledWikis) && is_array($wgDumpsDisabledWikis)) { $range[] = 'city_id NOT IN (' . implode(',', $wgDumpsDisabledWikis) . ')'; } /** * switch off limits for dumps */ $wgMaxShellTime = 0; $wgMaxShellFileSize = 0; if ($from !== false && $to !== false) { $range[] = sprintf("city_id >= %d AND city_id < %d", $from, $to); Wikia::log(__METHOD__, "info", "Running from {$from} to {$to}", true, true); } else { Wikia::log(__METHOD__, "info", "Running for all wikis", true, true); } $dbw = Wikifactory::db(DB_MASTER); $sth = $dbw->select(array("city_list"), array("city_id", "city_dbname"), $range, __METHOD__, array("ORDER BY" => "city_id")); while ($row = $dbw->fetchObject($sth)) { /** * get cluster for this wiki */ $cluster = WikiFactory::getVarValueByName("wgDBcluster", $row->city_id); $server = wfGetDB(DB_SLAVE, 'dumps', $row->city_dbname)->getProperty("mServer"); /** * build command */ $status = false; $basedir = getDirectory($row->city_dbname, $hide, $use_temp); if ($full || $both) { $path = sprintf("%s/%s_pages_full.xml.7z", $basedir, $row->city_dbname); $time = wfTime(); Wikia::log(__METHOD__, "info", "{$row->city_id} {$row->city_dbname} {$path}", true, true); $cmd = array("SERVER_ID={$row->city_id}", "php", "{$IP}/maintenance/dumpBackup.php", "--conf {$wgWikiaLocalSettingsPath}", "--aconf {$wgWikiaAdminSettingsPath}", "--full", "--xml", "--quiet", "--server={$server}", "--output=" . DumpsOnDemand::DEFAULT_COMPRESSION_FORMAT . ":{$path}"); wfShellExec(implode(" ", $cmd), $status); $time = Wikia::timeDuration(wfTime() - $time); Wikia::log(__METHOD__, "info", "{$row->city_id} {$row->city_dbname} status: {$status}, time: {$time}", true, true); if ($s3 && 0 == DumpsOnDemand::putToAmazonS3($path, !$hide, MimeMagic::singleton()->guessMimeType($path))) { unlink($path); } } if (!$full || $both) { $path = sprintf("%s/%s_pages_current.xml.7z", $basedir, $row->city_dbname); $time = wfTime(); Wikia::log(__METHOD__, "info", "{$row->city_id} {$row->city_dbname} {$path}", true, true); $cmd = array("SERVER_ID={$row->city_id}", "php", "{$IP}/maintenance/dumpBackup.php", "--conf {$wgWikiaLocalSettingsPath}", "--aconf {$wgWikiaAdminSettingsPath}", "--current", "--xml", "--quiet", "--server={$server}", "--output=" . DumpsOnDemand::DEFAULT_COMPRESSION_FORMAT . ":{$path}"); wfShellExec(implode(" ", $cmd), $status); $time = Wikia::timeDuration(wfTime() - $time); Wikia::log(__METHOD__, "info", "{$row->city_id} {$row->city_dbname} status: {$status}, time: {$time}", true, true); if ($s3 && 0 == DumpsOnDemand::putToAmazonS3($path, !$hide, MimeMagic::singleton()->guessMimeType($path))) { unlink($path); } } } }