/**
* 1. go through all wikis which are marked for closing and check which one
* want to have images packed.
*
* 2. pack images, send them via rsync to target server,
*
* 3. mark in city_list.city_flags that images are sent,
*
* 4. remove images
*
* @access public
*/
public function execute()
{
global $wgUploadDirectory, $wgDBname, $IP;
$first = isset($this->mOptions["first"]) ? true : false;
$sleep = isset($this->mOptions["sleep"]) ? $this->mOptions["sleep"] : 15;
$condition = array("ORDER BY" => "city_id");
$this->info('start', ['first' => $first, 'limit' => $this->mOptions["limit"] ?: false]);
/**
* if $first is set skip limit checking
*/
if (!$first) {
if (isset($this->mOptions["limit"]) && is_numeric($this->mOptions["limit"])) {
$condition["LIMIT"] = $this->mOptions["limit"];
}
}
$timestamp = wfTimestamp(TS_DB, strtotime(sprintf("-%d days", self::CLOSE_WIKI_DELAY)));
$dbr = WikiFactory::db(DB_SLAVE);
$sth = $dbr->select(array("city_list"), array("city_id", "city_flags", "city_dbname", "city_url", "city_public"), array("city_public" => array(0, -1), "city_flags <> 0 && city_flags <> 32", "city_last_timestamp < '{$timestamp}'"), __METHOD__, $condition);
$this->info('wikis to remove', ['wikis' => $sth->numRows()]);
while ($row = $dbr->fetchObject($sth)) {
/**
* reasonable defaults for wikis and some presets
*/
$hide = false;
$xdumpok = true;
$newFlags = 0;
$dbname = $row->city_dbname;
$cityid = $row->city_id;
$folder = WikiFactory::getVarValueByName("wgUploadDirectory", $cityid);
$cluster = WikiFactory::getVarValueByName("wgDBcluster", $cityid);
/**
* safety check, if city_dbname is not unique die with message
*/
$check = $dbr->selectRow(array("city_list"), array("count(*) as count"), array("city_dbname" => $dbname), __METHOD__, array("GROUP BY" => "city_dbname"));
if ($check->count > 1) {
echo "{$dbname} is not unique. Check city_list and rerun script";
die(1);
}
$this->log("city_id={$row->city_id} city_url={$row->city_url} city_dbname={$dbname} city_flags={$row->city_flags} city_public={$row->city_public}");
/**
* request for dump on remote server (now hardcoded for Iowa)
*/
if ($row->city_flags & WikiFactory::FLAG_HIDE_DB_IMAGES) {
$hide = true;
}
if ($row->city_flags & WikiFactory::FLAG_CREATE_DB_DUMP) {
$this->log("Dumping database on remote host");
list($remote) = explode(":", $this->mTarget, 2);
$script = $hide ? "--script='../extensions/wikia/WikiFactory/Dumps/runBackups.php --both --id={$cityid} --tmp --s3'" : "--script='../extensions/wikia/WikiFactory/Dumps/runBackups.php --both --id={$cityid} --hide --tmp --s3'";
$cmd = array("/usr/wikia/backend/bin/run_maintenance", "--id=177", $script);
$cmd = '/usr/wikia/backend/bin/run_maintenance --id=177 ' . wfEscapeShellArg($script);
$this->log($cmd);
$output = wfShellExec($cmd, $retval);
$xdumpok = empty($retval) ? true : false;
/**
* reset flag
*/
$newFlags = $newFlags | WikiFactory::FLAG_CREATE_DB_DUMP | WikiFactory::FLAG_HIDE_DB_IMAGES;
}
if ($row->city_flags & WikiFactory::FLAG_CREATE_IMAGE_ARCHIVE) {
if ($dbname && $folder) {
$source = $this->tarFiles($folder, $dbname, $cityid);
if ($source) {
$retval = DumpsOnDemand::putToAmazonS3($source, !$hide, MimeMagic::singleton()->guessMimeType($source));
if ($retval > 0) {
$this->log("putToAmazonS3 command failed.");
echo "Can't copy images to remote host. Please, fix that and rerun";
die(1);
} else {
$this->log("{$source} copied to S3 Amazon");
unlink($source);
$newFlags = $newFlags | WikiFactory::FLAG_CREATE_IMAGE_ARCHIVE | WikiFactory::FLAG_HIDE_DB_IMAGES;
}
} else {
/**
* actually it's better to die than remove
* images later without backup
*/
echo "Can't copy images to remote host. Source {$source} is not defined";
}
}
}
if ($row->city_flags & WikiFactory::FLAG_DELETE_DB_IMAGES || $row->city_flags & WikiFactory::FLAG_FREE_WIKI_URL) {
/**
* clear wikifactory tables, condition for city_public should
* be always true there but better safe than sorry
*/
WikiFactory::copyToArchive($row->city_id);
$dbw = WikiFactory::db(DB_MASTER);
$dbw->delete("city_list", array("city_public" => array(0, -1), "city_id" => $row->city_id), __METHOD__);
$this->log("{$row->city_id} removed from WikiFactory tables");
/**
* remove records from dataware
*/
global $wgExternalDatawareDB;
$datawareDB = wfGetDB(DB_MASTER, array(), $wgExternalDatawareDB);
$datawareDB->delete("pages", array("page_wikia_id" => $row->city_id), __METHOD__);
$this->log("{$row->city_id} removed from pages table");
/**
* remove images from D.I.R.T.
*/
$datawareDB->delete("image_review", array("wiki_id" => $row->city_id), __METHOD__);
$this->log("{$row->city_id} removed from image_review table");
$datawareDB->delete("image_review_stats", array("wiki_id" => $row->city_id), __METHOD__);
$this->log("{$row->city_id} removed from image_review_stats table");
$datawareDB->delete("image_review_wikis", array("wiki_id" => $row->city_id), __METHOD__);
$this->log("{$row->city_id} removed from image_review_wikis table");
$datawareDB->commit();
/**
* drop database, get db handler for proper cluster
*/
global $wgDBadminuser, $wgDBadminpassword;
$centralDB = empty($cluster) ? "wikicities" : "wikicities_{$cluster}";
/**
* get connection but actually we only need info about host
*/
$local = wfGetDB(DB_MASTER, array(), $centralDB);
$server = $local->getLBInfo('host');
try {
$dbw = new DatabaseMysql($server, $wgDBadminuser, $wgDBadminpassword, $centralDB);
$dbw->begin();
$dbw->query("DROP DATABASE `{$row->city_dbname}`");
$dbw->commit();
$this->log("{$row->city_dbname} dropped from cluster {$cluster}");
} catch (Exception $e) {
$this->log("{$row->city_dbname} database drop failed! {$e->getMessage()}");
$this->info('drop database', ['cluster' => $cluster, 'dbname' => $row->city_dbname, 'exception' => $e, 'server' => $server]);
}
/**
* update search index
*/
$indexer = new Wikia\Search\Indexer();
$indexer->deleteWikiDocs($row->city_id);
$this->log("Wiki documents removed from index");
/**
* there is nothing to set because row in city_list doesn't
* exists
*/
$newFlags = false;
}
/**
* reset flags, if database was dropped and data were removed from
* WikiFactory tables it will return false anyway
*/
if ($newFlags) {
WikiFactory::resetFlags($row->city_id, $newFlags);
}
$this->info('closed', ['cluster' => $cluster, 'city_id' => (int) $cityid, 'dbname' => $dbname]);
/**
* just one?
*/
if ($first) {
break;
}
sleep($sleep);
}
}
/**
* run backup for range of wikis
*/
function runBackups($from, $to, $full, $options)
{
global $IP, $wgWikiaLocalSettingsPath, $wgWikiaAdminSettingsPath, $wgMaxShellTime, $wgMaxShellFileSize, $wgDumpsDisabledWikis;
$range = array();
/**
* shortcut for full & current together
*/
$both = isset($options["both"]);
/**
* store backup in another folder, not available for users
*/
$hide = isset($options["hide"]);
/**
* store backup in the system tmp dir
*/
$use_temp = isset($options['tmp']);
/**
* send backup to Amazon S3 and delete the local copy
*/
$s3 = isset($options['s3']);
/**
* silly trick, if we have id defined we are defining $from & $to from it
* if we have db param defined we first resolve which id is connected to this
* database
*/
if (isset($options["db"]) && is_string($options["db"])) {
$city_id = WikiFactory::DBtoID($options["db"]);
if ($city_id) {
$from = $to = $city_id;
$to++;
}
} elseif (isset($options["id"]) && is_numeric($options["id"])) {
$from = $to = $options["id"];
$to++;
} elseif (isset($options["even"])) {
$range[] = "city_id % 2 = 0";
$range[] = "city_public = 1";
} elseif (isset($options["odd"])) {
$range[] = "city_id % 2 <> 0";
$range[] = "city_public = 1";
} else {
/**
* if all only for active
*/
$range[] = "city_public = 1";
}
/**
* exclude wikis with dumps disabled
*/
if (!empty($wgDumpsDisabledWikis) && is_array($wgDumpsDisabledWikis)) {
$range[] = 'city_id NOT IN (' . implode(',', $wgDumpsDisabledWikis) . ')';
}
/**
* switch off limits for dumps
*/
$wgMaxShellTime = 0;
$wgMaxShellFileSize = 0;
if ($from !== false && $to !== false) {
$range[] = sprintf("city_id >= %d AND city_id < %d", $from, $to);
Wikia::log(__METHOD__, "info", "Running from {$from} to {$to}", true, true);
} else {
Wikia::log(__METHOD__, "info", "Running for all wikis", true, true);
}
$dbw = Wikifactory::db(DB_MASTER);
$sth = $dbw->select(array("city_list"), array("city_id", "city_dbname"), $range, __METHOD__, array("ORDER BY" => "city_id"));
while ($row = $dbw->fetchObject($sth)) {
/**
* get cluster for this wiki
*/
$cluster = WikiFactory::getVarValueByName("wgDBcluster", $row->city_id);
$server = wfGetDB(DB_SLAVE, 'dumps', $row->city_dbname)->getProperty("mServer");
/**
* build command
*/
$status = false;
$basedir = getDirectory($row->city_dbname, $hide, $use_temp);
if ($full || $both) {
$path = sprintf("%s/%s_pages_full.xml.7z", $basedir, $row->city_dbname);
$time = wfTime();
Wikia::log(__METHOD__, "info", "{$row->city_id} {$row->city_dbname} {$path}", true, true);
$cmd = array("SERVER_ID={$row->city_id}", "php", "{$IP}/maintenance/dumpBackup.php", "--conf {$wgWikiaLocalSettingsPath}", "--aconf {$wgWikiaAdminSettingsPath}", "--full", "--xml", "--quiet", "--server={$server}", "--output=" . DumpsOnDemand::DEFAULT_COMPRESSION_FORMAT . ":{$path}");
wfShellExec(implode(" ", $cmd), $status);
$time = Wikia::timeDuration(wfTime() - $time);
Wikia::log(__METHOD__, "info", "{$row->city_id} {$row->city_dbname} status: {$status}, time: {$time}", true, true);
if ($s3 && 0 == DumpsOnDemand::putToAmazonS3($path, !$hide, MimeMagic::singleton()->guessMimeType($path))) {
unlink($path);
}
}
if (!$full || $both) {
$path = sprintf("%s/%s_pages_current.xml.7z", $basedir, $row->city_dbname);
$time = wfTime();
Wikia::log(__METHOD__, "info", "{$row->city_id} {$row->city_dbname} {$path}", true, true);
$cmd = array("SERVER_ID={$row->city_id}", "php", "{$IP}/maintenance/dumpBackup.php", "--conf {$wgWikiaLocalSettingsPath}", "--aconf {$wgWikiaAdminSettingsPath}", "--current", "--xml", "--quiet", "--server={$server}", "--output=" . DumpsOnDemand::DEFAULT_COMPRESSION_FORMAT . ":{$path}");
wfShellExec(implode(" ", $cmd), $status);
$time = Wikia::timeDuration(wfTime() - $time);
Wikia::log(__METHOD__, "info", "{$row->city_id} {$row->city_dbname} status: {$status}, time: {$time}", true, true);
if ($s3 && 0 == DumpsOnDemand::putToAmazonS3($path, !$hide, MimeMagic::singleton()->guessMimeType($path))) {
unlink($path);
}
}
}
}
