/**
* 1. go through all wikis which are marked for closing and check which one
* want to have images packed.
*
* 2. pack images, send them via rsync to target server,
*
* 3. mark in city_list.city_flags that images are sent,
*
* 4. remove images
*
* @access public
*/
public function execute()
{
global $wgUploadDirectory, $wgDBname, $IP;
$first = isset($this->mOptions["first"]) ? true : false;
$sleep = isset($this->mOptions["sleep"]) ? $this->mOptions["sleep"] : 15;
$condition = array("ORDER BY" => "city_id");
$this->info('start', ['first' => $first, 'limit' => $this->mOptions["limit"] ?: false]);
/**
* if $first is set skip limit checking
*/
if (!$first) {
if (isset($this->mOptions["limit"]) && is_numeric($this->mOptions["limit"])) {
$condition["LIMIT"] = $this->mOptions["limit"];
}
}
$timestamp = wfTimestamp(TS_DB, strtotime(sprintf("-%d days", self::CLOSE_WIKI_DELAY)));
$dbr = WikiFactory::db(DB_SLAVE);
$sth = $dbr->select(array("city_list"), array("city_id", "city_flags", "city_dbname", "city_url", "city_public"), array("city_public" => array(0, -1), "city_flags <> 0 && city_flags <> 32", "city_last_timestamp < '{$timestamp}'"), __METHOD__, $condition);
$this->info('wikis to remove', ['wikis' => $sth->numRows()]);
while ($row = $dbr->fetchObject($sth)) {
/**
* reasonable defaults for wikis and some presets
*/
$hide = false;
$xdumpok = true;
$newFlags = 0;
$dbname = $row->city_dbname;
$cityid = $row->city_id;
$folder = WikiFactory::getVarValueByName("wgUploadDirectory", $cityid);
$cluster = WikiFactory::getVarValueByName("wgDBcluster", $cityid);
/**
* safety check, if city_dbname is not unique die with message
*/
$check = $dbr->selectRow(array("city_list"), array("count(*) as count"), array("city_dbname" => $dbname), __METHOD__, array("GROUP BY" => "city_dbname"));
if ($check->count > 1) {
echo "{$dbname} is not unique. Check city_list and rerun script";
die(1);
}
$this->log("city_id={$row->city_id} city_url={$row->city_url} city_dbname={$dbname} city_flags={$row->city_flags} city_public={$row->city_public}");
/**
* request for dump on remote server (now hardcoded for Iowa)
*/
if ($row->city_flags & WikiFactory::FLAG_HIDE_DB_IMAGES) {
$hide = true;
}
if ($row->city_flags & WikiFactory::FLAG_CREATE_DB_DUMP) {
$this->log("Dumping database on remote host");
list($remote) = explode(":", $this->mTarget, 2);
$script = $hide ? "--script='../extensions/wikia/WikiFactory/Dumps/runBackups.php --both --id={$cityid} --tmp --s3'" : "--script='../extensions/wikia/WikiFactory/Dumps/runBackups.php --both --id={$cityid} --hide --tmp --s3'";
$cmd = array("/usr/wikia/backend/bin/run_maintenance", "--id=177", $script);
$cmd = '/usr/wikia/backend/bin/run_maintenance --id=177 ' . wfEscapeShellArg($script);
$this->log($cmd);
$output = wfShellExec($cmd, $retval);
$xdumpok = empty($retval) ? true : false;
/**
* reset flag
*/
$newFlags = $newFlags | WikiFactory::FLAG_CREATE_DB_DUMP | WikiFactory::FLAG_HIDE_DB_IMAGES;
}
if ($row->city_flags & WikiFactory::FLAG_CREATE_IMAGE_ARCHIVE) {
if ($dbname && $folder) {
$source = $this->tarFiles($folder, $dbname, $cityid);
if ($source) {
$retval = DumpsOnDemand::putToAmazonS3($source, !$hide, MimeMagic::singleton()->guessMimeType($source));
if ($retval > 0) {
$this->log("putToAmazonS3 command failed.");
echo "Can't copy images to remote host. Please, fix that and rerun";
die(1);
} else {
$this->log("{$source} copied to S3 Amazon");
unlink($source);
$newFlags = $newFlags | WikiFactory::FLAG_CREATE_IMAGE_ARCHIVE | WikiFactory::FLAG_HIDE_DB_IMAGES;
}
} else {
/**
* actually it's better to die than remove
* images later without backup
*/
echo "Can't copy images to remote host. Source {$source} is not defined";
}
}
}
if ($row->city_flags & WikiFactory::FLAG_DELETE_DB_IMAGES || $row->city_flags & WikiFactory::FLAG_FREE_WIKI_URL) {
/**
* clear wikifactory tables, condition for city_public should
* be always true there but better safe than sorry
*/
WikiFactory::copyToArchive($row->city_id);
$dbw = WikiFactory::db(DB_MASTER);
$dbw->delete("city_list", array("city_public" => array(0, -1), "city_id" => $row->city_id), __METHOD__);
$this->log("{$row->city_id} removed from WikiFactory tables");
/**
* remove records from dataware
*/
global $wgExternalDatawareDB;
$datawareDB = wfGetDB(DB_MASTER, array(), $wgExternalDatawareDB);
$datawareDB->delete("pages", array("page_wikia_id" => $row->city_id), __METHOD__);
$this->log("{$row->city_id} removed from pages table");
/**
* remove images from D.I.R.T.
*/
$datawareDB->delete("image_review", array("wiki_id" => $row->city_id), __METHOD__);
$this->log("{$row->city_id} removed from image_review table");
$datawareDB->delete("image_review_stats", array("wiki_id" => $row->city_id), __METHOD__);
$this->log("{$row->city_id} removed from image_review_stats table");
$datawareDB->delete("image_review_wikis", array("wiki_id" => $row->city_id), __METHOD__);
$this->log("{$row->city_id} removed from image_review_wikis table");
$datawareDB->commit();
/**
* drop database, get db handler for proper cluster
*/
global $wgDBadminuser, $wgDBadminpassword;
$centralDB = empty($cluster) ? "wikicities" : "wikicities_{$cluster}";
/**
* get connection but actually we only need info about host
*/
$local = wfGetDB(DB_MASTER, array(), $centralDB);
$server = $local->getLBInfo('host');
try {
$dbw = new DatabaseMysql($server, $wgDBadminuser, $wgDBadminpassword, $centralDB);
$dbw->begin();
$dbw->query("DROP DATABASE `{$row->city_dbname}`");
$dbw->commit();
$this->log("{$row->city_dbname} dropped from cluster {$cluster}");
} catch (Exception $e) {
$this->log("{$row->city_dbname} database drop failed! {$e->getMessage()}");
$this->info('drop database', ['cluster' => $cluster, 'dbname' => $row->city_dbname, 'exception' => $e, 'server' => $server]);
}
/**
* update search index
*/
$indexer = new Wikia\Search\Indexer();
$indexer->deleteWikiDocs($row->city_id);
$this->log("Wiki documents removed from index");
/**
* there is nothing to set because row in city_list doesn't
* exists
*/
$newFlags = false;
}
/**
* reset flags, if database was dropped and data were removed from
* WikiFactory tables it will return false anyway
*/
if ($newFlags) {
WikiFactory::resetFlags($row->city_id, $newFlags);
}
$this->info('closed', ['cluster' => $cluster, 'city_id' => (int) $cityid, 'dbname' => $dbname]);
/**
* just one?
*/
if ($first) {
break;
}
sleep($sleep);
}
}
<?php
/**
* Responsible for reindexing on staff and internal, using Indexer class.
* @package MediaWiki
* @addtopackage maintenance
*/
require_once "../../../../maintenance/commandLine.inc";
include "{$IP}/extensions/wikia/Search/WikiaSearch.setup.php";
global $wgContentNamespaces, $wgExtraNamespaces;
$wgUser = User::newFromName('Owen Davis');
$wgTitle = Title::newMainPage();
$c = RequestContext::getMain();
$c->setUser($wgUser);
$c->setTitle($wgTitle);
$indexer = new Wikia\Search\Indexer();
$dbr = wfGetDB(DB_MASTER);
$namespaces = !empty($wgExtraNamespaces) ? array_merge($wgContentNamespaces, $wgExtraNamespaces) : $wgContentNamespaces;
$select = $dbr->select('page', 'page_id', array('page_namespace' => $namespaces));
foreach ($select as $row) {
if ($row->page_id) {
$ids[] = $row->page_id;
}
}
$idCount = count($ids);
$sliceCount = 0;
foreach (array_chunk($ids, 10) as $idSlice) {
$sliceCount += 10;
$indexer->reindexBatch($idSlice);
echo "Reindexed {$sliceCount}/{$idCount} docs\n";
}
/**
* Deprecated functionality for indexing.
*/
public function getPages()
{
$this->wg->AllowMemcacheWrites = false;
$indexer = new Wikia\Search\Indexer();
$this->getResponse()->setData($indexer->getPages(explode('|', $this->getVal('ids'))));
$this->getResponse()->setFormat('json');
}
