/**
* run backup for range of wikis
*/
function runBackups($from, $to, $full, $options)
{
global $IP, $wgWikiaLocalSettingsPath, $wgWikiaAdminSettingsPath, $wgMaxShellTime, $wgMaxShellFileSize, $wgDumpsDisabledWikis;
$range = array();
/**
* shortcut for full & current together
*/
$both = isset($options["both"]) ? true : false;
/**
* store backup in another folder, not available for users
*/
$hide = isset($options["hide"]) ? true : false;
/**
* silly trick, if we have id defined we are defining $from & $to from it
* if we have db param defined we first resolve which id is connected to this
* database
*/
if (isset($options["db"]) && is_string($options["db"])) {
$city_id = WikiFactory::DBtoID($options["db"]);
if ($city_id) {
$from = $to = $city_id;
$to++;
}
} elseif (isset($options["id"]) && is_numeric($options["id"])) {
$from = $to = $options["id"];
$to++;
} elseif (isset($options["even"])) {
$range[] = "city_id % 2 = 0";
$range[] = "city_public = 1";
} elseif (isset($options["odd"])) {
$range[] = "city_id % 2 <> 0";
$range[] = "city_public = 1";
} else {
/**
* if all only for active
*/
$range[] = "city_public = 1";
}
/**
* exclude wikis with dumps disabled
*/
if (!empty($wgDumpsDisabledWikis) && is_array($wgDumpsDisabledWikis)) {
$range[] = 'city_id NOT IN (' . implode(',', $wgDumpsDisabledWikis) . ')';
}
/**
* switch off limits for dumps
*/
$wgMaxShellTime = 0;
$wgMaxShellFileSize = 0;
if ($from !== false && $to !== false) {
$range[] = sprintf("city_id >= %d AND city_id < %d", $from, $to);
Wikia::log(__METHOD__, "info", "Running from {$from} to {$to}", true, true);
} else {
Wikia::log(__METHOD__, "info", "Running for all wikis", true, true);
}
$dbw = Wikifactory::db(DB_MASTER);
$sth = $dbw->select(array("city_list"), array("city_id", "city_dbname"), $range, __METHOD__, array("ORDER BY" => "city_id"));
while ($row = $dbw->fetchObject($sth)) {
/**
* get cluster for this wiki
*/
$cluster = WikiFactory::getVarValueByName("wgDBcluster", $row->city_id);
$server = wfGetDB(DB_SLAVE, 'dumps', $row->city_dbname)->getProperty("mServer");
/**
* build command
*/
$status = false;
$basedir = getDirectory($row->city_dbname, $hide);
if ($full || $both) {
$path = sprintf("%s/pages_full.xml.gz", $basedir);
$time = wfTime();
Wikia::log(__METHOD__, "info", "{$row->city_id} {$row->city_dbname} {$path}", true, true);
$cmd = array("SERVER_ID={$row->city_id}", "php", "{$IP}/maintenance/dumpBackup.php", "--conf {$wgWikiaLocalSettingsPath}", "--aconf {$wgWikiaAdminSettingsPath}", "--full", "--xml", "--quiet", "--server={$server}", "--output=gzip:{$path}");
wfShellExec(implode(" ", $cmd), $status);
$time = Wikia::timeDuration(wfTime() - $time);
Wikia::log(__METHOD__, "info", "{$row->city_id} {$row->city_dbname} status: {$status}, time: {$time}", true, true);
}
if (!$full || $both) {
$path = sprintf("%s/pages_current.xml.gz", $basedir);
$time = wfTime();
Wikia::log(__METHOD__, "info", "{$row->city_id} {$row->city_dbname} {$path}", true, true);
$cmd = array("SERVER_ID={$row->city_id}", "php", "{$IP}/maintenance/dumpBackup.php", "--conf {$wgWikiaLocalSettingsPath}", "--aconf {$wgWikiaAdminSettingsPath}", "--current", "--xml", "--quiet", "--server={$server}", "--output=gzip:{$path}");
wfShellExec(implode(" ", $cmd), $status);
$time = Wikia::timeDuration(wfTime() - $time);
Wikia::log(__METHOD__, "info", "{$row->city_id} {$row->city_dbname} status: {$status}, time: {$time}", true, true);
}
/**
* generate index.json
*/
$jsonfile = sprintf("%s/index.json", $basedir);
$json = array();
/**
* open dir and read info about files
*/
if (is_dir($basedir)) {
$dh = opendir($basedir);
while (($file = readdir($dh)) !== false) {
$fullpath = $basedir . "/" . $file;
if (is_file($fullpath)) {
$json[$file] = array("name" => $file, "timestamp" => filectime($fullpath), "mwtimestamp" => wfTimestamp(TS_MW, filectime($fullpath)));
}
}
closedir($dh);
}
if (count($json)) {
file_put_contents($jsonfile, json_encode($json));
}
}
}
private function getFilters(&$query, &$filter_roster)
{
global $wgRequest, $wgUser;
$filter = array();
$created_type = $wgRequest->getVal('new_created_type', $wgRequest->getVal('created_type', 'exactly'));
$created_num = $wgRequest->getVal('new_created_number', $wgRequest->getVal('created_number', null));
$created_unit = $wgRequest->getVal('new_created_unit', $wgRequest->getVal('created_unit', 'day'));
if ($created_num) {
$filter[] = "DATE(created) " . ($created_type == 'exactly' ? '=' : '>=') . " DATE(DATE_SUB(NOW(), INTERVAL {$created_num} {$created_unit}))";
$query[] = "created_type={$created_type}";
$query[] = "created_number={$created_num}";
$query[] = "created_unit={$created_unit}";
if ($created_type == 'exactly') {
$filter_roster['CreatedTypeExact'] = 1;
}
if ($created_type == 'after') {
$filter_roster['CreatedTypeAfter'] = 1;
}
if ($created_unit == 'day') {
$filter_roster['CreatedUnitDays'] = 1;
}
if ($created_unit == 'week') {
$filter_roster['CreatedUnitWeeks'] = 1;
}
$filter_roster['CreatedNum'] = $created_num;
}
$attempted_type = $wgRequest->getVal('new_attempted_type', $wgRequest->getVal('attempted_type', 'exactly'));
$attempted_num = $wgRequest->getVal('new_attempted_number', $wgRequest->getVal('attempted_number', null));
$attempted_unit = $wgRequest->getVal('new_attempted_unit', $wgRequest->getVal('attempted_unit', 'day'));
if ($attempted_num) {
$filter[] = "DATE(attempted) " . ($attempted_type == 'exactly' ? '=' : '>=') . " DATE(DATE_SUB(NOW(), INTERVAL {$attempted_num} {$attempted_unit}))";
$query[] = "attempted_type={$attempted_type}";
$query[] = "attempted_number={$attempted_num}";
$query[] = "attempted_unit={$attempted_unit}";
if ($attempted_type == 'exactly') {
$filter_roster['AttemptedTypeExact'] = 1;
}
if ($attempted_type == 'after') {
$filter_roster['AttemptedTypeAfter'] = 1;
}
if ($attempted_unit == 'day') {
$filter_roster['AttemptedUnitDays'] = 1;
}
if ($attempted_unit == 'week') {
$filter_roster['AttemptedUnitWeeks'] = 1;
}
$filter_roster['attemptedNum'] = $attempted_num;
}
$transmitted_type = $wgRequest->getVal('new_transmitted_type', $wgRequest->getVal('transmitted_type', 'exactly'));
$transmitted_num = $wgRequest->getVal('new_transmitted_number', $wgRequest->getVal('transmitted_number', null));
$transmitted_unit = $wgRequest->getVal('new_transmitted_unit', $wgRequest->getVal('transmitted_unit', 'day'));
if ($transmitted_num) {
$filter[] = "DATE(transmitted) " . ($transmitted_type == 'exactly' ? '=' : '>=') . " DATE(DATE_SUB(NOW(), INTERVAL {$transmitted_num} {$transmitted_unit}))";
$query[] = "transmitted_type={$transmitted_type}";
$query[] = "transmitted_number={$transmitted_num}";
$query[] = "transmitted_unit={$transmitted_unit}";
if ($transmitted_type == 'exactly') {
$filter_roster['TransmittedTypeExact'] = 1;
}
if ($transmitted_type == 'after') {
$filter_roster['TransmittedTypeAfter'] = 1;
}
if ($transmitted_unit == 'day') {
$filter_roster['TransmittedUnitDays'] = 1;
}
if ($transmitted_unit == 'week') {
$filter_roster['TransmittedUnitWeeks'] = 1;
}
$filter_roster['TransmittedNum'] = $transmitted_num;
}
$filter_wiki_name = $wgRequest->getVal('new_filter_wiki_name', null);
if ($filter_wiki_name) {
$filter_wiki_name = preg_replace('!^http://|/$!', '', $filter_wiki_name);
$filter_wiki_name = preg_replace('!\\.wikia\\.com!', '', $filter_wiki_name);
$wgRequest->setVal('new_filter_wiki_id', Wikifactory::DBtoId($filter_wiki_name));
}
$filter_wiki_id = $wgRequest->getVal('new_filter_wiki_id', $wgRequest->getVal('filter_wiki_id', null));
$filter_dst = $wgRequest->getVal('new_filter_dst', $wgRequest->getVal('filter_dst', null));
// Force a few values depending on who the viewer is
if (!$wgUser->isAllowed("mailerlog")) {
$filter_dst = $wgUser->getEmail();
$forced_dst = true;
}
if ($wgRequest->getVal('off_filter_wiki_id', null) && !isset($forced_wiki_id)) {
$filter_wiki_id = null;
}
if ($filter_wiki_id) {
$filter[] = "city_id = {$filter_wiki_id}";
if (!isset($forced_wiki_id)) {
$query[] = "filter_wiki_id={$filter_wiki_id}";
$filter_roster['Wiki'] = array('value' => Wikifactory::IdtoDB($filter_wiki_id), 'off' => 'off_filter_wiki_id');
}
}
if ($wgRequest->getVal('off_filter_dst', null) && !isset($forced_dst)) {
$filter_dst = null;
}
if ($filter_dst) {
$filter[] = "dst = '{$filter_dst}'";
if (!isset($forced_dst)) {
$query[] = "filter_dst={$filter_dst}";
$filter_roster['Email'] = array('value' => $filter_dst, 'off' => 'off_filter_dst');
}
}
$filter_subject = $wgRequest->getVal('new_filter_subject', $wgRequest->getVal('filter_subject', null));
if ($filter_subject) {
$filter[] = "subj LIKE '%{$filter_subject}%'";
$query[] = "filter_subject={$filter_subject}";
$filter_roster['Subject'] = $filter_subject;
}
$filter_body = $wgRequest->getVal('new_filter_body', $wgRequest->getVal('filter_body', null));
if ($filter_body) {
$filter[] = "msg LIKE '%{$filter_body}%'";
$query[] = "filter_body={$filter_body}";
$filter_roster['Body'] = $filter_body;
}
$filter_errors = $wgRequest->getVal('new_filter_errors', $wgRequest->getVal('filter_errors', null));
if ($wgRequest->getVal('off_filter_errors', null)) {
$filter_errors = null;
}
if ($filter_errors) {
$filter[] = 'error_msg IS NOT NULL';
$query[] = "filter_errors={$filter_errors}";
$filter_roster['Errors'] = 'Show';
}
return $filter;
}
<?php
/**
* migrateUploadPath -- migrate images to new directory layout
*
* @addto maintenance
* @author Krzysztof Krzyżaniak (eloy)
*
*/
ini_set("include_path", dirname(__FILE__) . "/..");
require_once 'commandLine.inc';
/**
* connect to WF db
*/
$variable = "wgUploadDirectory";
$dbw = Wikifactory::db(DB_MASTER);
$sth = $dbw->select(array("city_variables"), array("*"), array("cv_variable_id = (SELECT cv_id FROM city_variables_pool WHERE cv_name = '{$variable}')"), __METHOD__, array("FOR UPDATE"));
while ($row = $dbw->fetchObject($sth)) {
$value = ltrim(unserialize($row->cv_value), "/");
#print "{$value}\n";
$parts = explode("/", $value);
/**
* first part is not interesting
*/
$prefix = array_shift($parts);
if (strlen($parts[0]) > 1) {
/*
* it's not converted yet
*/
$letter = strtolower(substr($parts[0], 0, 1));
$path = sprintf("/%s/%s/%s", $prefix, $letter, implode("/", $parts));
/**
* inherited "insert" function add job to current database, for this job
* we need to add job to newly created wiki
*
* @param integer $city_id wiki identifier in city_list table
* @param string $database target database name
*/
public function WFinsert($city_id, $database = false)
{
global $wgDBname, $wgErrorLog;
/**
* we can take local database from city_id in params array
*/
$oldValue = $wgErrorLog;
$wgErrorLog = true;
Wikia::log(__METHOD__, "id", $city_id);
if (!$database) {
$database = Wikifactory::IdtoDB($city_id);
}
Wikia::log(__METHOD__, "db", $database);
if ($database) {
$fields = $this->insertFields();
$dbw_local = wfGetDB(DB_MASTER, array(), $database);
if ($this->removeDuplicates) {
$res = $dbw_local->select('job', array('1'), $fields, __METHOD__);
if ($dbw_local->numRows($res)) {
return;
}
}
$fields['job_id'] = $dbw_local->nextSequenceValue('job_job_id_seq');
$dbw_local->insert('job', $fields, __METHOD__);
/**
* we need to commit before switching databases
*/
$dbw_local->commit();
}
$wgErrorLog = $oldValue;
}
/**
* run backup for range of wikis
*/
function runBackups($from, $to, $full, $options)
{
global $IP, $wgWikiaLocalSettingsPath, $wgWikiaAdminSettingsPath, $wgMaxShellTime, $wgMaxShellFileSize, $wgDumpsDisabledWikis;
$range = array();
/**
* shortcut for full & current together
*/
$both = isset($options["both"]);
/**
* store backup in another folder, not available for users
*/
$hide = isset($options["hide"]);
/**
* store backup in the system tmp dir
*/
$use_temp = isset($options['tmp']);
/**
* send backup to Amazon S3 and delete the local copy
*/
$s3 = isset($options['s3']);
/**
* silly trick, if we have id defined we are defining $from & $to from it
* if we have db param defined we first resolve which id is connected to this
* database
*/
if (isset($options["db"]) && is_string($options["db"])) {
$city_id = WikiFactory::DBtoID($options["db"]);
if ($city_id) {
$from = $to = $city_id;
$to++;
}
} elseif (isset($options["id"]) && is_numeric($options["id"])) {
$from = $to = $options["id"];
$to++;
} elseif (isset($options["even"])) {
$range[] = "city_id % 2 = 0";
$range[] = "city_public = 1";
} elseif (isset($options["odd"])) {
$range[] = "city_id % 2 <> 0";
$range[] = "city_public = 1";
} else {
/**
* if all only for active
*/
$range[] = "city_public = 1";
}
/**
* exclude wikis with dumps disabled
*/
if (!empty($wgDumpsDisabledWikis) && is_array($wgDumpsDisabledWikis)) {
$range[] = 'city_id NOT IN (' . implode(',', $wgDumpsDisabledWikis) . ')';
}
/**
* switch off limits for dumps
*/
$wgMaxShellTime = 0;
$wgMaxShellFileSize = 0;
if ($from !== false && $to !== false) {
$range[] = sprintf("city_id >= %d AND city_id < %d", $from, $to);
Wikia::log(__METHOD__, "info", "Running from {$from} to {$to}", true, true);
} else {
Wikia::log(__METHOD__, "info", "Running for all wikis", true, true);
}
$dbw = Wikifactory::db(DB_MASTER);
$sth = $dbw->select(array("city_list"), array("city_id", "city_dbname"), $range, __METHOD__, array("ORDER BY" => "city_id"));
while ($row = $dbw->fetchObject($sth)) {
/**
* get cluster for this wiki
*/
$cluster = WikiFactory::getVarValueByName("wgDBcluster", $row->city_id);
$server = wfGetDB(DB_SLAVE, 'dumps', $row->city_dbname)->getProperty("mServer");
/**
* build command
*/
$status = false;
$basedir = getDirectory($row->city_dbname, $hide, $use_temp);
if ($full || $both) {
$path = sprintf("%s/%s_pages_full.xml.7z", $basedir, $row->city_dbname);
$time = wfTime();
Wikia::log(__METHOD__, "info", "{$row->city_id} {$row->city_dbname} {$path}", true, true);
$cmd = array("SERVER_ID={$row->city_id}", "php", "{$IP}/maintenance/dumpBackup.php", "--conf {$wgWikiaLocalSettingsPath}", "--aconf {$wgWikiaAdminSettingsPath}", "--full", "--xml", "--quiet", "--server={$server}", "--output=" . DumpsOnDemand::DEFAULT_COMPRESSION_FORMAT . ":{$path}");
wfShellExec(implode(" ", $cmd), $status);
$time = Wikia::timeDuration(wfTime() - $time);
Wikia::log(__METHOD__, "info", "{$row->city_id} {$row->city_dbname} status: {$status}, time: {$time}", true, true);
if ($s3 && 0 == DumpsOnDemand::putToAmazonS3($path, !$hide, MimeMagic::singleton()->guessMimeType($path))) {
unlink($path);
}
}
if (!$full || $both) {
$path = sprintf("%s/%s_pages_current.xml.7z", $basedir, $row->city_dbname);
$time = wfTime();
Wikia::log(__METHOD__, "info", "{$row->city_id} {$row->city_dbname} {$path}", true, true);
$cmd = array("SERVER_ID={$row->city_id}", "php", "{$IP}/maintenance/dumpBackup.php", "--conf {$wgWikiaLocalSettingsPath}", "--aconf {$wgWikiaAdminSettingsPath}", "--current", "--xml", "--quiet", "--server={$server}", "--output=" . DumpsOnDemand::DEFAULT_COMPRESSION_FORMAT . ":{$path}");
wfShellExec(implode(" ", $cmd), $status);
$time = Wikia::timeDuration(wfTime() - $time);
Wikia::log(__METHOD__, "info", "{$row->city_id} {$row->city_dbname} status: {$status}, time: {$time}", true, true);
if ($s3 && 0 == DumpsOnDemand::putToAmazonS3($path, !$hide, MimeMagic::singleton()->guessMimeType($path))) {
unlink($path);
}
}
}
}
