public static function loadData($forceRefresh = false, $forceLanguage = null)
{
global $wgMemc, $wgStatsDB, $wgContLang, $wgExternalSharedDB, $wgStatsDBEnabled;
wfProfileIn(__METHOD__);
self::$mLanguage = !empty($forceLanguage) ? $forceLanguage : $wgContLang->getCode();
$cacheKey = self::CACHE_KEY_TOKEN . ':' . strtoupper(self::$mLanguage);
self::$mData = $wgMemc->get($cacheKey);
if (empty(self::$mData) || $forceRefresh) {
self::$mData = array();
$wikisIDs = array();
// get all the active wikis selected by the sales team
$wikiFactoryRecommended = WikiFactory::getVarByName(self::WF_VAR_NAME, null);
self::$mData['recommended'] = array();
if (!empty($wikiFactoryRecommended) && !empty($wikiFactoryRecommended->cv_variable_id)) {
$dbr = WikiFactory::db(DB_SLAVE);
$res = $dbr->select(array('city_list', 'city_variables'), 'city_id', array('city_id = cv_city_id', 'city_public' => 1, 'city_lang' => self::$mLanguage, 'cv_variable_id' => $wikiFactoryRecommended->cv_variable_id, 'cv_value' => serialize(true)));
while ($row = $dbr->fetchObject($res)) {
self::$mData['recommended'][] = $row->city_id;
}
$dbr->freeResult($res);
}
$counter = 0;
self::$mData['hubs'] = array();
if (!empty($wgStatsDBEnabled)) {
$langs = array(self::$mLanguage);
$wikis = DataMartService::getTopWikisByPageviews(DataMartService::PERIOD_ID_MONTHLY, 200, $langs, null, 1);
$minPageViews = isset(self::$pageviewsLimits[self::$mLanguage]) ? self::$pageviewsLimits[self::$mLanguage] : self::$pageviewsLimits['default'];
foreach ($wikis as $wikiID => $pvCount) {
if ($pvCount >= $minPageViews) {
$hub = WikiFactoryHub::getInstance();
$cat_id = $hub->getCategoryId($wikiID);
if (!$cat_id) {
continue;
}
if (!isset(self::$mData['hubs'][$cat_id])) {
self::$mData['hubs'][$cat_id] = array();
}
self::$mData['hubs'][$cat_id][] = $wikiID;
$counter++;
}
}
}
// removing entries from hubs that have a match in recommended
if (!empty(self::$mData['recommended']) && !empty(self::$mData['hubs'])) {
$counter = 0;
foreach (self::$mData['hubs'] as $hubID => &$item) {
$item = array_diff($item, self::$mData['recommended']);
$counter += count($item);
}
}
self::$mData['total'] = $counter;
$wgMemc->set($cacheKey, self::$mData, 3600 * self::CACHE_EXPIRY);
}
wfProfileOut(__METHOD__);
}
/**
* Get number of items queued in TaskManager
*
* @return int number of items in the queue
*/
private static function getTasksCount()
{
global $wgMemc;
$key = __METHOD__;
$cnt = $wgMemc->get($key);
if (!is_numeric($cnt)) {
$dbr = WikiFactory::db(DB_SLAVE);
$cnt = $dbr->estimateRowCount('wikia_tasks');
$wgMemc->set($key, $cnt, 60 * 1);
}
return $cnt;
}
function submitForm()
{
global $wgRequest, $wgOut, $IP, $wgUser;
$articles = $this->mArguments;
$mode = $this->mMode;
$username = $this->mUsername;
$tempUser = User::newFromName($username);
$sParams = serialize(array('articles' => $articles, 'username' => $username, 'mode' => $mode, 'admin' => $this->mAdmin));
$dbw = WikiFactory::db(DB_MASTER);
$dbw->insert('wikia_tasks', array('task_user_id' => $wgUser->getID(), 'task_type' => $this->mType, 'task_priority' => 1, 'task_status' => 1, 'task_added' => wfTimestampNow(), 'task_started' => '', 'task_finished' => '', 'task_arguments' => $sParams));
$task_id = $dbw->insertId();
$dbw->commit();
return $task_id;
}
public function execute()
{
// get all closed wikis
$WF_db = WikiFactory::db(DB_SLAVE);
$closedBefore = wfTimestamp(TS_MW, strtotime('-3 months'));
$closedWikis = $WF_db->selectFieldValues('city_list', 'city_id', ['city_public' => WikiFactory::CLOSE_ACTION, sprintf('city_lastdump_timestamp < "%s"', $closedBefore)], __METHOD__);
$batches = array_chunk($closedWikis, self::BATCH);
$this->output(sprintf("Got %d closed wikis (before %s) in %d batches\n", count($closedWikis), $closedBefore, count($batches)));
$this->output("Starting in 5 seconds...\n");
sleep(5);
foreach ($batches as $n => $batch) {
$this->cleanupBatch($batch);
}
$this->output("\nDone\n");
}
function submitForm()
{
global $wgRequest, $wgOut, $IP, $wgUser;
$articles = $this->mArguments;
$mode = $this->mMode;
$username = "******";
$tempUser = User::newFromName($username);
#--- all should be correct at this point
#--- first prepare serialized info with params
$sel_articles = $articles;
$sParams = serialize(array("articles" => $sel_articles["articles"], "username" => $username, "mode" => $mode, "admin" => $this->mAdmin, "motherTask" => $this->mMotherTask));
$dbw = WikiFactory::db(DB_MASTER);
$dbw->insert("wikia_tasks", array("task_user_id" => $wgUser->getID(), "task_type" => $this->mType, "task_priority" => 10, "task_status" => 1, "task_added" => wfTimestampNow(), "task_started" => "", "task_finished" => "", "task_arguments" => $sParams));
$task_id = $dbw->insertId();
$dbw->commit();
return $task_id;
}
/**
* Remove variable from WikiFactory (delete from city_variables_pool table)
* @param array $varData
* @return Status
*/
function removeFromVariablesPool($varData)
{
$log = WikiaLogger::instance();
$dbw = WikiFactory::db(DB_MASTER);
$dbw->begin();
try {
$dbw->delete("city_variables_pool", array("cv_id" => $varData['cv_id']), __METHOD__);
$log->info("Remove variable from city_variables_pool table.", $varData);
$dbw->commit();
$status = Status::newGood();
} catch (DBQueryError $e) {
$log->error("Database error: Cannot remove variable from city_variables_pool table.", $varData);
$dbw->rollback();
$status = Status::newFatal("Database error: Cannot remove variable from city_variables_pool table (" . $e->getMessage() . ").");
}
return $status;
}
function generateList($format)
{
global $wgMemc, $wgExternalSharedDB;
$func = "begin_" . $format;
$res = $func();
$dbr = WikiFactory::db(DB_SLAVE, array(), $wgExternalSharedDB);
$sth = $dbr->select(array("city_list"), array("city_title", "city_lang", "city_url", "city_id"), array("city_public = 1"), __METHOD__);
while ($row = $dbr->fetchObject($sth)) {
$row->category = WikiFactory::getCategory($row->city_id);
$func = "body_" . $format;
$res .= $func($row);
}
$func = "end_" . $format;
$res .= $func();
if (!empty($res)) {
$gz_res = gzdeflate($res, 3);
$wgMemc->set(wfSharedMemcKey("{$format}-city-list"), $gz_res, 3600 * 6);
}
}
/**
* @author Federico "Lox" Lucignano
* @param $wikiCityID int the city_id for the wiki
* @return string the name of the cluster the wiki DB belongs to
*
* Retrieves the name of the cluster in which the local DB for the specified wiki is stored
*/
public static function getCityCluster($wikiCityID)
{
wfProfileIn(__METHOD__);
//check for non admitted values
if (empty($wikiCityID) || !is_int($wikiCityID)) {
wfProfileOut(__METHOD__);
return false;
}
wfDebugLog(__CLASS__ . '::' . __METHOD__, "Looking up cluster for wiki with ID {$wikiCityID}");
//WikiFactory implementation
$value = WikiFactory::getVarValueByName('wgDBcluster', $wikiCityID);
//if not found fall back to city_list implementation
if (empty($value)) {
$dbr = WikiFactory::db(DB_SLAVE);
$res = $dbr->selectField('city_list', 'city_cluster', array('city_id' => $wikiCityID));
$value = $res;
}
wfDebugLog(__CLASS__ . '::' . __METHOD__, "Cluster for wiki with ID {$wikiCityID} is '{$value}'" . (empty($value) ? ' (main shared DB)' : null));
wfProfileOut(__METHOD__);
return empty($value) ? self::CLUSTER_DEFAULT : $value;
}
/**
* updateCityDescription
*
* static method called as hook
*
* @static
* @access public
*
* @param WikiPage $article
* @param $user
* @return bool
*/
public static function updateCityDescription(&$article, &$user)
{
global $wgCityId;
if (strtolower($article->getTitle()) == "mediawiki:description") {
$out = trim(strip_tags(wfMsg('description')));
$db = WikiFactory::db(DB_MASTER);
$db->update("city_list", ["city_description" => $out], ["city_id" => $wgCityId], __METHOD__);
}
return true;
}
/**
* axWFactoryDomainQuery
*
* used in autocompletion
*
* @return string JSON encoded array
*/
function axWFactoryDomainQuery()
{
global $wgRequest;
$query = $wgRequest->getVal("query", false);
$return = array("query" => $query, "suggestions" => array(), "data" => array());
$exact = array("suggestions" => array(), "data" => array());
$match = array("suggestions" => array(), "data" => array());
// query terms: wik, wiki, wikia take too much memory
// and end up with fatal errors
if (substr("wikia", 0, strlen((string) $query)) === $query) {
$query = false;
}
if ($query) {
/**
* maybe not very effective but used only by staff anyway
*/
$query = strtolower($query);
$dbr = WikiFactory::db(DB_SLAVE);
$sth = $dbr->select(array("city_domains"), array("city_id", "city_domain"), array("city_domain not like 'www.%'", "city_domain not like '%.wikicities.com'", "city_domain like '%{$query}%'"), __METHOD__);
while ($domain = $dbr->fetchObject($sth)) {
$domain->city_domain = strtolower($domain->city_domain);
if (preg_match("/^{$query}/", $domain->city_domain)) {
$exact["suggestions"][] = $domain->city_domain;
$exact["data"][] = $domain->city_id;
} elseif (preg_match("/{$query}/", $domain->city_domain)) {
$match["suggestions"][] = $domain->city_domain;
$match["data"][] = $domain->city_id;
}
}
$return["suggestions"] = array_merge($exact["suggestions"], $match["suggestions"]);
$return["data"] = array_merge($exact["data"], $match["suggestions"]);
}
return json_encode($return);
}
/**
* remove rows stored in archive database
*
* @access private
*/
private function removeRows()
{
wfProfileIn(__METHOD__);
$dbw = WikiFactory::db(DB_MASTER);
foreach ($this->mData as $task) {
if (!empty($task->moved)) {
$dbw->delete("wikia_tasks", array("task_id" => $task->task_id), __METHOD__);
Wikia::log(__METHOD__, "", sprintf("Task id=%d type=%s added=%s removed.", $task->task_id, $task->task_type, $task->task_added));
}
}
wfProfileOut(__METHOD__);
return true;
}
/**
* used in autocompleting form
*
* @access public
* @static
*
* @return String json-ized answer
*/
public static function axQuery()
{
global $wgRequest;
$query = $wgRequest->getVal("query", false);
$return = array("query" => $query, "suggestions" => array(), "data" => array());
if ($query) {
$query = strtolower($query);
$dbr = WikiFactory::db(DB_SLAVE);
$sth = $dbr->select(array("city_tag"), array("id", "name"), array("name like '%{$query}%'"), __METHOD__);
while ($row = $dbr->fetchObject($sth)) {
$return["suggestions"][] = $row->name;
}
}
return json_encode($return);
}
/**
* 1. go through all wikis which are marked for closing and check which one
* want to have images packed.
*
* 2. pack images, send them via rsync to target server,
*
* 3. mark in city_list.city_flags that images are sent,
*
* 4. remove images
*
* @access public
*/
public function execute()
{
global $wgUploadDirectory, $wgDBname;
if (!isset($this->mOptions['wiki_id'])) {
echo "Wiki Id is not valid";
die(1);
}
$where = array("city_id" => intval($this->mOptions['wiki_id']));
$dbr = WikiFactory::db(DB_SLAVE);
$row = $dbr->selectRow(array("city_list"), array("city_id", "city_flags", "city_dbname", "city_url", "city_public"), $where, __METHOD__);
if (is_object($row)) {
/**
* reasonable defaults for wikis and some presets
*/
$hide = false;
$xdumpok = true;
$newFlags = 0;
$dbname = $row->city_dbname;
$folder = WikiFactory::getVarValueByName("wgUploadDirectory", $row->city_id);
$cluster = WikiFactory::getVarValueByName("wgDBcluster", $row->city_id);
/**
* safety check, if city_dbname is not unique die with message
*/
$check = $dbr->selectRow(array("city_list"), array("count(*) as count"), array("city_dbname" => $dbname), __METHOD__, array("GROUP BY" => "city_dbname"));
if ($check->count > 1) {
echo "{$dbname} is not unique. Check city_list and rerun script";
die(1);
}
Wikia::log(__CLASS__, "info", "city_id={$row->city_id} city_url={$row->city_url} city_dbname={$dbname} city_flags={$row->city_flags} city_public={$row->city_public}");
Wikia::log(__CLASS__, "info", "removing folder {$folder}");
if (is_dir($wgUploadDirectory)) {
/**
* what should we use here?
*/
$cmd = "rm -rf {$folder}";
wfShellExec($cmd, $retval);
if ($retval) {
/**
* info removing folder was not possible
*/
}
}
/**
* clear wikifactory tables, condition for city_public should
* be always true there but better safe than sorry
*/
$dbw = WikiFactory::db(DB_MASTER);
$dbw->delete("city_list", array("city_public" => array(0, -1), "city_id" => $row->city_id), __METHOD__);
Wikia::log(__CLASS__, "info", "{$row->city_id} removed from WikiFactory tables");
/**
* drop database, get db handler for proper cluster
*/
global $wgDBadminuser, $wgDBadminpassword;
$centralDB = empty($cluster) ? "wikicities" : "wikicities_{$cluster}";
/**
* get connection but actually we only need info about host
*/
$local = wfGetDB(DB_MASTER, array(), $centralDB);
$server = $local->getLBInfo('host');
$dbw = new DatabaseMysql($server, $wgDBadminuser, $wgDBadminpassword, $centralDB);
$dbw->begin();
$dbw->query("DROP DATABASE `{$row->city_dbname}`");
$dbw->commit();
Wikia::log(__CLASS__, "info", "{$row->city_dbname} dropped from cluster {$cluster}");
}
}
/**
* Processes shared database (wikicities) and makes all needed changes
*/
public function updateGlobal()
{
wfProfileIn(__METHOD__);
global $wgStatsDB, $wgStatsDBEnabled;
// wikicities
$this->addLog("Updating global shared database: wikicities.");
$dbw = WikiFactory::db(DB_MASTER);
$dbw->begin();
$tasks = self::$mGlobalDefaults;
$hookName = 'UserRename::Global';
$this->addLog("Broadcasting hook: {$hookName}");
wfRunHooks($hookName, array($dbw, $this->mUserId, $this->mOldUsername, $this->mNewUsername, $this, &$tasks));
foreach ($tasks as $task) {
$this->addLog("Updating {$task['table']}.{$task['username_column']}.");
$this->renameInTable($dbw, $task['table'], $this->mUserId, $this->mOldUsername, $this->mNewUsername, $task);
}
$hookName = 'UserRename::AfterGlobal';
$this->addLog("Broadcasting hook: {$hookName}");
wfRunHooks($hookName, array($dbw, $this->mUserId, $this->mOldUsername, $this->mNewUsername, $this, &$tasks));
$dbw->commit();
$this->addLog("Finished updating shared database: wikicities.");
wfProfileOut(__METHOD__);
}
<?php
ini_set( "include_path", dirname(__FILE__)."/.." );
require_once( "commandLine.inc" );
$dbr = WikiFactory::db( DB_SLAVE );
$sth = $dbr->select(
array( "city_list" ),
array( "city_dbname", "city_id" ),
array( "city_public" => 1, "city_cluster" => "c3" ),
__METHOD__
);
while( $row = $dbr->fetchObject( $sth ) ) {
if( $row->city_id == 177 ) {
continue;
}
$dbc = wfGetDB( DB_SLAVE, array( ), $row->city_dbname );
# echo "{$row->city_id} {$row->city_dbname}\n";
if( ! $dbc->fieldExists( "logging", "log_user_text" ) ) {
echo "$row->city_id\n";
}
$dbc->close();
}
/**
* get top wikis
*
* @author Tomasz Odrobny
* @access public
*
*/
public function getTopWikis($tag_id, $lang, $limit, $show_hide = false, $force_reload = false)
{
global $wgMemc, $wgDotDisplay;
wfProfileIn(__METHOD__);
$mcKey = wfSharedMemcKey("auto_hubs", "wikis_top", $tag_id, $lang, $limit);
if (!$force_reload) {
$out = $wgMemc->get($mcKey);
if (!empty($out)) {
wfProfileOut(__METHOD__);
return $out;
}
}
if (empty($this->dbEnabled)) {
wfProfileOut(__METHOD__);
return array("value" => array(), "age" => time());
}
$tag_id = (int) $tag_id;
$limit = 40;
$lang_id = WikiFactory::LangCodeToId($lang);
$conditions = array("tag_id" => $tag_id, "city_lang" => $lang_id);
$res = $this->dbs->select(array('specials.page_views_summary_tags'), array('tag_id as tag_id, city_id as city_id, pv_views as count '), $conditions, __METHOD__, array('ORDER BY' => 'count DESC', 'LIMIT' => $limit));
if ($this->dbs->numRows($res) == 0 && !empty($wgDotDisplay)) {
$date = date('Ymd', time() - 7 * 24 * 60 * 60);
$conditions = array("pv_use_date > {$date}", 'ct.city_id = pv.pv_city_id');
$res = $this->dbs->select(array('stats.page_views as pv', 'wikicities.city_tag_map as ct'), array('ct.tag_id as tag_id,
pv_city_id as city_id,
sum(pv_views) as count '), $conditions, __METHOD__, array('GROUP BY' => ' ct.tag_id,pv_city_id ', 'ORDER BY' => ' count DESC ', 'LIMIT' => $limit));
}
$limits = $this->loadHideLimits("city");
$limits_array = array();
if (!empty($limits[$tag_id]) && count($limits[$tag_id]) > 0) {
foreach ($limits[$tag_id] as $value) {
$limits_array[] = $value['city_id'];
}
} else {
$limits_array = array();
}
$pre_out = 1;
$count = 0;
$city_array = array();
$numberOne = 1;
while ($value = $this->dbs->fetchRow($res)) {
$in_limits = in_array($value['city_id'], $limits_array);
if (!$in_limits || $show_hide) {
$row = array('count' => $value['count']);
if ($in_limits) {
$row['hide'] = true;
}
$city_array[$value['city_id']] = $row;
$count++;
if ($count == $limit) {
break;
}
if ($count == 1) {
$numberOne = $value['city_id'];
}
}
}
$dbs = WikiFactory::db(DB_SLAVE);
if (empty($city_array)) {
return array("value" => array(), "age" => time());
}
$res = $dbs->select(array("city_list"), array("city_id, city_description, city_sitename, city_url, city_title"), array("city_id" => array_keys($city_array)), __METHOD__);
while ($value = $dbs->fetchRow($res)) {
if (!empty($city_array[$value['city_id']])) {
$city_array[$value['city_id']] = array_merge($value, $city_array[$value['city_id']]);
$city_array[$value['city_id']]['city_description'] = trim($city_array[$value['city_id']]['city_description']);
}
}
$out = array("value" => $city_array, "age" => time());
$wgMemc->set($mcKey, $out, 60 * 60 * 12);
wfProfileOut(__METHOD__);
return $out;
}
/**
* axWFactoryDomainQuery
*
* used in autocompletion
*
* @return string JSON encoded array
*/
function axWFactoryDomainQuery()
{
global $wgRequest;
$query = $wgRequest->getVal("query", false);
$return = array("query" => $query, "suggestions" => array(), "data" => array());
$exact = array("suggestions" => array(), "data" => array());
$match = array("suggestions" => array(), "data" => array());
// query terms: wik, wiki, wikia take too much memory
// and end up with fatal errors
if (substr("wikia", 0, strlen((string) $query)) === $query) {
$query = false;
}
if ($query) {
/**
* maybe not very effective but used only by staff anyway
*/
$query = strtolower($query);
$dbr = WikiFactory::db(DB_SLAVE);
$cityDomainLike = $dbr->buildLike($dbr->anyString(), $query, $dbr->anyString());
$sth = $dbr->select(["city_domains"], ["city_id", "city_domain"], ["city_domain not like 'www.%'", "city_domain not like '%.wikicities.com'", "city_domain {$cityDomainLike}"], __METHOD__, ['LIMIT' => 15]);
while ($domain = $dbr->fetchObject($sth)) {
$domain->city_domain = strtolower($domain->city_domain);
if (preg_match("/^{$query}/", $domain->city_domain)) {
$exact["suggestions"][] = $domain->city_domain;
$exact["data"][] = $domain->city_id;
} elseif (preg_match("/{$query}/", $domain->city_domain)) {
$match["suggestions"][] = $domain->city_domain;
$match["data"][] = $domain->city_id;
}
}
$return["suggestions"] = array_merge($exact["suggestions"], $match["suggestions"]);
$return["data"] = array_merge($exact["data"], $match["suggestions"]);
}
$resp = new AjaxResponse(json_encode($return));
$resp->setContentType('application/json; charset=utf-8');
return $resp;
}
/**
* @static
* @access public
*/
public static function sendMail($sDbName = null, $iCityId = null, $bHidden = false, $bClose = false)
{
if (is_null($sDbName)) {
global $wgDBname;
$sDbName = $wgDBname;
}
if (is_null($iCityId)) {
global $wgCityId;
$iCityId = $wgCityId;
}
global $wgServer, $wgUser;
$title = SpecialPage::getTitleFor("Statistics");
$body = sprintf("Database dump request for %s, city id %d, hidden %d, closeWiki %d\nurl %s\nRequested by %s\n", $sDbName, $iCityId, (int) $bHidden, (int) $bClose, $title->getFullUrl(), $wgUser->getName());
UserMailer::send(new MailAddress("*****@*****.**"), new MailAddress("*****@*****.**"), "Database dump request for {$sDbName}", $body, null, null, 'DumpRequest');
/**
* @todo universal WikiFactory metod for that
*/
$dbw = WikiFactory::db(DB_MASTER);
$dbw->update("city_list", array("city_lastdump_timestamp" => wfTimestampNow()), array("city_id" => $iCityId), __METHOD__);
}
/**
* check if domain exist and is active
* return false if doesn't exist
* return 0 if exist but not active
* return id if exists and active
*/
static function checkWiki($name)
{
$dbr = WikiFactory::db(DB_MASTER);
$oRow = $dbr->selectRow(array("city_list", "city_domains"), array("city_public", "city_list.city_id"), array("city_list.city_id = city_domains.city_id", "city_domain" => $name), __METHOD__);
if (empty($oRow->city_id)) {
return false;
} elseif ($oRow->city_public != 1) {
return 0;
} else {
return $oRow->city_id;
}
}
/**
* constructor
*
* @access public
*/
function __construct($par = null)
{
global $wgRequest, $wgUser;
$parms = explode('/', $par = $par !== null ? $par : '');
if (isset($parms[0]) && !empty($parms[0])) {
$this->firstChar = $parms[0];
}
if (isset($parms[1]) && !empty($parms[1])) {
$this->lang = $parms[1];
}
$this->lang = $this->lang != '' ? $this->lang : $wgRequest->getVal('language');
$this->firstChar = $this->firstChar != '' ? $this->firstChar : $wgRequest->getText('start');
$this->hub = $wgRequest->getText('hub');
if ($wgUser->isAllowed('newwikislist')) {
$this->more_details = true;
}
parent::__construct();
/**
* overwrite database handler
*/
$this->mDb = WikiFactory::db(DB_SLAVE);
// create a Linker object so we don't have to create it every time in formatRow
$this->linker = new Linker();
}
/**
* get all defined tags from city_tag list
*/
private function getTags()
{
global $wgMemc;
if (isset($this->mCachedTags) && is_array($this->mCachedTags)) {
return $this->mCachedTags;
}
wfProfileIn(__METHOD__);
/**
* first from cache, cache is set for 1h
*/
$this->mCachedTags = $wgMemc->get($this->cacheKey());
if (!isset($this->mCachedTags) || !is_array($this->mCachedTags)) {
/**
* if not success then from database
*/
$dbr = WikiFactory::db(DB_SLAVE);
$sth = $dbr->select(array("city_tag"), array("*"), false, __METHOD__);
while ($row = $dbr->fetchObject($sth)) {
$this->mCachedTags[$row->name] = $row->id;
}
/**
* store for hour
*/
$wgMemc->set($this->cacheKey(), $this->mCachedTags, 3600);
}
wfProfileOut(__METHOD__);
return $this->mCachedTags;
}
private function getData()
{
/**
* build query
*/
wfProfileIn(__METHOD__);
$offset = $this->mOffset ? array("OFFSET" => $this->mOffset) : array();
$dbr = WikiFactory::db(DB_SLAVE);
$sth = $dbr->select(array("city_domains"), array("distinct city_id"), array("city_domain " . $dbr->buildLike($dbr->anyString(), $this->mPart, $dbr->anyString())), __METHOD__, array("ORDER BY" => "city_id", "LIMIT" => $this->mLimit + 1) + $offset);
$data = array();
$counter = 0;
while ($row = $dbr->fetchObject($sth)) {
$obj = new stdClass();
$obj->wiki = WikiFactory::getWikiByID($row->city_id);
$obj->domains = WikiFactory::getDomains($row->city_id);
if ($counter <= $this->mLimit) {
$data[] = $obj;
} else {
/**
* there's next page
*/
}
}
wfProfileOut(__METHOD__);
return $data;
}
* @author: Federico 'Lox' Lucignano <*****@*****.**>
*
* @usage: SERVER_ID=177 php achievements_fix_luckyedit_score.php --conf /usr/wikia/conf/current/wiki.factory/LocalSettings.php --aconf /usr/wikia/conf/current/AdminSettings.php
*/
ini_set("include_path", dirname(__FILE__) . "/../");
$options = array('help');
require_once 'commandLine.inc';
global $IP, $wgCityId;
echo "Update Achievements lucky edits score\n\n";
if (isset($options['help']) && $options['help']) {
echo "Usage: php achievements_takeRankingSnapshot.php\n\n";
exit(0);
}
require_once "{$IP}/extensions/wikia/AchievementsII/Ach_setup.php";
echo "Loading list of users to process";
$dbw = WikiFactory::db(DB_MASTER);
$users = $dbw->select('ach_user_badges', array("DISTINCT user_id", 'wiki_id'), array('badge_type_id' => BADGE_LUCKYEDIT));
$usersCount = $users->numRows();
echo ": {$usersCount} user(s) to process\n\n";
if ($usersCount) {
while ($currentUser = $dbw->fetchObject($users)) {
echo "- Processing user {$currentUser->user_id} on wiki {$currentUser->wiki_id}: ";
$badges = $dbw->selectField('ach_user_badges', array('COUNT(*) as cnt'), array('wiki_id' => $currentUser->wiki_id, 'user_id' => $currentUser->user_id, 'badge_type_id' => BADGE_LUCKYEDIT));
if ($badges > 1) {
$score = $dbw->selectField('ach_user_score', array('score'), array('wiki_id' => $currentUser->wiki_id, 'user_id' => $currentUser->user_id));
echo "lucky edits {$badges}, original score {$score}";
$wgCityId = $currentUser->wiki_id;
$srvc = new AchAwardingService();
$srvc->migration($currentUser->user_id);
sleep(3);
$score = $dbw->selectField('ach_user_score', array('score'), array('wiki_id' => $currentUser->wiki_id, 'user_id' => $currentUser->user_id));
/**
* submitForm
*
* Creates the corresponding entry in the TaskManager queue.
*
* @return boolean true
* @access public
*/
function submitForm()
{
global $wgOut, $wgUser, $wgRequest;
$wikiDomain = Wikia::fixDomainName($wgRequest->getText('task-usp-wiki'));
// check if the wiki exists
$dbr = WikiFactory::db(DB_MASTER);
$oRow = $dbr->selectRow(array('city_list', 'city_domains'), array('city_public', 'city_list.city_id'), array('city_list.city_id = city_domains.city_id', 'city_domain' => $wikiDomain), __METHOD__);
if (empty($oRow->city_id) || 1 != $oRow->city_public) {
return false;
}
$wikiId = $oRow->city_id;
if (!empty($wikiId)) {
$this->mTaskID = $this->createTask(array('wikiId' => $wikiId));
}
$wgOut->addHTML(Wikia::successbox('Task added'));
return true;
}
function loadFromSnapshot()
{
global $wgCityId;
$dbr = WikiFactory::db(DB_SLAVE);
$res = $dbr->select('ach_ranking_snapshots', array('data'), array('wiki_id' => $wgCityId));
if ($row = $dbr->fetchObject($res)) {
return unserialize($row->data);
}
return null;
}
function Ach_TakeRankingSnapshot($force = false)
{
global $wgCityId;
$dbw = WikiFactory::db(DB_MASTER);
$res = $dbw->select('ach_ranking_snapshots', array('date'), array('wiki_id' => $wgCityId));
$rankingService = new AchRankingService();
if ($row = $dbw->fetchObject($res)) {
if (strtotime($row->date) <= time() - 60 * 60 * 24 || $force) {
$dbw->update('ach_ranking_snapshots', array('date' => date('Y-m-d H:i:s'), 'data' => $rankingService->serialize()), array('wiki_id' => $wgCityId));
echo "\t* Snapshot for the wiki with ID {$wgCityId} has been updated\n";
$dbw->commit();
} else {
echo "\t* A user ranking snapshot already exists for the wiki with ID {$wgCityId} and is still valid (taken on {$row->date})\n";
}
} else {
$dbw->insert('ach_ranking_snapshots', array('wiki_id' => $wgCityId, 'date' => date('Y-m-d H:m:s'), 'data' => $rankingService->serialize()));
echo "\t* Snapshot for the wiki with ID {$wgCityId} has been taken\n";
$dbw->commit();
}
$dbw->freeResult($res);
}
public function execute()
{
global $wgUser, $wgDBname, $wgCityId;
$wgUser = User::newFromName(self::USER);
$this->isDryRun = $this->hasOption('dry-run');
$this->dbr = WikiFactory::db(DB_SLAVE);
$this->cityId = $wgCityId;
$this->output("Running for '{$wgDBname}' (city #{$this->cityId})...\n");
if ($this->isDryRun) {
$this->output("\n### DRY RUN MODE ###\n\n");
}
// first, get the current values from DB
$this->getCurrentVariables();
// now get values from city_list entry
$this->getVariablesFromWikiData();
// extract values from WikiFactory log
$this->getVariablesFromLog();
// store in DB
// if not in --dry-run mode
if (!$this->isDryRun) {
$this->storeInWikiFactory();
}
$this->output("\nDone!\n");
}
/**
* Hooks into WikiFactory to force use of the wiki which the developer
* has explicitly set using this panel (if applicable).
*
* @return boolean true to allow the WikiFactoryLoader to do its other necessary initalization.
*/
function wfDevBoxForceWiki(WikiFactoryLoader $wikiFactoryLoader)
{
global $wgDevelEnvironment, $wgExternalSharedDB, $wgCommandLineMode, $wgDevboxDefaultWikiDomain;
if ($wgDevelEnvironment) {
$forcedWikiDomain = getForcedWikiValue();
$cityId = WikiFactory::DomainToID($forcedWikiDomain);
if (!$cityId) {
// If the overridden name doesn't exist AT ALL, use a default just to let the panel run.
$forcedWikiDomain = $wgDevboxDefaultWikiDomain;
$cityId = WikiFactory::DomainToID($forcedWikiDomain);
wfDebug(__METHOD__ . ": domain forced to {$forcedWikiDomain}\n");
}
if ($wgCommandLineMode) {
$cityId = getenv("SERVER_ID");
if (is_numeric($cityId)) {
$wikiFactoryLoader->mCityID = $cityId;
$wikiFactoryLoader->mWikiID = $cityId;
} else {
$dbName = getenv("SERVER_DBNAME");
/**
* find city_id by database name
*/
$dbr = WikiFactory::db(DB_SLAVE);
$cityId = $dbr->selectField("city_list", array("city_id"), array("city_dbname" => $dbName), __METHOD__);
if (is_numeric($cityId)) {
$wikiFactoryLoader->mCityID = $cityId;
$wikiFactoryLoader->mWikiID = $cityId;
}
}
}
if ($cityId) {
$wikiFactoryLoader->mServerName = $forcedWikiDomain;
// Need to set both in order to get our desired effects.
$wikiFactoryLoader->mCityID = $cityId;
$wikiFactoryLoader->mWikiID = $cityId;
}
// This section allows us to use c1 or c2 as a source for wiki databases
// Be aware that this means the database has to be loaded in the right cluster according to wikicities!
$db = WikiFactory::db(DB_SLAVE);
$cluster = $db->selectField('city_list', 'city_cluster', ['city_id' => $cityId], __METHOD__);
$wikiFactoryLoader->mVariables["wgDBcluster"] = $cluster;
// Final sanity check to make sure our database exists
if ($forcedWikiDomain != $wgDevboxDefaultWikiDomain) {
// check if the wiki exist on a cluster
wfDebug(__METHOD__ . ": checking if wiki #{$cityId} exists on {$cluster} cluster...\n");
$dbname = WikiFactory::DomainToDB($forcedWikiDomain);
$db = wfGetDB(DB_MASTER, [], $wgExternalSharedDB . '_' . $cluster);
// force master - @see PLATFORM-528
$res = $db->query('SHOW DATABASES ' . $db->buildLike($dbname), __METHOD__);
// SHOW DATABASES LIKE 'muppet'
if ($res->numRows() === 0) {
header('HTTP/1.1 503');
header('X-Error: missing database');
header('Content-Type: text/plain');
die("No local copy of database [{$dbname}] was found on {$cluster} cluster [using {$db->getServer()} DB].");
}
}
}
return true;
}
/**
* can create database?
* @todo this code is probably duplication of other checkers
*/
private function canCreateDatabase()
{
// default response
$can = true;
// check local cluster
$row = $this->mNewWiki->dbw->selectRow("INFORMATION_SCHEMA.SCHEMATA", array("SCHEMA_NAME as name"), array('SCHEMA_NAME' => $this->mNewWiki->dbname), __METHOD__);
if (isset($row->name) && $row->name === $this->mNewWiki->dbname) {
wfDebugLog("createwiki", __METHOD__ . ": database {$this->mNewWiki->dbname} already exists on active cluster\n");
$can = false;
} else {
// check city_list
$dbw = WikiFactory::db(DB_MASTER);
$row = $dbw->selectRow("city_list", array("count(*) as count"), array("city_dbname" => $this->mNewWiki->dbname), __METHOD__);
if ($row->count > 0) {
wfDebugLog("createwiki", __METHOD__ . ": database {$this->mNewWiki->dbname} already used in city_list\n");
$can = false;
} else {
// check domain
$row = $dbw->selectRow("city_list", array("count(*) as count"), array("city_url" => $this->mNewWiki->url), __METHOD__);
if ($row->count > 0) {
wfDebugLog("createwiki", __METHOD__ . ": domain {$this->mNewWiki->url} already used in city_list\n");
$can = false;
}
}
}
return $can;
}
/**
* 1. go through all wikis which are marked for closing and check which one
* want to have images packed.
*
* 2. pack images, send them via rsync to target server,
*
* 3. mark in city_list.city_flags that images are sent,
*
* 4. remove images
*
* @access public
*/
public function execute()
{
global $wgUploadDirectory, $wgDBname, $IP;
$first = isset($this->mOptions["first"]) ? true : false;
$sleep = isset($this->mOptions["sleep"]) ? $this->mOptions["sleep"] : 15;
$condition = array("ORDER BY" => "city_id");
$this->info('start', ['first' => $first, 'limit' => $this->mOptions["limit"] ?: false]);
/**
* if $first is set skip limit checking
*/
if (!$first) {
if (isset($this->mOptions["limit"]) && is_numeric($this->mOptions["limit"])) {
$condition["LIMIT"] = $this->mOptions["limit"];
}
}
$timestamp = wfTimestamp(TS_DB, strtotime(sprintf("-%d days", self::CLOSE_WIKI_DELAY)));
$dbr = WikiFactory::db(DB_SLAVE);
$sth = $dbr->select(array("city_list"), array("city_id", "city_flags", "city_dbname", "city_url", "city_public"), array("city_public" => array(0, -1), "city_flags <> 0 && city_flags <> 32", "city_last_timestamp < '{$timestamp}'"), __METHOD__, $condition);
$this->info('wikis to remove', ['wikis' => $sth->numRows()]);
while ($row = $dbr->fetchObject($sth)) {
/**
* reasonable defaults for wikis and some presets
*/
$hide = false;
$xdumpok = true;
$newFlags = 0;
$dbname = $row->city_dbname;
$cityid = $row->city_id;
$folder = WikiFactory::getVarValueByName("wgUploadDirectory", $cityid);
$cluster = WikiFactory::getVarValueByName("wgDBcluster", $cityid);
/**
* safety check, if city_dbname is not unique die with message
*/
$check = $dbr->selectRow(array("city_list"), array("count(*) as count"), array("city_dbname" => $dbname), __METHOD__, array("GROUP BY" => "city_dbname"));
if ($check->count > 1) {
echo "{$dbname} is not unique. Check city_list and rerun script";
die(1);
}
$this->log("city_id={$row->city_id} city_url={$row->city_url} city_dbname={$dbname} city_flags={$row->city_flags} city_public={$row->city_public}");
/**
* request for dump on remote server (now hardcoded for Iowa)
*/
if ($row->city_flags & WikiFactory::FLAG_HIDE_DB_IMAGES) {
$hide = true;
}
if ($row->city_flags & WikiFactory::FLAG_CREATE_DB_DUMP) {
$this->log("Dumping database on remote host");
list($remote) = explode(":", $this->mTarget, 2);
$script = $hide ? "--script='../extensions/wikia/WikiFactory/Dumps/runBackups.php --both --id={$cityid} --tmp --s3'" : "--script='../extensions/wikia/WikiFactory/Dumps/runBackups.php --both --id={$cityid} --hide --tmp --s3'";
$cmd = array("/usr/wikia/backend/bin/run_maintenance", "--id=177", $script);
$cmd = '/usr/wikia/backend/bin/run_maintenance --id=177 ' . wfEscapeShellArg($script);
$this->log($cmd);
$output = wfShellExec($cmd, $retval);
$xdumpok = empty($retval) ? true : false;
/**
* reset flag
*/
$newFlags = $newFlags | WikiFactory::FLAG_CREATE_DB_DUMP | WikiFactory::FLAG_HIDE_DB_IMAGES;
}
if ($row->city_flags & WikiFactory::FLAG_CREATE_IMAGE_ARCHIVE) {
if ($dbname && $folder) {
$source = $this->tarFiles($folder, $dbname, $cityid);
if ($source) {
$retval = DumpsOnDemand::putToAmazonS3($source, !$hide, MimeMagic::singleton()->guessMimeType($source));
if ($retval > 0) {
$this->log("putToAmazonS3 command failed.");
echo "Can't copy images to remote host. Please, fix that and rerun";
die(1);
} else {
$this->log("{$source} copied to S3 Amazon");
unlink($source);
$newFlags = $newFlags | WikiFactory::FLAG_CREATE_IMAGE_ARCHIVE | WikiFactory::FLAG_HIDE_DB_IMAGES;
}
} else {
/**
* actually it's better to die than remove
* images later without backup
*/
echo "Can't copy images to remote host. Source {$source} is not defined";
}
}
}
if ($row->city_flags & WikiFactory::FLAG_DELETE_DB_IMAGES || $row->city_flags & WikiFactory::FLAG_FREE_WIKI_URL) {
/**
* clear wikifactory tables, condition for city_public should
* be always true there but better safe than sorry
*/
WikiFactory::copyToArchive($row->city_id);
$dbw = WikiFactory::db(DB_MASTER);
$dbw->delete("city_list", array("city_public" => array(0, -1), "city_id" => $row->city_id), __METHOD__);
$this->log("{$row->city_id} removed from WikiFactory tables");
/**
* remove records from dataware
*/
global $wgExternalDatawareDB;
$datawareDB = wfGetDB(DB_MASTER, array(), $wgExternalDatawareDB);
$datawareDB->delete("pages", array("page_wikia_id" => $row->city_id), __METHOD__);
$this->log("{$row->city_id} removed from pages table");
/**
* remove images from D.I.R.T.
*/
$datawareDB->delete("image_review", array("wiki_id" => $row->city_id), __METHOD__);
$this->log("{$row->city_id} removed from image_review table");
$datawareDB->delete("image_review_stats", array("wiki_id" => $row->city_id), __METHOD__);
$this->log("{$row->city_id} removed from image_review_stats table");
$datawareDB->delete("image_review_wikis", array("wiki_id" => $row->city_id), __METHOD__);
$this->log("{$row->city_id} removed from image_review_wikis table");
$datawareDB->commit();
/**
* drop database, get db handler for proper cluster
*/
global $wgDBadminuser, $wgDBadminpassword;
$centralDB = empty($cluster) ? "wikicities" : "wikicities_{$cluster}";
/**
* get connection but actually we only need info about host
*/
$local = wfGetDB(DB_MASTER, array(), $centralDB);
$server = $local->getLBInfo('host');
try {
$dbw = new DatabaseMysql($server, $wgDBadminuser, $wgDBadminpassword, $centralDB);
$dbw->begin();
$dbw->query("DROP DATABASE `{$row->city_dbname}`");
$dbw->commit();
$this->log("{$row->city_dbname} dropped from cluster {$cluster}");
} catch (Exception $e) {
$this->log("{$row->city_dbname} database drop failed! {$e->getMessage()}");
$this->info('drop database', ['cluster' => $cluster, 'dbname' => $row->city_dbname, 'exception' => $e, 'server' => $server]);
}
/**
* update search index
*/
$indexer = new Wikia\Search\Indexer();
$indexer->deleteWikiDocs($row->city_id);
$this->log("Wiki documents removed from index");
/**
* there is nothing to set because row in city_list doesn't
* exists
*/
$newFlags = false;
}
/**
* reset flags, if database was dropped and data were removed from
* WikiFactory tables it will return false anyway
*/
if ($newFlags) {
WikiFactory::resetFlags($row->city_id, $newFlags);
}
$this->info('closed', ['cluster' => $cluster, 'city_id' => (int) $cityid, 'dbname' => $dbname]);
/**
* just one?
*/
if ($first) {
break;
}
sleep($sleep);
}
}
