/**
* Get the top wikis by weekly pageviews optionally filtering by vertical (hub) and/or language
*
* @param string $hub [OPTIONAL] The name of the vertical (e.g. Gaming, Entertainment,
* Lifestyle, etc.) to use as a filter
* @param string $lang [OPTIONAL] The language code (e.g. en, de, fr, es, it, etc.) to use as a filter
*
* @return array A collection of results with id, name, hub, language, topic and domain
*/
public function getTop($lang = null, $hub = null)
{
$this->wf->profileIn(__METHOD__);
$cacheKey = $this->wf->SharedMemcKey(__METHOD__, self::CACHE_VERSION, $lang, $hub);
$results = $this->wg->Memc->get($cacheKey);
if (!is_array($results)) {
$results = array();
$wikis = DataMartService::getTopWikisByPageviews(DataMartService::PERIOD_ID_WEEKLY, self::MAX_RESULTS, $lang, $hub, 1);
foreach ($wikis as $wikiId => $wiki) {
//fetching data from WikiFactory
//the table is indexed and values cached separately
//so making one query for all of them or a few small
//separate ones doesn't make any big difference while
//this respects WF's data abstraction layer
//also: WF data is heavily cached
$name = WikiFactory::getVarValueByName('wgSitename', $wikiId);
$hubName = !empty($hub) ? $hub : $this->getVerticalByWikiId($wikiId);
$langCode = WikiFactory::getVarValueByName('wgLanguageCode', $wikiId);
$topic = WikiFactory::getVarValueByName('wgWikiTopics', $wikiId);
$domain = $this->getDomainByWikiId($wikiId);
$results[] = array('id' => $wikiId, 'name' => !empty($name) ? $name : null, 'hub' => $hubName, 'language' => !empty($langCode) ? $langCode : null, 'topic' => !empty($topic) ? $topic : null, 'domain' => $domain);
}
$this->wg->Memc->set($cacheKey, $results, 86400);
}
$this->wf->profileOut(__METHOD__);
return $results;
}
public static function loadData($forceRefresh = false, $forceLanguage = null)
{
global $wgMemc, $wgStatsDB, $wgContLang, $wgExternalSharedDB, $wgStatsDBEnabled;
wfProfileIn(__METHOD__);
self::$mLanguage = !empty($forceLanguage) ? $forceLanguage : $wgContLang->getCode();
$cacheKey = self::CACHE_KEY_TOKEN . ':' . strtoupper(self::$mLanguage);
self::$mData = $wgMemc->get($cacheKey);
if (empty(self::$mData) || $forceRefresh) {
self::$mData = array();
$wikisIDs = array();
// get all the active wikis selected by the sales team
$wikiFactoryRecommended = WikiFactory::getVarByName(self::WF_VAR_NAME, null);
self::$mData['recommended'] = array();
if (!empty($wikiFactoryRecommended) && !empty($wikiFactoryRecommended->cv_variable_id)) {
$dbr = WikiFactory::db(DB_SLAVE);
$res = $dbr->select(array('city_list', 'city_variables'), 'city_id', array('city_id = cv_city_id', 'city_public' => 1, 'city_lang' => self::$mLanguage, 'cv_variable_id' => $wikiFactoryRecommended->cv_variable_id, 'cv_value' => serialize(true)));
while ($row = $dbr->fetchObject($res)) {
self::$mData['recommended'][] = $row->city_id;
}
$dbr->freeResult($res);
}
$counter = 0;
self::$mData['hubs'] = array();
if (!empty($wgStatsDBEnabled)) {
$langs = array(self::$mLanguage);
$wikis = DataMartService::getTopWikisByPageviews(DataMartService::PERIOD_ID_MONTHLY, 200, $langs, null, 1);
$minPageViews = isset(self::$pageviewsLimits[self::$mLanguage]) ? self::$pageviewsLimits[self::$mLanguage] : self::$pageviewsLimits['default'];
foreach ($wikis as $wikiID => $pvCount) {
if ($pvCount >= $minPageViews) {
$hub = WikiFactoryHub::getInstance();
$cat_id = $hub->getCategoryId($wikiID);
if (!$cat_id) {
continue;
}
if (!isset(self::$mData['hubs'][$cat_id])) {
self::$mData['hubs'][$cat_id] = array();
}
self::$mData['hubs'][$cat_id][] = $wikiID;
$counter++;
}
}
}
// removing entries from hubs that have a match in recommended
if (!empty(self::$mData['recommended']) && !empty(self::$mData['hubs'])) {
$counter = 0;
foreach (self::$mData['hubs'] as $hubID => &$item) {
$item = array_diff($item, self::$mData['recommended']);
$counter += count($item);
}
}
self::$mData['total'] = $counter;
$wgMemc->set($cacheKey, self::$mData, 3600 * self::CACHE_EXPIRY);
}
wfProfileOut(__METHOD__);
}
/**
* Provided an Article, queries the database for weekly and monthly pageviews.
* @return array
*/
public function execute()
{
wfProfileIn(__METHOD__);
$service = $this->getService();
if ($this->result !== null || !$service->isOnDbCluster()) {
wfProfileOut(__METHOD__);
return $this->result;
}
$stringKey = 'WikiaSearchPageViews';
$result = $this->service->getCacheResultFromString($stringKey);
if ($result !== false && ($result->weekly > 0 || $result->monthly > 0)) {
wfProfileOut(__METHOD__);
$this->result = array('wikiviews_weekly' => (int) $result->weekly, 'wikiviews_monthly' => (int) $result->monthly);
return $this->result;
}
$row = new \stdClass();
$row->weekly = 0;
$row->monthly = 0;
$datamart = new \DataMartService();
$startDate = date('Y-m-d', strtotime('-1 week'));
$endDate = date('Y-m-01', strtotime('now'));
$pageviews_weekly = $datamart->getPageviewsWeekly($startDate, $endDate, $this->service->getWikiId());
if (!empty($pageviews_weekly)) {
foreach ($pageviews_weekly as $pview) {
$row->weekly += $pview;
}
}
$startDate = date('Y-m-01', strtotime('-1 month'));
$pageviews_monthly = $datamart->getPageviewsMonthly($startDate, $endDate, $this->service->getWikiId());
if (!empty($pageviews_monthly)) {
foreach ($pageviews_monthly as $pview) {
$row->monthly += $pview;
}
}
$wrapper = new \Wikia\Util\GlobalStateWrapper(array('wgAllowMemcacheWrites' => true));
$wrapper->wrap(function () use($stringKey, $row) {
$this->service->setCacheFromStringKey($stringKey, $row, self::WIKIPAGES_CACHE_TTL);
});
$this->result = array('wikiviews_weekly' => (int) $row->weekly, 'wikiviews_monthly' => (int) $row->monthly);
wfProfileOut(__METHOD__);
return $this->result;
}
protected function getDailyPageViews(array &$stats)
{
$this->wf->ProfileIn(__METHOD__);
$week = date('Y-m-d', strtotime('-7 day'));
$pageviews = DataMartService::getPageviewsDaily($week);
$stats['totals']['pageviews'] = 0;
foreach ($pageviews as $date => $value) {
$stats[$date]['pageviews'] = $value;
$stats['totals']['pageviews'] += $value;
}
$this->wf->ProfileOut(__METHOD__);
}
protected function getPageViews($period)
{
$timestamp = strtotime($period);
$startDate = date('Y-m-d', $timestamp);
$endDate = date('Y-m-d', strtotime('-1 day'));
$res = 0;
$pageviews = DataMartService::getPageviewsMonthly($startDate, $endDate, $this->id);
if (!empty($pageviews) && is_array($pageviews)) {
foreach ($pageviews as $date => $value) {
$res += $value;
}
}
return intval($res);
}
/**
* Get 10 most popular cross wiki articles
*
* @param string $hubName
* @param string $langCode
* @return array of articles - format
* [
* ['wikiId' => 1, 'articleId' => 2],
* ['wikiId' => 2, 'articleId' => 3],
* ]
*/
protected function getTopArticles($hubName, $langCode)
{
wfProfileIn(__METHOD__);
$topArticles = [];
$results = \DataMartService::getTopCrossWikiArticlesByPageview($hubName, [$langCode], [NS_MAIN]);
$articlesCount = ceil(self::MAX_LIMIT / count($results));
foreach ($results as $wikiResult) {
for ($i = 0; $i < $articlesCount; $i++) {
$topArticles[] = ['wikiId' => $wikiResult['wiki']['id'], 'articleId' => $wikiResult['articles'][$i]['id']];
}
}
wfDebug(sprintf("%s: returning %s items\n", __METHOD__, count($topArticles)));
wfProfileOut(__METHOD__);
return $topArticles;
}
/**
* Returns page views count in last month.
* @param $pageIds array|int Array of article ids (or single integer to check one article)
* @return array|int
*/
protected static function getPageViews($pageIds)
{
global $wgStatsDB, $wgCityId, $wgDevelEnvironment, $wgStatsDBEnabled;
wfProfileIn(__METHOD__);
if (empty($pageIds)) {
wfProfileOut(__METHOD__);
return is_array($pageIds) ? array() : 0;
}
//fix for BugId:33086
//use DataMart for pageviews data as all the other
//data sources are stale/obsolete
$data = array();
$ids = is_array($pageIds) ? $pageIds : array($pageIds);
$rows = DataMartService::getTopArticlesByPageview($wgCityId, $ids, null, false, count($ids));
foreach ($rows as $id => $pv) {
$data[intval($id)] = intval($pv);
}
// If asked with plain integer return a single integer too
if (!is_array($pageIds)) {
$data = !empty($data) ? reset($data) : 0;
}
wfProfileOut(__METHOD__);
return $data;
}
/**
*
* Returns Categories under a given Tag
*
* @param $content
* @param $requestTag
*
* @responseReturn Array|false Categories or false if tag was not found
*/
private function getTagCategories($content, $requestTag)
{
wfProfileIn(__METHOD__);
$ret = false;
foreach ($content as $tag) {
if ($requestTag == $tag['title']) {
$ret = $tag['categories'];
}
}
if (!empty($ret)) {
$sort = $this->request->getVal('sort');
if (!empty($sort)) {
if ($sort == 'alpha') {
usort($ret, function ($a, $b) {
return strcasecmp($a['title'], $b['title']);
});
} else {
if ($sort == 'hot') {
$hot = array_keys(DataMartService::getTopArticlesByPageview($this->wg->CityId, array_reduce($ret, function ($ret, $item) {
$ret[] = $item['id'];
return $ret;
}), null, false, count($ret)));
$sorted = [];
$left = [];
foreach ($ret as $value) {
$key = array_search($value['id'], $hot);
if ($key === false) {
$left[] = $value;
} else {
$sorted[$key] = $value;
}
}
ksort($sorted);
$ret = array_merge($sorted, $left);
} else {
wfProfileOut(__METHOD__);
throw new InvalidParameterApiException('sort');
}
}
}
//Use 'id' instead of image_id
foreach ($ret as &$value) {
if (!empty($value['image_id'])) {
$value['id'] = $value['image_id'];
}
unset($value['image_id']);
}
$this->response->setVal('items', $ret);
} else {
if ($requestTag !== '') {
wfProfileOut(__METHOD__);
throw new InvalidParameterApiException('tag');
}
}
wfProfileOut(__METHOD__);
}
/**
* Get the top articles by pageviews for a hub optionally filtering by namespace and/or language,
* available only on the www.wikia.com main domain (see examples)
*
* @requestParam string $hub The name of the vertical (e.g. Gaming, Entertainment, Lifestyle, etc.) to use as a filter
* @requestParam string $lang [OPTIONAL] The language code (e.g. en, de, fr, es, it, etc.) to use as a filter
* @requestParam array $namespaces [OPTIONAL] The ID's of the namespaces (e.g. 0, 14, 6, etc.) to use as a filter, comma separated
*
* @responseParam array $items The list of top 10 wikis, each with a collection of top articles, the schema is as follows:[ [ wiki => [ id, name, language, domain ], articles => [ id, ns ] ] ]
*
* @example http://www.wikia.com/wikia.php?controller=ArticlesApi&method=getTopByHub&hub=Gaming
* @example http://www.wikia.com/wikia.php?controller=ArticlesApi&method=getTopByHub&hub=Gaming&namespaces=0,14
* @example http://www.wikia.com/wikia.php?controller=ArticlesApi&method=getTopByHub&hub=Gaming&lang=de
*/
public function getTopByHub()
{
wfProfileIn(__METHOD__);
if ($this->wg->DBname == 'wikiaglobal') {
$hub = trim($this->request->getVal(self::PARAMETER_HUB, null));
$langs = $this->request->getArray(self::PARAMETER_LANGUAGES);
$namespaces = self::processNamespaces($this->request->getArray(self::PARAMETER_NAMESPACES, null), __METHOD__);
if (empty($hub)) {
wfProfileOut(__METHOD__);
throw new MissingParameterApiException(self::PARAMETER_HUB);
}
if (!empty($langs) && count($langs) > self::LANGUAGES_LIMIT) {
wfProfileOut(__METHOD__);
throw new LimitExceededApiException(self::PARAMETER_LANGUAGES, self::LANGUAGES_LIMIT);
}
$res = DataMartService::getTopCrossWikiArticlesByPageview($hub, $langs, $namespaces);
wfProfileOut(__METHOD__);
if (empty($res)) {
wfProfileOut(__METHOD__);
throw new NotFoundApiException();
}
$this->response->setVal('items', $res);
} else {
wfProfileOut(__METHOD__);
throw new BadRequestApiException();
}
}
private function getWikisByNbrPageviews()
{
global $wgMemc, $wgStatsDB, $wgStatsDBEnabled;
$pageViews = $this->axNbrPageviews;
$pageViewsDays = $this->axNbrPageviewsDays;
if (empty($pageViewsDays)) {
$pageViewsDays = self::DEF_DAYS_PVIEWS;
}
$memkey = __METHOD__ . "v:" . $pageViews . "vd:" . $pageViewsDays . "vc:" . md5(implode(",", $this->cityIds));
$cities = $wgMemc->get($memkey);
if (empty($cities) && !empty($wgStatsDBEnabled)) {
$startDate = date('Y-m-01', strtotime('-3 month'));
$endDate = date('Y-m-01', strtotime('now'));
$pageviews = DataMartService::getPageviewsMonthly($startDate, $endDate, $this->cityIds);
if (empty($pageviews)) {
foreach ($pageviews as $wiki_id => $wiki_data) {
#---
if ($wiki_data['SUM'] > intval($pageViews)) {
continue;
}
$this->mPageViews[$wiki_id] = $wiki_data['SUM'];
$cities[] = $wiki_id;
}
}
$wgMemc->set($memkey, $cities, 3600);
}
return $cities;
}
/**
* get popular wikis
* @param integer $wikiId
* @param integer $limit (number of users)
* @return array $popularWikis
*/
protected function getPopularWikis($limit = 50)
{
$this->wf->ProfileIn(__METHOD__);
$memKey = $this->wf->SharedMemcKey('userlogin', 'popular_wikis');
$popularWikis = $this->wg->Memc->get($memKey);
if (empty($popularWikis)) {
$popularWikis = array_keys(DataMartService::getTopWikisByPageviews(DataMartService::PERIOD_ID_MONTHLY, $limit));
if (empty($popularWikis)) {
$popularWikis[] = self::WIKIA_CITYID_COMMUNITY;
}
$this->wg->Memc->set($memKey, $popularWikis, 60 * 60 * 24);
}
$this->wf->ProfileOut(__METHOD__);
return $popularWikis;
}
public function execute()
{
global $wgExternalSharedDB, $wgWikiaDatacenter;
$this->output("Wikis migration started ... \n");
$this->time = time();
$migrated = 0;
$limit = $this->getOption('limit', -1);
$debug = $this->hasOption('debug');
$force = $this->hasOption('force');
$wikis = $this->getOption('wiki', null);
$forceAll = $this->hasOption('all');
$this->dryRun = $this->hasOption('dry-run');
$this->noDeletes = $this->hasOption('no-deletes');
$this->calculateMd5 = $this->hasOption('md5');
$this->syncSecondary = $this->hasOption('sync');
if ($this->syncSecondary) {
$force = true;
$forceAll = true;
}
$this->dc = $this->getOption('dc', 'sjc,res');
# don't migrate top 200 Wikis
$top200Wikis = array();
if (!$forceAll) {
$top200Wikis = DataMartService::getWAM200Wikis();
if (count($top200Wikis) != 200) {
$this->output("Number of Top 200 Wikis is different than 200 !\n");
exit;
}
if ($debug) {
$this->output("Top 200 Wikis: " . implode(", ", $top200Wikis) . "\n");
}
}
# don't migrate video.wikia.com & corp.wikia.com
$this->disabled_wikis = array_merge($top200Wikis, $this->disabled_wikis);
foreach ($this->disabled_wikis as $k => $v) {
$this->disabled_wikis[$k] = intval($v);
}
if ($this->syncSecondary) {
$this->disabled_wikis = array();
}
$wikiIds = null;
if ($wikis !== null) {
$wikiIds = array();
foreach (explode(',', $wikis) as $id) {
if (is_numeric($id) && $id >= 0) {
$wikiIds[] = $id;
}
}
if (count($wikiIds) == 0) {
$wikiIds = null;
}
}
$this->db = $this->getDB(DB_SLAVE, array(), $wgExternalSharedDB);
$order = $this->hasOption('reverse') ? " DESC" : "";
$res = $this->db->select(array('city_list', 'city_variables'), array('city_id', 'city_dbname'), array_merge(array('city_public' => 1), !$force ? array('cv_value is null or cv_value != "b:1;"') : array(), is_array($wikiIds) ? array('city_id' => $wikiIds) : array()), __CLASS__, array_merge(array('ORDER BY' => "city_last_timestamp{$order}, city_id{$order}"), $limit > 0 ? array('LIMIT' => $limit) : array()), array('city_variables' => array('LEFT JOIN', 'city_list.city_id = city_variables.cv_city_id and cv_variable_id = 1334')));
$this->output(sprintf("Found %d wikis in database...\n", $res->numRows()));
$this->output("Building list of wiki IDs...\n");
$removedCount = 0;
$queue = array();
while ($row = $res->fetchObject()) {
$id = intval($row->city_id);
$dbname = $row->city_dbname;
if (!in_array($id, $this->disabled_wikis)) {
$queue[$id] = $dbname;
} else {
$removedCount++;
}
}
$this->output(sprintf("Skipped %d wikis that are on blacklist...\n", $removedCount));
$this->output(sprintf("Scheduling %d wikis for migration...\n", count($queue)));
if ($this->hasOption('stats-only')) {
return;
}
$this->output("\nRun migrateImagesToSwift script \n");
$this->output("Building list of processes to run...\n");
$processes = array();
foreach ($queue as $id => $dbname) {
if ($this->calculateMd5) {
$process = $this->getMd5Process($id, $dbname);
} elseif ($this->syncSecondary) {
$process = $this->getSyncProcess($id, $dbname);
} else {
$process = $this->getMigrationProcess($id, $dbname);
}
$processes[] = $process;
}
$threads = $this->getOption('threads', self::THREADS_DEFAULT);
$threads = intval($threads);
$threadDelay = $this->getOption('delay', self::DELAY_DEFAULT);
$threadDelay = intval($threadDelay);
$this->output("Using {$threads} threads...\n");
$runner = new \Wikia\Swift\Process\Runner($processes, $threads, $threadDelay);
$runner->run();
$this->output(sprintf("\nMigrated %d Wikis in %s\n", $migrated, Wikia::timeDuration(time() - $this->time)));
$this->output("\nDone!\n");
}
protected function getTopArticles($wikiId, $lang)
{
return \WikiaDataAccess::cache(wfSharedMemcKey("CombinedSearchService", $wikiId, $lang), self::TOP_ARTICLES_CACHE_TIME, function () use($wikiId, $lang) {
$timer = Time::start(["CombinedSearchService", "getTopArticles"]);
$requestedFields = ["title", "url", "id", "score", "pageid", "lang", "wid", "article_quality_i", Utilities::field('html', $lang)];
$topArticlesMap = \DataMartService::getTopArticlesByPageview($wikiId, null, [NS_MAIN], false, self::TOP_ARTICLES_PER_WIKI + 1);
$query = " +(" . Utilities::valueForField("wid", $wikiId) . ") ";
$query .= " +( " . implode(" OR ", array_map(function ($x) {
return Utilities::valueForField("pageid", $x);
}, array_keys($topArticlesMap))) . ") ";
$query .= " +(is_main_page:false) ";
$searchConfig = new Config();
$searchConfig->setLimit(self::TOP_ARTICLES_PER_WIKI)->setQuery($query)->setPage(1)->setRequestedFields($requestedFields)->setDirectLuceneQuery(true)->setWikiId($wikiId);
$resultSet = (new Factory())->getFromConfig($searchConfig)->search();
$currentResults = $resultSet->toArray($requestedFields);
$articles = [];
foreach ($currentResults as $article) {
$articles[$article['pageid']] = $this->processArticle($article);
if (sizeof($articles) >= self::TOP_ARTICLES_PER_WIKI) {
break;
}
}
$result = [];
foreach ($topArticlesMap as $id => $a) {
if (isset($articles[$id])) {
$result[] = $articles[$id];
}
}
$timer->stop();
return $result;
});
}
public function getPageViews($cityID)
{
global $wgStatsDB, $wgStatsDBEnabled, $wgDevelEnvironment;
wfProfileIn(__METHOD__);
$today = date('Y-m-d', strtotime('-1 day'));
$pageviews = DataMartService::getPageviewsDaily($today, null, $cityID);
$views = isset($pageviews[$today]) ? $pageviews[$today] : 0;
wfProfileOut(__METHOD__);
return $views;
}
/**
* Return array of most visited articles
* Author: Inez Korczynski (inez at wikia.com)
* Author: Adrian 'ADi' Wieczorek (adi(at)wikia.com)
* @return array
*/
public static final function GetMostVisitedArticles($limit = 7, $ns = array(NS_MAIN), $fillUpMostPopular = true)
{
wfProfileIn(__METHOD__);
global $wgCityId;
$results = array();
$data = DataMartService::getTopArticlesByPageview($wgCityId, null, $ns, false, 2 * $limit);
if (!empty($data)) {
$mainPage = Title::newMainPage();
foreach ($data as $article_id => $row) {
$title = Title::newFromID($article_id);
if (is_object($title)) {
if (!$mainPage->equals($title)) {
$article = array('url' => $title->getLocalUrl(), 'text' => $title->getPrefixedText(), 'count' => $row['pageviews']);
$results[] = $article;
}
}
}
self::removeAdultPages($results);
if (!empty($results)) {
$results = array_slice($results, 0, $limit);
}
}
wfProfileOut(__METHOD__);
return $results;
}
protected function getTopWikis()
{
wfProfileIn(__METHOD__);
$key = wfMemcKey(__CLASS__, self::MEMC_VERSION, __METHOD__);
$data = $this->wg->memc->get($key, null);
if (!empty($data)) {
wfProfileOut(__METHOD__);
return $data;
}
$ids = DataMartService::getTopWikisByPageviews(DataMartService::PERIOD_ID_MONTHLY, 200);
$this->wg->memc->set($key, $ids, 86400);
wfProfileOut(__METHOD__);
return $ids;
}
public function getTopWikiArticles()
{
if ($this->wikiId == 3125 && $this->app->wg->develEnvironment) {
// mock data for devbox
return array(555118, 515985, 1000000001, 8917, 538188, 555190, 557683, 462105, 558329, 558347, 352876, 508820, 558551, 558346, 522515, 558349, 508610, 508604, 464461, 9947, 499151, 11095, 558620, 508883, 16585, 494944, 555346, 28027, 527672, 12581, 542097, 508274, 558004, 557802, 478287, 7396, 330385, 542128, 510727, 347365, 540972, 541716, 2793, 558453, 7582, 12260, 540187, 539434, 66998, 510708, 515860, 557859, 12655, 231308, 8241, 383993, 12645, 386142, 542154, 1000000005, 295154, 558059, 508034, 508121, 525634, 11092, 516886, 11420, 1881, 3126, 11945, 102675, 346620, 265165, 526114, 340164, 557784, 463607, 13012, 141183, 539437, 557807, 2051, 558103, 197844, 2502, 2519, 542106, 555347, 87058, 555348, 539432, 557785, 527578, 305450, 539436, 499717, 479686, 336449, 514548, 11021, 246281, 526287, 11093, 463600, 495341, 103097, 542052, 540180, 10996, 7991, 555349, 10391, 558466, 71619, 14889, 17924, 555350, 1000000002, 8740, 273630, 344941, 522500, 11094, 1000000000, 526588, 558436, 197894, 10608, 501630, 508541, 526338, 16527, 2022, 13134, 545200, 197863, 265238, 558060, 508401, 463459, 13250, 557655, 330668, 273256, 330653, 508421, 220998, 2345, 309380, 522504, 557808, 542303, 11131, 10606, 490365, 71603, 231298, 481029, 547547, 276156, 161089, 502996, 396534, 540703, 3369, 476459, 508286, 273550, 490298, 548270, 492123, 0, 277937, 158170, 490297, 519330, 554509, 508694, 503268, 526985, 1797, 3681, 492134, 469776, 508792, 499138, 16502, 482416, 542100, 198083, 7487, 11011, 548103, 464776, 17576, 383534, 529481, 231726, 520467);
}
$articles = DataMartService::getTopArticlesByPageview($this->wikiId, null, null, null, self::MAX_ELEM_IN_RANK);
$articleId = array();
foreach ($articles as $id => $a) {
$articleId[] = $id;
}
return $articleId;
}
/**
* Fetches page views data for a given set of articles. The data includes
* number of views for the last four time ids (data points).
*
* @param $articlesIds An array of IDs of articles to fetch views for
* @return array An array with views for the last four time ids
*/
public function getPageViewsData($articlesIds)
{
global $wgCityId;
/**
* Get pv for the last 4 Sundays
*/
$pvTimes = InsightsHelper::getLastFourTimeIds();
$pvData = [];
foreach ($pvTimes as $timeId) {
$pvData[] = DataMartService::getPageViewsForArticles($articlesIds, $timeId, $wgCityId);
}
return $pvData;
}
/**
* Checks if a given wikia is in the top 200 in terms of pageviews
* @param int $city_id
* @return boolean
*/
public function isTop200($city_id)
{
wfProfileIn(__METHOD__);
$sCacheKey = wfSharedMemcKey(__CLASS__, __METHOD__);
// Check in memcache before using DataMartService
if (!is_null($this->app->wg->Memc->get($sCacheKey))) {
$aTop200Wikis = $this->app->wg->Memc->get($sCacheKey);
} else {
$aTop200Wikis = DataMartService::getTopWikisByPageviews(DataMartService::PERIOD_ID_MONTHLY);
$this->app->wg->Memc->set($sCacheKey, $aTop200Wikis, \WikiaResponse::CACHE_LONG);
}
// city_ids are keys; return true if that one is set.
if (isset($aTop200Wikis[$city_id])) {
wfProfileOut(__METHOD__);
return true;
}
wfProfileOut(__METHOD__);
return false;
}
public function execute()
{
global $wgExternalSharedDB;
$this->output("Wikis migration started ... \n");
$this->time = time();
$migrated = 0;
$limit = $this->getOption('limit', self::DEFAULT_LIMIT);
$debug = $this->hasOption('debug');
$procs = $this->getOption('procs', self::MIGRATE_PROCS);
$force = $this->hasOption('force');
$wikis = $this->getOption('wiki', '');
# don't migrate top 200 Wikis
$top200Wikis = DataMartService::getWAM200Wikis();
if (count($top200Wikis) != 200) {
$this->output("Number of Top 200 Wikis is different than 200 !\n");
exit;
}
# don't migrate video.wikia.com & corp.wikia.com
$this->disabled_wikis = array_merge($top200Wikis, $this->disabled_wikis);
if ($debug) {
$this->output("Top 200 Wikis: " . implode(", ", $top200Wikis) . "\n");
}
$this->db = $this->getDB(DB_SLAVE, array(), $wgExternalSharedDB);
$where = ['city_public' => 1, 'city_image_migrate.city_id is null'];
if (!empty($wikis)) {
$where['city_list.city_id'] = explode(",", $wikis);
}
$join = ['city_image_migrate.city_id = city_list.city_id', 'city_image_migrate.locked is not null'];
$res = $this->db->select(['city_list', 'city_image_migrate'], ['city_list.city_id', 'city_list.city_dbname'], $where, 'MigrateImagesToSwift', ['ORDER BY' => 'city_last_timestamp, city_id', 'LIMIT' => $limit], ['city_image_migrate' => ['LEFT JOIN', $join]]);
$to_migrate = [];
$i = 0;
$x = 0;
while ($row = $res->fetchObject()) {
$this->output("\tAdd {$row->city_dbname} to migration package ... ");
if (in_array($row->city_id, $this->disabled_wikis)) {
$this->output("don't migrate it now \n");
continue;
}
$to_migrate[$row->city_id] = $row->city_dbname;
$this->output("done \n ");
$i++;
}
$this->output("\n\nRun migrateImagesToSwift script \n");
foreach ($to_migrate as $id => $dbname) {
# check how many Wikis is locked and sleep if needed
$this->isLocked($procs);
# run main migration script written by Macbre
$this->output("\tMigrate Wiki {$id}: {$dbname} ... ");
$cmd = sprintf(self::CMD, $this->getOption('conf'), $id, $force ? ' --force' : '', self::SCRIPT_PROCS, $this->makePath($dbname));
if ($debug) {
$this->output("\n\tRun cmd: {$cmd} \n");
}
global $wgMaxShellTime;
$wgMaxShellTime = 0;
$result = wfShellExec($cmd, $retval);
if ($retval) {
$this->output("Error code {$retval}: {$result} \n");
} else {
$this->output("Done in " . Wikia::timeDuration(time() - $this->time) . "\n");
}
$migrated++;
}
$this->output(sprintf("\nMigrated %d Wikis in %s\n", $migrated, Wikia::timeDuration(time() - $this->time)));
$this->output("\nDone!\n");
}
/**
* @param string $sCategoryDBKey
* @param array $mNamespace
* @param bool $limit
* @param bool $negative
* @return array
*/
public static function getMostVisited($sCategoryDBKey, $mNamespace = null, $limit = false, $negative = false)
{
global $wgCityId, $wgStatsDBEnabled;
wfProfileIn(__METHOD__);
if (empty($wgStatsDBEnabled)) {
Wikia::log(__METHOD__, ' Stats DB is disabled');
wfProfileOut(__METHOD__);
return array();
}
$where = array('cl_to' => $sCategoryDBKey);
if (!empty($mNamespace)) {
$where[] = 'page_namespace ' . ($negative ? 'NOT ' : '') . 'IN(' . implode(',', $mNamespace) . ')';
}
$dbr = wfGetDB(DB_SLAVE);
$res = $dbr->select(array('page', 'categorylinks'), array('page_id', 'cl_to'), $where, __METHOD__, array('ORDER BY' => 'page_title'), array('categorylinks' => array('INNER JOIN', 'cl_from = page_id')));
if ($dbr->numRows($res) > 0) {
Wikia::log(__METHOD__, ' Found some data in categories. Proceeding ');
$aCategoryArticles = self::tableFromResult($res);
$catKeys = array_keys($aCategoryArticles);
Wikia::log(__METHOD__, ' Searching for prepared data');
//fix BugId:33086
//use DataMart for pageviews data
//as all the other data sources are disabled/obsolete
$items = DataMartService::getTopArticlesByPageview($wgCityId, $catKeys, $mNamespace, $negative, count($catKeys));
if (is_array($items) && count($items) > 0) {
Wikia::log(__METHOD__, ' Found some data. Lets find a commmon part with categories ');
$aResult = array();
$aResultCount = 0;
$keys = array();
$reversedCatKeys = array_flip($catKeys);
$time = microtime(true);
foreach ($items as $id => $pv) {
$key = intval($id);
$keys[$key] = $key;
}
foreach ($keys as $key) {
if (isset($reversedCatKeys[$key])) {
$aResultCount++;
unset($aCategoryArticles[$key]);
$aResult[$key] = array('page_id' => $key);
if (!empty($limit) && $aResultCount >= $limit) {
self::logProcessingTime($time);
wfProfileOut(__METHOD__);
return $aResult;
}
}
}
self::logProcessingTime($time);
$ret = !empty($aResult) ? $aResult + $aCategoryArticles : $aCategoryArticles;
if (!empty($limit) && count($ret) > $limit) {
$ret = array_slice($ret, 0, $limit, true);
}
wfProfileOut(__METHOD__);
return $ret;
} else {
Wikia::log(__METHOD__, 'No data at all. Quitting.');
wfProfileOut(__METHOD__);
return array();
}
} else {
Wikia::log(__METHOD__, ' No articles in category found - quitting');
wfProfileOut(__METHOD__);
return array();
}
}
