public function addArticles(&$data) { $rows = array(); $dbw = wfGetDB(DB_MASTER); $timestamp = wfTimestampNow(); foreach ($data as $datum) { // Get user id and user text if one exists $userId = 0; $userText = ""; if (!empty($datum['userUrl'])) { $uname = ConciergeUtil::getUserNameFromUserUrl($datum['userUrl']); $u = ConciergeArtist::newFromName($uname); $uid = $u->getId(); if (!empty($uid)) { $userId = $uid; $userText = $u->getName(); } } // create the row data $t = $datum['t']; $rows[] = array('ct_page_id' => $t->getArticleId(), 'ct_page_title' => $dbw->strencode($t->getDBKey()), 'ct_catinfo' => Categoryhelper::getTitleCategoryMask($t), 'ct_categories' => implode(",", ConciergeDB::getTopLevelCategories($t)), 'ct_user_id' => $userId, 'ct_user_text' => $dbw->strencode($userText), 'ct_tag_list' => $dbw->strencode(implode(",", $datum['tags'])), 'ct_completed' => $datum['completed'], 'ct_completed_timestamp' => $timestamp, 'ct_reserved_timestamp' => $timestamp); } if (!empty($rows)) { $chunks = array_chunk($rows, 1000); foreach ($chunks as $chunk) { $sql = ConciergeUtil::makeBulkInsertStatement($chunk, 'concierge_articles'); $dbw->query($sql); } } }
$opts = array(); } $res = $dbr->select('page', array('page_namespace', 'page_title'), array('page_namespace' => NS_MAIN, 'page_is_redirect' => 0, 'page_catinfo' => 0), "init_toplevelcategories.php", $opts); $count = 0; $updates = array(); $titles = array(); while ($row = $dbr->fetchObject($res)) { $t = Title::makeTitle($row->page_namespace, $row->page_title); if (!$t) { continue; } $titles[] = $t; } // FIGURE OUT WHAT THE CATINFO COLUMN IS SUPPOSED TO BE foreach ($titles as $t) { $val = Categoryhelper::getTitleCategoryMask($t); $count++; #if ($count % 1000 == 0) { # print "Done $count\n"; #} $updates[] = "UPDATE page set page_catinfo={$val} where page_id={$t->getArticleID()};"; } // DO THE UPDATES print "doing " . sizeof($updates) . " updates\n"; $count = 0; $dbw = wfGetDB(DB_MASTER); foreach ($updates as $u) { $dbw = wfGetDB(DB_MASTER); $dbw->query($u); $count++; #if ($count % 1000 == 0) {
function wfUpdateCatInfoMask(&$article, &$user) { if ($article) { $title = $article->getTitle(); if ($title && $title->getNamespace() == NS_MAIN) { $mask = Categoryhelper::getTitleCategoryMask($title); $dbw = wfGetDB(DB_MASTER); $dbw->update('page', array('page_catinfo' => $mask), array('page_id' => $article->getID()), __METHOD__); } } return true; }
protected function insertUrls(&$aids, $langCode, $fields = array(), $override = false) { $dbw = wfGetDB(DB_MASTER); $data = array(); // Escape all the data foreach ($fields as $k => $field) { $fields[$k] = $dbw->strencode($field); } // Add title-specific info foreach ($aids as $aid) { $t = Title::newFromId($aid); $datum = $fields; $datum['ct_page_id'] = $t->getArticleId(); $datum['ct_lang_code'] = $langCode; $datum['ct_page_title'] = $dbw->strencode($t->getDBKey()); $datum['ct_catinfo'] = Categoryhelper::getTitleCategoryMask($t); $datum['ct_categories'] = implode(",", $this->getTopLevelCategories($t)); $data[] = $datum; } if (!empty($data)) { $table = $this->getWAPConfig()->getArticleTableName(); $sql = WAPUtil::makeBulkInsertStatement($data, $table, $override); $dbw->query($sql, __METHOD__); } }
protected function importBatch(&$data) { $dbw = wfGetDB(DB_MASTER); $rowPos = $this->rowPos; $langs = $this->getWAPConfig()->getSupportedLanguages(); $invalidUrls = array(); // Urls that are invalid for some reason or another $batchSize = sizeof($data); foreach ($langs as $lang) { $checkBatch = true; $tagMap = array(); // Tags to be added to babelfish $rows = array(); // The rows to be inserted into Babelfish foreach ($data as $i => $datum) { $url = $datum[$rowPos['url']]; // Special case for Chinese. Certain articles are excluded from being put into // babelfish for translation to stay on the good side of the Chinese firewalls if ($lang == 'zh' && 1 == $datum[$rowPos['zh_exclude']]) { // A bit of a hack to account for the special zh case to exclude certain // articles from babelfish. Normally any url, so long as valid (ie title // exists and not excluded) would be available for all languages in Babelfish. // For zh we'll skip the article if excluded and set a flag not to check this // batch (as the counts will be off between batchSize and valid/invalid urls) // rather than unset the data since it will be imported by other // languages in the next iteration of the foreach $checkBatch = false; } else { $state = $this->getArticleState($url, $lang); if ($state == WAPArticle::STATE_INVALID || $state == WAPArticle::STATE_EXCLUDED) { $invalidUrls[$url] = $state; unset($data[$i]); continue; } $aid = $datum[$rowPos['aid']]; $t = Title::newFromId($aid); if ($t) { // Add tags for language $row['ct_page_id'] = $t->getArticleId(); $row['ct_lang_code'] = $lang; $row['ct_page_title'] = $dbw->strencode($t->getDBKey()); $row['ct_catinfo'] = Categoryhelper::getTitleCategoryMask($t); $row['ct_categories'] = implode(",", $this->getTopLevelCategories($t)); $row['ct_rank'] = $datum[$rowPos['rank']]; $row['ct_score'] = $datum[$rowPos['score']]; $rows[] = $row; // Add tags to tag map for language $tagMap[$lang][] = $aid; } else { $invalidUrls[$url] = 'title not found'; } } } if ($checkBatch) { $this->checkBatch($batchSize, $rows, $invalidUrls); } $this->insertRows($rows); $this->addTags($tagMap, $lang); // Complete articles that are in the Babelfish DB and have been translated $this->completeTranslatedArticlesInAids($this->getAidsFromRows($rows), $lang); } return $invalidUrls; }