public function addArticles(&$data)
{
$rows = array();
$dbw = wfGetDB(DB_MASTER);
$timestamp = wfTimestampNow();
foreach ($data as $datum) {
// Get user id and user text if one exists
$userId = 0;
$userText = "";
if (!empty($datum['userUrl'])) {
$uname = ConciergeUtil::getUserNameFromUserUrl($datum['userUrl']);
$u = ConciergeArtist::newFromName($uname);
$uid = $u->getId();
if (!empty($uid)) {
$userId = $uid;
$userText = $u->getName();
}
}
// create the row data
$t = $datum['t'];
$rows[] = array('ct_page_id' => $t->getArticleId(), 'ct_page_title' => $dbw->strencode($t->getDBKey()), 'ct_catinfo' => Categoryhelper::getTitleCategoryMask($t), 'ct_categories' => implode(",", ConciergeDB::getTopLevelCategories($t)), 'ct_user_id' => $userId, 'ct_user_text' => $dbw->strencode($userText), 'ct_tag_list' => $dbw->strencode(implode(",", $datum['tags'])), 'ct_completed' => $datum['completed'], 'ct_completed_timestamp' => $timestamp, 'ct_reserved_timestamp' => $timestamp);
}
if (!empty($rows)) {
$chunks = array_chunk($rows, 1000);
foreach ($chunks as $chunk) {
$sql = ConciergeUtil::makeBulkInsertStatement($chunk, 'concierge_articles');
$dbw->query($sql);
}
}
}
$opts = array();
}
$res = $dbr->select('page', array('page_namespace', 'page_title'), array('page_namespace' => NS_MAIN, 'page_is_redirect' => 0, 'page_catinfo' => 0), "init_toplevelcategories.php", $opts);
$count = 0;
$updates = array();
$titles = array();
while ($row = $dbr->fetchObject($res)) {
$t = Title::makeTitle($row->page_namespace, $row->page_title);
if (!$t) {
continue;
}
$titles[] = $t;
}
// FIGURE OUT WHAT THE CATINFO COLUMN IS SUPPOSED TO BE
foreach ($titles as $t) {
$val = Categoryhelper::getTitleCategoryMask($t);
$count++;
#if ($count % 1000 == 0) {
# print "Done $count\n";
#}
$updates[] = "UPDATE page set page_catinfo={$val} where page_id={$t->getArticleID()};";
}
// DO THE UPDATES
print "doing " . sizeof($updates) . " updates\n";
$count = 0;
$dbw = wfGetDB(DB_MASTER);
foreach ($updates as $u) {
$dbw = wfGetDB(DB_MASTER);
$dbw->query($u);
$count++;
#if ($count % 1000 == 0) {
function wfUpdateCatInfoMask(&$article, &$user)
{
if ($article) {
$title = $article->getTitle();
if ($title && $title->getNamespace() == NS_MAIN) {
$mask = Categoryhelper::getTitleCategoryMask($title);
$dbw = wfGetDB(DB_MASTER);
$dbw->update('page', array('page_catinfo' => $mask), array('page_id' => $article->getID()), __METHOD__);
}
}
return true;
}
protected function insertUrls(&$aids, $langCode, $fields = array(), $override = false)
{
$dbw = wfGetDB(DB_MASTER);
$data = array();
// Escape all the data
foreach ($fields as $k => $field) {
$fields[$k] = $dbw->strencode($field);
}
// Add title-specific info
foreach ($aids as $aid) {
$t = Title::newFromId($aid);
$datum = $fields;
$datum['ct_page_id'] = $t->getArticleId();
$datum['ct_lang_code'] = $langCode;
$datum['ct_page_title'] = $dbw->strencode($t->getDBKey());
$datum['ct_catinfo'] = Categoryhelper::getTitleCategoryMask($t);
$datum['ct_categories'] = implode(",", $this->getTopLevelCategories($t));
$data[] = $datum;
}
if (!empty($data)) {
$table = $this->getWAPConfig()->getArticleTableName();
$sql = WAPUtil::makeBulkInsertStatement($data, $table, $override);
$dbw->query($sql, __METHOD__);
}
}
protected function importBatch(&$data)
{
$dbw = wfGetDB(DB_MASTER);
$rowPos = $this->rowPos;
$langs = $this->getWAPConfig()->getSupportedLanguages();
$invalidUrls = array();
// Urls that are invalid for some reason or another
$batchSize = sizeof($data);
foreach ($langs as $lang) {
$checkBatch = true;
$tagMap = array();
// Tags to be added to babelfish
$rows = array();
// The rows to be inserted into Babelfish
foreach ($data as $i => $datum) {
$url = $datum[$rowPos['url']];
// Special case for Chinese. Certain articles are excluded from being put into
// babelfish for translation to stay on the good side of the Chinese firewalls
if ($lang == 'zh' && 1 == $datum[$rowPos['zh_exclude']]) {
// A bit of a hack to account for the special zh case to exclude certain
// articles from babelfish. Normally any url, so long as valid (ie title
// exists and not excluded) would be available for all languages in Babelfish.
// For zh we'll skip the article if excluded and set a flag not to check this
// batch (as the counts will be off between batchSize and valid/invalid urls)
// rather than unset the data since it will be imported by other
// languages in the next iteration of the foreach
$checkBatch = false;
} else {
$state = $this->getArticleState($url, $lang);
if ($state == WAPArticle::STATE_INVALID || $state == WAPArticle::STATE_EXCLUDED) {
$invalidUrls[$url] = $state;
unset($data[$i]);
continue;
}
$aid = $datum[$rowPos['aid']];
$t = Title::newFromId($aid);
if ($t) {
// Add tags for language
$row['ct_page_id'] = $t->getArticleId();
$row['ct_lang_code'] = $lang;
$row['ct_page_title'] = $dbw->strencode($t->getDBKey());
$row['ct_catinfo'] = Categoryhelper::getTitleCategoryMask($t);
$row['ct_categories'] = implode(",", $this->getTopLevelCategories($t));
$row['ct_rank'] = $datum[$rowPos['rank']];
$row['ct_score'] = $datum[$rowPos['score']];
$rows[] = $row;
// Add tags to tag map for language
$tagMap[$lang][] = $aid;
} else {
$invalidUrls[$url] = 'title not found';
}
}
}
if ($checkBatch) {
$this->checkBatch($batchSize, $rows, $invalidUrls);
}
$this->insertRows($rows);
$this->addTags($tagMap, $lang);
// Complete articles that are in the Babelfish DB and have been translated
$this->completeTranslatedArticlesInAids($this->getAidsFromRows($rows), $lang);
}
return $invalidUrls;
}
