/**
* Get language and article info
*/
function execute()
{
$params = $this->extractRequestParams();
$command = $params['subcmd'];
$result = $this->getResult();
$module = $this->getModuleName();
switch ($command) {
case 'article':
if (!isset($params['page_id']) || !isset($params['language_code'])) {
$error = "pageId or lang parameters not set";
}
$pageId = $params['page_id'];
$lang = $params['language_code'];
$dbr = wfGetDB(DB_SLAVE);
$t = new TitusDB();
$sql = "select * from titus_intl where ti_page_id=" . $dbr->addQuotes($pageId) . " AND ti_language_code=" . $dbr->addQuotes($lang);
$res = $t->performTitusQuery($sql);
$found = false;
foreach ($res as $row) {
$result->addValue(null, $module, get_object_vars($row));
$found = true;
}
if (!$found) {
$error = "No data for article found";
}
break;
default:
$error = "Command " . $command . " not found";
}
if ($error) {
$result->addValue(null, $module, array('error' => $error));
}
}
public function insertDaysData($datestamp = null)
{
$articles = array();
$this->batchArray = array();
$this->batchCount = 0;
echo "Starting " . __METHOD__ . " at " . wfTimestamp(TS_MW) . "\n";
$beginTime = microtime(true);
$titus = new TitusDB();
if ($datestamp == null) {
$rows = $titus->getRecords();
} else {
$rows = $titus->getOldRecords($datestamp);
}
foreach ($rows as $row) {
$newRow = (array) $row;
$newRow['page_id_language'] = $newRow['ti_language_code'] . $newRow['ti_page_id'];
//need this to actually be an integer for DynamoDB
$articles[] = $newRow;
}
echo "Putting " . count($articles) . " rows into dynamodb\n";
$this->batchArray[] = array();
$this->batchCount++;
foreach ($articles as $articleData) {
$this->addDataToBatch($articleData, "PutRequest");
}
echo "Getting ready to process " . $this->batchCount . " batches\n\n";
$errorCount = 0;
for ($i = 0; $i < $this->batchCount; $i++) {
$startTime = microtime(true);
$this->processBatch($this->batchArray[$i], "PutRequest", $errorCount);
$finishTime = microtime(true);
echo "batch #{$i} of {$this->batchCount} " . ($finishTime - $startTime) . "s, ";
if ($errorCount >= $this->maxErrors) {
echo "Too many errors found. Stopping this run.\n\n";
mail('*****@*****.**', 'Dynamo Quitting', 'Quitting Dynamo run...too many errors.');
break;
}
while ($finishTime - $startTime < 1) {
//need to wait till a second is up.
$finishTime = microtime(true);
}
$currentTime = microtime(true);
if ($currentTime - $beginTime > $this->maxTime) {
echo "Taking too long. Stopping this run.\n\n";
mail('*****@*****.**', 'Dynamo Quitting', 'Quitting Dynamo run...taking too long.');
break;
}
}
echo "\nFinished " . __METHOD__ . " at " . wfTimestamp(TS_MW) . " for a total time of " . ($currentTime - $beginTime) . " seconds.\n";
}
<?php
require_once 'commandLine.inc';
require_once "{$IP}/extensions/wikihow/titus/Titus.class.php";
require_once "{$IP}/extensions/wikihow/DatabaseHelper.class.php";
$dbw = DatabaseBase::factory('mysql');
$dbw->open(TITUS_DB_HOST, WH_DATABASE_MAINTENANCE_USER, WH_DATABASE_MAINTENANCE_PASSWORD, TitusDB::getDBName());
$rows = DatabaseHelper::batchSelect('titus_intl', 'ti_page_id', array('ti_language_code' => $wgLanguageCode), __METHOD__, array(), 2000, $dbw);
$deletedPageIds = array();
foreach ($rows as $row) {
$t = Title::newFromId($row->ti_page_id);
if (!($t && $t->exists() && $t->getNamespace() == NS_MAIN)) {
$deletedPageIds[] = $row->ti_page_id;
}
}
$chunks = array_chunk($deletedPageIds, 500);
foreach ($chunks as $chunk) {
$aids = "(" . implode(",", $chunk) . ")";
$sql = "DELETE FROM titus_intl where ti_page_id IN {$aids} AND ti_language_code='{$wgLanguageCode}'";
var_dump($sql);
$dbw->query($sql);
}
<?php
require_once 'commandLine.inc';
require_once "{$IP}/extensions/wikihow/titus/Titus.class.php";
$titus = new TitusDB(true);
$allStats['PageId'] = 1;
$allStats['Title'] = 1;
$allStats['Timestamp'] = 1;
$allStats['LanguageCode'] = 1;
$allStats['BabelfishData'] = 1;
$titus->calcStatsForAllPages($allStats);
<?php
require_once 'commandLine.inc';
require_once "{$IP}/extensions/wikihow/titus/Titus.class.php";
$statsToCalc = TitusConfig::getDailyEditStats();
$titus = new TitusDB(true);
/*
$titus->calcStatsForAllPages($statsToCalc);*/
$ids = array(163806);
$titus->calcStatsForPageIds($statsToCalc, $ids);
/*$dailyEditStats = TitusConfig::getDailyEditStats();
$titus->calcLatestEdits($dailyEditStats);*/
<?php
require_once 'commandLine.inc';
require_once "{$IP}/extensions/wikihow/titus/Titus.class.php";
$dbw = DatabaseBase::factory('mysql');
$dbw->open(TITUS_DB_HOST, WH_DATABASE_MAINTENANCE_USER, WH_DATABASE_MAINTENANCE_PASSWORD, TitusDB::getDBName());
$sql = "delete t.* from " . TitusDB::getDBName() . ".titus_intl t left join " . Misc::getLangDB($wgLanguageCode) . ".page p on ti_page_id = p.page_id WHERE p.page_is_redirect=1 AND ti_language_code='{$wgLanguageCode}'";
$dbw->query($sql);
print $dbw->affectedRows() . " rows deleted in {$wgLanguageCode} ";
public function repairTitus($lookBack = 1)
{
$titus = new TitusDB(true);
$dailyEditStats = TitusConfig::getDailyEditStats();
$titus->calcLatestEdits($dailyEditStats, $lookBack);
$nightlyStats = TitusConfig::getNightlyStats();
$titus->calcStatsForAllPages($nightlyStats);
}
<?php
require_once 'commandLine.inc';
require_once "{$IP}/extensions/wikihow/titus/Titus.class.php";
$titus = new TitusDB(true);
$stats = TitusConfig::getDailyEditStats();
$stats['RobotPolicy'] = 0;
$stats['Social'] = 0;
$dbr = new Database(TITUS_DB_HOST, WH_DATABASE_MAINTENANCE_USER, WH_DATABASE_MAINTENANCE_PASSWORD, TitusDB::TITUS_DB_NAME);
$sql = "select ti_page_id from titus_intl where ti_language_code='{$wgLanguageCode}' AND ti_num_steps=0";
$res = $dbr->query($sql);
$ids = array();
foreach ($res as $row) {
$ids[] = $row->ti_page_id;
}
$batch = array();
foreach ($ids as $id) {
$batch[] = $id;
if (sizeof($batch) == 999) {
$titus->calcStatsForPageIds($stats, $batch);
$batch = array();
}
}
$titus->calcStatsForPageIds($stats, $batch);
<?php
require_once 'commandLine.inc';
require_once "{$IP}/extensions/wikihow/titus/Titus.class.php";
$statsToCalc = TitusConfig::getBasicStats();
$statsToCalc['Title'] = 0;
$statsToCalc['Accuracy'] = 1;
$statsToCalc['Templates'] = 1;
$titus = new TitusDB(true);
/*
$titus->calcStatsForAllPages($statsToCalc);
$ids = array(1132186,1126217,722887,510006);
$titus->calcStatsForPageIds($statsToCalc, $ids);
*/
$dailyEditStats = TitusConfig::getDailyEditStats();
$titus->calcLatestEdits($dailyEditStats);
<?php
require_once 'commandLine.inc';
require_once "{$IP}/extensions/wikihow/titus/Titus.class.php";
require_once "{$IP}/extensions/wikihow/DatabaseHelper.class.php";
global $wgLanguageCode;
$sql = "select rev_page, max(rev_timestamp) as last_revision, min(rev_timestamp) as first_revision_after, ti_last_patrolled_edit_timestamp as last_patrolled from titusdb.titus_intl ti join " . Misc::getLangDB($wgLanguageCode) . ".revision on ti_page_id = rev_page WHERE rev_timestamp > ti_last_patrolled_edit_timestamp AND ti_language_code='{$wgLanguageCode}' AND not (rev_comment like '%move%') group by rev_page";
$dbr = new Database(TITUS_DB_HOST, WH_DATABASE_MAINTENANCE_USER, WH_DATABASE_MAINTENANCE_PASSWORD, TitusDB::getDBName());
$res = $dbr->query($sql);
$rows = array();
foreach ($res as $row) {
$rows[] = $row;
}
if (sizeof($rows) > 0) {
$now = wfTimestampNow();
$msg = "<table><thead><tr><td>Title</td><td>Page Id</td><td>Latest Revision</td><td>First Revision After Patrolled</td><td>Last Patrolled Revision</td></tr></thead>\n<tbody>\n";
foreach ($rows as $row) {
$lts = wfTimestamp(TS_MW, $row->first_revision_after);
if ($lts < $now - 24 * 60 * 60) {
$t = Title::newFromId($row->rev_page);
$msg .= "<tr><td>" . $t->getText() . "</td><td>" . $row->rev_page . "</td><td>" . wfTimestamp(TS_DB, $row->last_revision) . "</td><td>" . wfTimestamp(TS_DB, $row->first_revision_after) . "</td><td>" . wfTimestamp(TS_DB, $row->last_patrolled) . "</td></tr>\n";
}
}
$msg .= "</tbody></table>\n";
if ($msg != "") {
$msg = "<p>The following pages on {$wgLanguageCode} have been modified over 24 hours ago, but aren't showing up in Titus</p>\n" . $msg;
print $msg;
$to = new MailAddress("*****@*****.**");
$from = new MailAddress("*****@*****.**");
$subject = "Out of date fields in titus:\n";
$content_type = "text/html; charset={$wgOutputEncoding}";
private function getSpreadsheet(&$dbr)
{
global $wgLanguageCode;
print "Getting ratings spreadsheet\n";
try {
$gs = new GoogleSpreadsheet();
$gs->login(WH_TITUS_GOOGLE_LOGIN, WH_TITUS_GOOGLE_PW);
$cols = $gs->getCols(WH_TITUS_EDITOR_GOOGLE_DOC, 1, 3, 2);
$ids = array();
$badDates = 0;
foreach ($cols as $col) {
if (is_numeric($col[0])) {
$output = array($this->fixDate($col[1]), $col[2]);
if ($output[1] == NULL) {
$badDates++;
}
if (isset($this->_kwl[$col[0]])) {
$this->reportError("Duplicate entry for article " . $col[0]);
}
$this->_kwl[$col[0]] = $output;
$ids[] = $col[0];
}
}
if ($badDates > 100) {
$this->reportError("Unable to parse over 100 dates in spreadsheet");
$this->_gotSpreadsheet = true;
$this->_badSpreadsheet = true;
return;
}
if (sizeof($ids) < 1000) {
$this->reportError("Less than 1000 ratings in ratings spreadsheet found");
$this->_gotSpreadsheet = true;
$this->_badSpreadsheet = true;
return;
}
$this->checkForRedirects($dbr, $ids);
$this->checkForMissing($dbr, $ids);
$query = "select ti_page_id, ti_last_fellow_edit, ti_last_fellow_edit_timestamp FROM " . TitusDB::getDBName() . "." . TitusDB::TITUS_INTL_TABLE_NAME . " WHERE ti_language_code=" . $dbr->addquotes($wgLanguageCode);
$res = $dbr->query($query, __METHOD__);
$pageIds = array();
foreach ($res as $row) {
if (isset($this->_kwl[$row->ti_page_id])) {
if ($this->_kwl[$row->ti_page_id][0] != $row->ti_last_fellow_edit_timestamp || $this->_kwl[$row->ti_page_id][1] != $row->ti_last_fellow_edit) {
$pageIds[] = $row->ti_page_id;
}
} else {
if ($row->ti_last_fellow_edit_timestamp != NULL && $row->ti_last_fellow_edit_timestamp != "" || $row->ti_last_fellow_edit != NULL && $row->ti_last_fellow_edit != "") {
$pageIds[] = $row->ti_page_id;
}
}
}
$this->_ids = $pageIds;
$this->_gotSpreadsheet = true;
$this->_badSpreadsheet = false;
} catch (Exception $e) {
$this->_gotSpreadsheet = true;
$this->_badSpreadsheet = true;
$this->reportError("Problem fetching spreadsheet :" . $e->getMessage());
}
}