예제 #1
0
 /**
  * Get language and article info 
  */
 function execute()
 {
     $params = $this->extractRequestParams();
     $command = $params['subcmd'];
     $result = $this->getResult();
     $module = $this->getModuleName();
     switch ($command) {
         case 'article':
             if (!isset($params['page_id']) || !isset($params['language_code'])) {
                 $error = "pageId or lang parameters not set";
             }
             $pageId = $params['page_id'];
             $lang = $params['language_code'];
             $dbr = wfGetDB(DB_SLAVE);
             $t = new TitusDB();
             $sql = "select * from titus_intl where ti_page_id=" . $dbr->addQuotes($pageId) . " AND ti_language_code=" . $dbr->addQuotes($lang);
             $res = $t->performTitusQuery($sql);
             $found = false;
             foreach ($res as $row) {
                 $result->addValue(null, $module, get_object_vars($row));
                 $found = true;
             }
             if (!$found) {
                 $error = "No data for article found";
             }
             break;
         default:
             $error = "Command " . $command . " not found";
     }
     if ($error) {
         $result->addValue(null, $module, array('error' => $error));
     }
 }
예제 #2
0
 public function insertDaysData($datestamp = null)
 {
     $articles = array();
     $this->batchArray = array();
     $this->batchCount = 0;
     echo "Starting " . __METHOD__ . " at " . wfTimestamp(TS_MW) . "\n";
     $beginTime = microtime(true);
     $titus = new TitusDB();
     if ($datestamp == null) {
         $rows = $titus->getRecords();
     } else {
         $rows = $titus->getOldRecords($datestamp);
     }
     foreach ($rows as $row) {
         $newRow = (array) $row;
         $newRow['page_id_language'] = $newRow['ti_language_code'] . $newRow['ti_page_id'];
         //need this to actually be an integer for DynamoDB
         $articles[] = $newRow;
     }
     echo "Putting " . count($articles) . " rows into dynamodb\n";
     $this->batchArray[] = array();
     $this->batchCount++;
     foreach ($articles as $articleData) {
         $this->addDataToBatch($articleData, "PutRequest");
     }
     echo "Getting ready to process " . $this->batchCount . " batches\n\n";
     $errorCount = 0;
     for ($i = 0; $i < $this->batchCount; $i++) {
         $startTime = microtime(true);
         $this->processBatch($this->batchArray[$i], "PutRequest", $errorCount);
         $finishTime = microtime(true);
         echo "batch #{$i} of {$this->batchCount} " . ($finishTime - $startTime) . "s, ";
         if ($errorCount >= $this->maxErrors) {
             echo "Too many errors found. Stopping this run.\n\n";
             mail('*****@*****.**', 'Dynamo Quitting', 'Quitting Dynamo run...too many errors.');
             break;
         }
         while ($finishTime - $startTime < 1) {
             //need to wait till a second is up.
             $finishTime = microtime(true);
         }
         $currentTime = microtime(true);
         if ($currentTime - $beginTime > $this->maxTime) {
             echo "Taking too long. Stopping this run.\n\n";
             mail('*****@*****.**', 'Dynamo Quitting', 'Quitting Dynamo run...taking too long.');
             break;
         }
     }
     echo "\nFinished " . __METHOD__ . " at " . wfTimestamp(TS_MW) . " for a total time of " . ($currentTime - $beginTime) . " seconds.\n";
 }
<?php

require_once 'commandLine.inc';
require_once "{$IP}/extensions/wikihow/titus/Titus.class.php";
require_once "{$IP}/extensions/wikihow/DatabaseHelper.class.php";
$dbw = DatabaseBase::factory('mysql');
$dbw->open(TITUS_DB_HOST, WH_DATABASE_MAINTENANCE_USER, WH_DATABASE_MAINTENANCE_PASSWORD, TitusDB::getDBName());
$rows = DatabaseHelper::batchSelect('titus_intl', 'ti_page_id', array('ti_language_code' => $wgLanguageCode), __METHOD__, array(), 2000, $dbw);
$deletedPageIds = array();
foreach ($rows as $row) {
    $t = Title::newFromId($row->ti_page_id);
    if (!($t && $t->exists() && $t->getNamespace() == NS_MAIN)) {
        $deletedPageIds[] = $row->ti_page_id;
    }
}
$chunks = array_chunk($deletedPageIds, 500);
foreach ($chunks as $chunk) {
    $aids = "(" . implode(",", $chunk) . ")";
    $sql = "DELETE FROM titus_intl where ti_page_id IN {$aids} AND ti_language_code='{$wgLanguageCode}'";
    var_dump($sql);
    $dbw->query($sql);
}
<?php

require_once 'commandLine.inc';
require_once "{$IP}/extensions/wikihow/titus/Titus.class.php";
$titus = new TitusDB(true);
$allStats['PageId'] = 1;
$allStats['Title'] = 1;
$allStats['Timestamp'] = 1;
$allStats['LanguageCode'] = 1;
$allStats['BabelfishData'] = 1;
$titus->calcStatsForAllPages($allStats);
예제 #5
0
<?php

require_once 'commandLine.inc';
require_once "{$IP}/extensions/wikihow/titus/Titus.class.php";
$statsToCalc = TitusConfig::getDailyEditStats();
$titus = new TitusDB(true);
/*
$titus->calcStatsForAllPages($statsToCalc);*/
$ids = array(163806);
$titus->calcStatsForPageIds($statsToCalc, $ids);
/*$dailyEditStats = TitusConfig::getDailyEditStats();
$titus->calcLatestEdits($dailyEditStats);*/
<?php

require_once 'commandLine.inc';
require_once "{$IP}/extensions/wikihow/titus/Titus.class.php";
$dbw = DatabaseBase::factory('mysql');
$dbw->open(TITUS_DB_HOST, WH_DATABASE_MAINTENANCE_USER, WH_DATABASE_MAINTENANCE_PASSWORD, TitusDB::getDBName());
$sql = "delete t.* from " . TitusDB::getDBName() . ".titus_intl t left join " . Misc::getLangDB($wgLanguageCode) . ".page p on ti_page_id = p.page_id WHERE p.page_is_redirect=1 AND ti_language_code='{$wgLanguageCode}'";
$dbw->query($sql);
print $dbw->affectedRows() . " rows deleted in {$wgLanguageCode} ";
예제 #7
0
 public function repairTitus($lookBack = 1)
 {
     $titus = new TitusDB(true);
     $dailyEditStats = TitusConfig::getDailyEditStats();
     $titus->calcLatestEdits($dailyEditStats, $lookBack);
     $nightlyStats = TitusConfig::getNightlyStats();
     $titus->calcStatsForAllPages($nightlyStats);
 }
예제 #8
0
<?php

require_once 'commandLine.inc';
require_once "{$IP}/extensions/wikihow/titus/Titus.class.php";
$titus = new TitusDB(true);
$stats = TitusConfig::getDailyEditStats();
$stats['RobotPolicy'] = 0;
$stats['Social'] = 0;
$dbr = new Database(TITUS_DB_HOST, WH_DATABASE_MAINTENANCE_USER, WH_DATABASE_MAINTENANCE_PASSWORD, TitusDB::TITUS_DB_NAME);
$sql = "select ti_page_id from titus_intl where ti_language_code='{$wgLanguageCode}' AND ti_num_steps=0";
$res = $dbr->query($sql);
$ids = array();
foreach ($res as $row) {
    $ids[] = $row->ti_page_id;
}
$batch = array();
foreach ($ids as $id) {
    $batch[] = $id;
    if (sizeof($batch) == 999) {
        $titus->calcStatsForPageIds($stats, $batch);
        $batch = array();
    }
}
$titus->calcStatsForPageIds($stats, $batch);
예제 #9
0
파일: titus.php 프로젝트: ErdemA/wikihow
<?php

require_once 'commandLine.inc';
require_once "{$IP}/extensions/wikihow/titus/Titus.class.php";
$statsToCalc = TitusConfig::getBasicStats();
$statsToCalc['Title'] = 0;
$statsToCalc['Accuracy'] = 1;
$statsToCalc['Templates'] = 1;
$titus = new TitusDB(true);
/*
$titus->calcStatsForAllPages($statsToCalc);
$ids = array(1132186,1126217,722887,510006);
$titus->calcStatsForPageIds($statsToCalc, $ids);
*/
$dailyEditStats = TitusConfig::getDailyEditStats();
$titus->calcLatestEdits($dailyEditStats);
예제 #10
0
<?php

require_once 'commandLine.inc';
require_once "{$IP}/extensions/wikihow/titus/Titus.class.php";
require_once "{$IP}/extensions/wikihow/DatabaseHelper.class.php";
global $wgLanguageCode;
$sql = "select rev_page, max(rev_timestamp) as last_revision, min(rev_timestamp) as first_revision_after, ti_last_patrolled_edit_timestamp as last_patrolled from titusdb.titus_intl ti join " . Misc::getLangDB($wgLanguageCode) . ".revision on ti_page_id = rev_page WHERE rev_timestamp > ti_last_patrolled_edit_timestamp  AND ti_language_code='{$wgLanguageCode}' AND not (rev_comment like '%move%') group by rev_page";
$dbr = new Database(TITUS_DB_HOST, WH_DATABASE_MAINTENANCE_USER, WH_DATABASE_MAINTENANCE_PASSWORD, TitusDB::getDBName());
$res = $dbr->query($sql);
$rows = array();
foreach ($res as $row) {
    $rows[] = $row;
}
if (sizeof($rows) > 0) {
    $now = wfTimestampNow();
    $msg = "<table><thead><tr><td>Title</td><td>Page Id</td><td>Latest Revision</td><td>First Revision After Patrolled</td><td>Last Patrolled Revision</td></tr></thead>\n<tbody>\n";
    foreach ($rows as $row) {
        $lts = wfTimestamp(TS_MW, $row->first_revision_after);
        if ($lts < $now - 24 * 60 * 60) {
            $t = Title::newFromId($row->rev_page);
            $msg .= "<tr><td>" . $t->getText() . "</td><td>" . $row->rev_page . "</td><td>" . wfTimestamp(TS_DB, $row->last_revision) . "</td><td>" . wfTimestamp(TS_DB, $row->first_revision_after) . "</td><td>" . wfTimestamp(TS_DB, $row->last_patrolled) . "</td></tr>\n";
        }
    }
    $msg .= "</tbody></table>\n";
    if ($msg != "") {
        $msg = "<p>The following pages on {$wgLanguageCode} have been modified over 24 hours ago, but aren't showing up in Titus</p>\n" . $msg;
        print $msg;
        $to = new MailAddress("*****@*****.**");
        $from = new MailAddress("*****@*****.**");
        $subject = "Out of date fields in titus:\n";
        $content_type = "text/html; charset={$wgOutputEncoding}";
예제 #11
0
 private function getSpreadsheet(&$dbr)
 {
     global $wgLanguageCode;
     print "Getting ratings spreadsheet\n";
     try {
         $gs = new GoogleSpreadsheet();
         $gs->login(WH_TITUS_GOOGLE_LOGIN, WH_TITUS_GOOGLE_PW);
         $cols = $gs->getCols(WH_TITUS_EDITOR_GOOGLE_DOC, 1, 3, 2);
         $ids = array();
         $badDates = 0;
         foreach ($cols as $col) {
             if (is_numeric($col[0])) {
                 $output = array($this->fixDate($col[1]), $col[2]);
                 if ($output[1] == NULL) {
                     $badDates++;
                 }
                 if (isset($this->_kwl[$col[0]])) {
                     $this->reportError("Duplicate entry for article " . $col[0]);
                 }
                 $this->_kwl[$col[0]] = $output;
                 $ids[] = $col[0];
             }
         }
         if ($badDates > 100) {
             $this->reportError("Unable to parse over 100 dates in spreadsheet");
             $this->_gotSpreadsheet = true;
             $this->_badSpreadsheet = true;
             return;
         }
         if (sizeof($ids) < 1000) {
             $this->reportError("Less than 1000 ratings in ratings spreadsheet found");
             $this->_gotSpreadsheet = true;
             $this->_badSpreadsheet = true;
             return;
         }
         $this->checkForRedirects($dbr, $ids);
         $this->checkForMissing($dbr, $ids);
         $query = "select ti_page_id, ti_last_fellow_edit, ti_last_fellow_edit_timestamp FROM " . TitusDB::getDBName() . "." . TitusDB::TITUS_INTL_TABLE_NAME . " WHERE ti_language_code=" . $dbr->addquotes($wgLanguageCode);
         $res = $dbr->query($query, __METHOD__);
         $pageIds = array();
         foreach ($res as $row) {
             if (isset($this->_kwl[$row->ti_page_id])) {
                 if ($this->_kwl[$row->ti_page_id][0] != $row->ti_last_fellow_edit_timestamp || $this->_kwl[$row->ti_page_id][1] != $row->ti_last_fellow_edit) {
                     $pageIds[] = $row->ti_page_id;
                 }
             } else {
                 if ($row->ti_last_fellow_edit_timestamp != NULL && $row->ti_last_fellow_edit_timestamp != "" || $row->ti_last_fellow_edit != NULL && $row->ti_last_fellow_edit != "") {
                     $pageIds[] = $row->ti_page_id;
                 }
             }
         }
         $this->_ids = $pageIds;
         $this->_gotSpreadsheet = true;
         $this->_badSpreadsheet = false;
     } catch (Exception $e) {
         $this->_gotSpreadsheet = true;
         $this->_badSpreadsheet = true;
         $this->reportError("Problem fetching spreadsheet :" . $e->getMessage());
     }
 }