/** * Grab all the articles */ private static function cycleThroughAllArticles() { $csv = fopen(self::CSV_FILE, 'a'); if (!$csv) { print "error: opening a file\n"; exit; } fputcsv($csv, array('url', 'type', 'old', 'new'), chr(9)); $res = DatabaseHelper::batchSelect('page', array('page_title'), array('page_namespace' => NS_MAIN, 'page_is_redirect' => 0)); print 'articles: ' . count($res) . "\n"; $count = 0; $num = 1; foreach ($res as $row) { //print $num.' - '; $title = Title::newFromRow($row); if ($title) { //print $title->getDBKey(); if (self::processSubheaders($title, $csv)) { $count++; } if ($count >= 5000) { break; } } //print "\n"; $num++; } print "\nchanged: " . $count . "\n"; return; }
function addAll404Redirects() { $rows = DatabaseHelper::batchSelect('page', array('page_title', 'page_id'), array('page_namespace' => NS_MAIN, 'page_is_redirect' => 0), __METHOD__); foreach ($rows as $row) { $title = Title::newFromDBkey($row->page_title); if ($title) { Misc::modify404Redirect($row->page_id, $title); } } }
/** * Grab all the articles */ private static function cycleThroughAllArticles() { $res = DatabaseHelper::batchSelect('page', array('page_title'), array('page_namespace' => NS_MAIN, 'page_is_redirect' => 0), __METHOD__, array('LIMIT' => self::CHUNK_LIMIT)); print 'articles: ' . count($res) . "\n"; $count = 0; $num = 1; foreach ($res as $row) { //print $num.' - '; $title = Title::newFromRow($row); if ($title) { //print $title->getDBKey(); if (self::processSubheaders($title)) { $count++; } } //print "\n"; $num++; } //print "\nchanged: ".$count."\n"; print "\ntotal: " . $count . "\n"; return; }
} $sql .= "('" . $row->{$ratingTool->getTablePrefix() . 'page'} . "', '{$row->R}', '{$row->C}')"; $count++; if ($count % $insertMax == 0) { $dbw->query($sql, __FILE__); $sql = $sqlStart; } } } if ($sql != $sqlStart) { $dbw->query($sql, __FILE__); } //////////////////////////////////// //now refresh the SAMPLE ratings $ratingTool = new RatingSample(); $res = DatabaseHelper::batchSelect($ratingTool->getTableName(), array('*'), array('rats_isdeleted' => 0), __FILE__); $newStats = array(); $deletedPages = array(); foreach ($res as $row) { if ($row->ratsisdeleted != 0) { } else { if ($newStats[$row->rats_page] == null) { $newStats[$row->rats_page] = array("count" => 0, "ratingCount" => 0); } $newStats[$row->rats_page]["count"]++; $newStats[$row->rats_page]["ratingCount"] += $row->rats_rating; } } //dump the table $dbw->query("delete from {$ratingTool->getLowTableName()}", __FILE__); $sqlStart = "INSERT into {$ratingTool->getLowTableName()} ({$ratingTool->getLowTablePrefix()}page, {$ratingTool->getLowTablePrefix()}avg, {$ratingTool->getLowTablePrefix()}count) VALUES ";
<?php require_once 'commandLine.inc'; require_once 'EditPageWrapper.php'; $rows = DatabaseHelper::batchSelect('page', array('page_id', 'page_title', 'page_is_redirect'), array('page_namespace' => NS_MAIN, 'page_is_redirect' => 0), __METHOD__); foreach ($rows as $row) { $title = Title::newFromDBkey($row->page_title); if (!$title) { continue; } $text = $title->getText(); $formatted = EditPageWrapper::formatTitle($text); if ($formatted != $text && strlen($formatted) != strlen($text)) { $redir = $row->page_is_redirect ? ' (redirect)' : ''; print "{$title}{$redir} ->\n{$formatted}\n"; } }
function moveCaps() { $dbr = wfGetDB(DB_SLAVE); $dbw = wfGetDB(DB_MASTER); $words = DatabaseHelper::batchSelect(wikiHowDictionary::CAPS_TABLE, array('*')); foreach ($words as $word) { $dbw->insert(wikiHowDictionary::WHITELIST_TABLE, array(wikiHowDictionary::WORD_FIELD => $word->sc_word, "sw_user" => $word->sc_user, "sw_active" => "1")); // sleep for 0.5s usleep(500000); } }
<?php require_once 'commandLine.inc'; $res = DatabaseHelper::batchSelect("nfd", array('nfd_page'), array('nfd_status' => 0), __FILE__, array("GROUP BY" => 'nfd_page')); foreach ($res as $result) { $title = Title::newFromID($result->nfd_page); if (!$title) { NFDProcessor::markPreviousAsInactive($result->nfd_page); echo "Removing {$result->nfd_page}\n"; usleep(500000); } }
$methodTitle = "A method title"; //$methodSteps = "#12. 12 12 12 12 12 12 12 12 12\n#34$"; $methodSteps = "#ass. IS. A. GOOD. THING.\n#AND ANOTHER. THING. IS. REALLY. GOOD. TO. SAY. IN. A CASE OF SOMETHING\n"; $title = Title::newFromText("Kiss"); $result = MethodGuardian::checkContent($title, $methodTitle, $methodSteps, true); if ($result) { echo "GOOD\n\n"; } else { echo "BAD\n\n"; } break; case "delete": $delete = true; $dbw = wfGetDB(DB_MASTER); case "all": $res = DatabaseHelper::batchSelect(MethodGuardian::TABLE_NAME, array("*"), array("ama_patrolled" => 0)); $rows = array(); foreach ($res as $row) { $rows[] = $row; } $total = 0; $bad = 0; foreach ($rows as $row) { $title = Title::newFromID($row->ama_page); $result = MethodGuardian::checkContent($title, $row->ama_method, $row->ama_steps, true); if (!$result) { if ($title) { echo "ARTICLE: " . $title->getText() . "\n"; } echo "METHOD NAME: " . $row->ama_method . "\nSTEPS: " . $row->ama_steps . "\n\n\n"; if ($delete) {
<?php require_once 'commandLine.inc'; require_once "{$IP}/extensions/wikihow/DatabaseHelper.class.php"; require_once "{$IP}/extensions/wikihow/authors/AuthorEmailNotification.php"; $day = intval(date("j")); if ($day < 1 || $day > 6) { echo "We don't send emails on the {$day} day of the month. Exiting.\n"; exit; } $startTime = microtime(true); $users = DatabaseHelper::batchSelect('user', array('user_id'), array(), __FILE__); $dbr = wfGetDB(DB_SLAVE); //TESTING CODE /*$res = $dbr->select('user', array('user_id'), array('user_name' => 'Bsteudel'), __FILE__, array("LIMIT" => 100)); $users = array(); foreach($res as $user) $users[] = $user; */ $todayUnix = wfTimestamp(TS_UNIX); $minUnix = strtotime("-1 month", $todayUnix); $minDate = wfTimestamp(TS_MW, $minUnix); //$testDate = wfTimestamp(TS_MW, strtotime("-2 months", $todayUnix)); echo "looking for data from {$minDate} and forward\n"; $emailCount = 0; $userNames = ""; foreach ($users as $userInfo) { $user = User::newFromId($userInfo->user_id); $name = $user->getName(); $firstLetter = strtolower(substr($name, 0, 1)); $omit = false;
* 2) The ip address in question has not made any edits in the last 6 months * * Once deleted, the url for that talk page will return a 404 (by default) * * All talk pages that are deleted will be echo'd out to be * stored in the log for this script. ****/ require_once 'commandLine.inc'; global $IP, $wgUser; require_once "{$IP}/extensions/wikihow/DatabaseHelper.class.php"; $wgUser = User::newFromName("MiscBot"); $maxAge = 60 * 60 * 24 * 31 * 6; //6 months measured in seconds $dbr = wfGetDB(DB_SLAVE); echo "Checking talk pages from anonymous users on " . date("F j, Y") . "\n"; $articles = DatabaseHelper::batchSelect('page', array('page_id', 'page_title'), array('page_namespace' => NS_USER_TALK, 'page_is_redirect' => 0)); /***** //TESTING CODE// $res = $dbr->select('page', array('page_id', 'page_title'), array('page_namespace' => NS_USER_TALK, 'page_is_redirect' => 0), __FUNCTION__, array('limit' => 200)); $res = $dbr->select('page', array('page_id', 'page_title'), array('page_namespace' => NS_USER_TALK, 'page_is_redirect' => 0, 'page_title' => '67.168.160.94'), __FUNCTION__, array('limit' => 10)); echo "SQL command done.\n"; $articles = array(); while($row = $dbr->fetchObject($res)) { $articles[] = $row; } *****/ echo "About to check " . count($articles) . " pages\n"; $i = 0; foreach ($articles as $article) {
* will grab this field, subtract the value for 30 days ago (gotten out * of the titus_historical table) and add on the new value (gotten out * of the titus table) * * The whole script took about 6 minutes on June 13, 2012 -Reuben. * ******/ require_once 'commandLine.inc'; require_once "{$IP}/extensions/wikihow/DatabaseHelper.class.php"; $dbr = wfGetDB(DB_SLAVE); $dbw = wfGetDB(DB_MASTER); $startTime = microtime(true); define(BATCHSIZE, 1000); // first grab all the pages -- this first section takes about 1 minute as // of June 13 2012 (Reuben). $articles = DatabaseHelper::batchSelect('page', array('page_id', 'page_title'), array('page_namespace' => NS_MAIN, 'page_is_redirect' => 0), __FILE__); // now we need to recalculate the pv data $start = time(); $now = wfTimestamp(TS_MW); $monthAgo = substr(wfTimestamp(TS_MW, $start - 60 * 60 * 24 * 30), 0, 8); // 30 days print "Starting from " . $monthAgo . " and processing " . count($articles) . " in approx " . ceil(count($articles) / BATCHSIZE) . " batches\n"; $rows = array(); $nulls = 0; foreach ($articles as $articleData) { $total = 0; $row = $dbr->selectRow('titus_historical', array('ti_datestamp', 'SUM(ti_daily_views) as ti_sum'), array('ti_page_id' => $articleData->page_id, "ti_datestamp > {$monthAgo}"), __FILE__); $field = $dbr->selectField('titus', 'ti_daily_views', array('ti_page_id' => $articleData->page_id), __FILE__); if ($row) { $total += intval($row->ti_sum); if ($field !== false) {
<?php /************* * * With the move to Fastly we can no longer rely on the old varnish method of counting * visits. So we're usin the data that we're already getting from stu to increment the * page_counter field. This will happen once a day and this script must be called AFTER * updateMontlyPageView.php script in the nightly cron. * */ require_once 'commandLine.inc'; require_once "{$IP}/extensions/wikihow/DatabaseHelper.class.php"; $start = microtime(true); $articles = DatabaseHelper::batchSelect('pageview', array('pv_page', 'pv_1day')); echo "done getting all the articles\n"; $dbw = wfGetDB(DB_MASTER); $articleCount = 0; foreach ($articles as $article) { $dbw->update('page', array('page_counter = page_counter + ' . $article->pv_1day), array('page_id' => $article->pv_page), __FILE__); $articleCount++; if ($articleCount % 1000 == 0) { sleep(5); } } $end = microtime(true); echo "Finished in " . ($end - $start) . " seconds.\n";
<?php /* * * Script that aggregates all external links from a text file created by * echo 'select el_from, el_to from externallinks' |mysql wikidb_112 > externallinks.txt * */ global $IP, $wgTitle; require_once '../commandLine.inc'; require_once "{$IP}/extensions/wikihow/DatabaseHelper.class.php"; echo "Starting batchSelect " . date('H:i') . "\n"; $dbr = wfGetDB(DB_SLAVE); $linksArray = DatabaseHelper::batchSelect('externallinks', array('el_to', 'el_from'), array(), __FILE__); /******** //TESTING CODE $dbr = wfGetDB(DB_SLAVE); $linksArray = $dbr->select('externallinks', array('*'), array(), __FILE__, array("LIMIT" => 10)); $obj->el_to = "http://emiliastone.blog.com/2009/11/03/wholesale-dvds-a-treasure-trove-of-deals/"; $obj->el_from = 636659; $obj2->el_to = "http://hubpages.com/hub/Authority_Summit "; $obj2->el_from = 6666; $linksArray[] = $obj; $linksArray[] = $obj2; /********/ echo "Done batchSelect " . date('H:i') . "\n"; //putting this up here so the database doesn't crap out $whitelistString = wfMsg('Spam-whitelist'); $domainsString = "\n"; foreach ($linksArray as $link) { //while ( !feof( $fi ) ) {
<?php /****************** * * Outputs a list of all users that have earned * the featured author badge * *****************/ require_once 'commandLine.inc'; require_once '../extensions/wikihow/DatabaseHelper.class.php'; $dbr = wfGetDB(DB_SLAVE); $users = DatabaseHelper::batchSelect('user', 'user_id'); foreach ($users as $userObj) { $user = User::newFromId($userObj->user_id); $resFA = $dbr->select(array('firstedit', 'templatelinks'), '*', array('fe_page=tl_from', 'fe_user' => $user->getID(), 'tl_title = "Fa" OR tl_title = "FA"'), __FUNCTION__, array('GROUP BY' => 'fe_page')); $resRS = $dbr->select(array('firstedit', 'pagelist'), '*', array('fe_page=pl_page', 'fe_user' => $user->getID()), __FUNCTION__, array('GROUP BY' => 'fe_page')); if ($dbr->numRows($resFA) + $dbr->numRows($resRS) >= 5) { $lastTime = $dbr->selectField(array('logging'), array('log_timestamp'), array('log_user' => $userObj->user_id), __FUNCTION__, array("ORDER BY" => "log_timestamp DESC", "LIMIT" => 1)); $lastDate = date("n/j/Y", wfTimestamp(TS_UNIX, $lastTime)); echo $user->getUserPage()->getFullURL() . " " . $lastDate . "\n"; } }
<?php /**** * Script that handles the special logging table used for method editor logging. * We only log "keeps" and only keep 35 days. This script can be used to populate * the table initially and the other will be run from the cron to get rid of unused * rows in the table. */ require_once 'commandLine.inc'; $action = $argv[0]; if ($action == "") { echo "You need to indicate what you want to do (init/cull)\n"; exit(0); } $dbw = wfGetDB(DB_MASTER); switch ($action) { case "cull": $expired = wfTimestamp(TS_MW, strtotime("-35 day", strtotime(date('Ymd', time())))); $res = $dbw->delete(MethodEditor::LOGGING_TABLE_NAME, array("mel_timestamp <= {$expired}"), __FILE__); break; case "init": $dbw->delete(MethodEditor::LOGGING_TABLE_NAME, '*', __FILE__); $edits = DatabaseHelper::batchSelect('logging', array('log_timestamp', 'log_user'), array('log_type' => 'methedit'), __FILE__); foreach ($edits as $edit) { $dbw->insert(MethodEditor::LOGGING_TABLE_NAME, array('mel_timestamp' => $edit->log_timestamp, 'mel_user' => $edit->log_user)); } break; }
<?php #define('WH_USE_BACKUP_DB', true); require_once 'commandLine.inc'; global $IP, $wgUser; require_once "{$IP}/extensions/wikihow/DatabaseHelper.class.php"; $whitelist = array("example.com", "wikihow.com", "wikipedia.org", "youtube.com", "wikia.com", "blogger.com", "blogspot.com", "about.me", "aboutme.com", "me.com", "mac.com", "tumblr.com", "twitter.com", "4chan.com", "uncyclopedia.org", "myspace.com", "wikiHow.com", "fanfiction.net", "deviantart.com", "google.com", "facebook.com", "pinterest.com", "posterous.com", "wordpress.com", "livejournal.com", "wordpress.org", "typepad.com", "wikimedia.org", "wiktionary.org", "flickr.com", "dailybooth.com", "creativecommons.com", "bebo.com", "linkedin.com", "freenode.net", "photobucket.com", "yahoo.com", "vimeo.com", "stumbleupon.com", "delicious.com", "ning.com", "digg.com", "reddit.com", "msn.com", "microsoft.com", "huffingtonpost.com", "answers.com", "live.com", "comcast.net", "aol.com", "examiner.com", "legacy.com", "monster.com", "apple.com", "bleacherreport.com", "cafemom.com", "yardbarker.com", "buzzfeed.com", "path.com", "instagram.com", "43things.com", "aviary.com", "backflip.com", "badoo.com", "bebo.com", "blogcatalog.com", "blurb.com", "buzznet.com", "cafemom.com", "current.com", "dailybooth.com", "dailymotion.com", "delicious.com", "digg.com", "diigo.com", "epinions.com", "esqueak.com", "facebook.com", "fark.com", "fotolog.com", "friendfeed.com", "friendster.com", "funnyordie.com", "gather.com", "gawkk.com", "hellotxt.com", "hi5.com", "hootsuite.com", "huffingtonpost.com", "jaiku.com", "last.fm", "livevideo.com", "mixx.com", "multiply.com", "myspace.com", "ping.fm", "plurk.com", "propeller.com", "reddit.com", "revver.com", "ryze.com", "seesmic.com", "sphinn.com", "stumbleupon.com", "tipjoy.com", "tinychat.com", "tribe.net", "tumblr.com", "twitter.com", "xanga.com", "yahoo.com", "gmail.com"); $wgUser = User::newFromName("MiscBot"); $maxAge = 60 * 60 * 24 * 14; // 2 weeks $dbr = getDBR(); echo "Checking user pages on " . date("F j, Y") . "\n"; $articles = DatabaseHelper::batchSelect('page', array('page_id', 'page_title', 'page_counter'), array('page_namespace' => NS_USER, 'page_is_redirect' => 0), __METHOD__, array(), DatabaseHelper::DEFAULT_BATCH_SIZE, $dbr); /***** //TESTING CODE// $res = $dbr->select('page', array('page_id', 'page_title'), array('page_namespace' => NS_USER, 'page_is_redirect' => 0, "page_title like 'A%'"), __FUNCTION__, array('LIMIT' => 100000)); echo "SQL command done.\n"; $articles = array(); while($row = $dbr->fetchObject($res)) { $articles[] = $row; } /*****/ echo "About to check " . count($articles) . " pages\n"; $i = 0; $j = 0; foreach ($articles as $article) { $i++; //first check to see if the page in not anonymous user if (filter_var($article->page_title, FILTER_VALIDATE_IP) === true) { continue;
public function getCompletedArticles($langCode, $fromDate) { global $IP; $dbr = self::getDBR(); require_once "{$IP}/extensions/wikihow/DatabaseHelper.class.php"; $articleTable = $this->config->getArticleTableName(); $defaultUser = $this->config->getDefaultUserName(); $rows = DatabaseHelper::batchSelect($articleTable, array('*'), array("ct_completed" => 1, "ct_lang_code" => $langCode, "ct_completed_timestamp > '{$fromDate}'", "ct_user_text != '{$defaultUser}'"), __METHOD__, array(), DatabaseHelper::DEFAULT_BATCH_SIZE, $dbr); $this->formatData($rows); return self::getReportArray(WAPUtil::generateTSVOutput($rows)); }
/** * Grab the status of all articles processed. */ private static function dbGetArticlesUpdatedAll() { $articles = array(); $dbr = self::getDB('read'); $res = DatabaseHelper::batchSelect('wikivideo_article_status', array('article_id', 'processed', 'error', 'retry'), '', __METHOD__, array(), DatabaseHelper::DEFAULT_BATCH_SIZE, $dbr); foreach ($res as $row) { // convert MW timestamp to unix timestamp $row->processed = wfTimestamp(TS_UNIX, $row->processed); $articles[$row->article_id] = (array) $row; } return $articles; }
<?php /** * One time use script to port all of the current values * for the wH preference 'disablewarning' to the new * core preference 'useeditwarning'. */ require_once 'commandLine.inc'; $users = DatabaseHelper::batchSelect('user', array('user_id')); foreach ($users as $user) { $user = User::newFromId($user->user_id); $disableWarning = $user->getOption('disablewarning', "0"); $useEditWarning = $user->getOption('useeditwarning', "1"); $hasChanged = false; if ($disableWarning == "0" && $useEditWarning == "0") { $user->setOption('useeditwarning', 1); $hasChanged = true; } else { if ($disableWarning == "1" && $useEditWarning == "1") { $user->setOption('useeditwarning', 0); $hasChanged = true; } } if ($hasChanged) { $user->saveSettings(); } }
<?php require_once 'commandLine.inc'; require_once "{$IP}/extensions/wikihow/titus/Titus.class.php"; require_once "{$IP}/extensions/wikihow/DatabaseHelper.class.php"; $dbw = DatabaseBase::factory('mysql'); $dbw->open(TITUS_DB_HOST, WH_DATABASE_MAINTENANCE_USER, WH_DATABASE_MAINTENANCE_PASSWORD, TitusDB::getDBName()); $rows = DatabaseHelper::batchSelect('titus_intl', 'ti_page_id', array('ti_language_code' => $wgLanguageCode), __METHOD__, array(), 2000, $dbw); $deletedPageIds = array(); foreach ($rows as $row) { $t = Title::newFromId($row->ti_page_id); if (!($t && $t->exists() && $t->getNamespace() == NS_MAIN)) { $deletedPageIds[] = $row->ti_page_id; } } $chunks = array_chunk($deletedPageIds, 500); foreach ($chunks as $chunk) { $aids = "(" . implode(",", $chunk) . ")"; $sql = "DELETE FROM titus_intl where ti_page_id IN {$aids} AND ti_language_code='{$wgLanguageCode}'"; var_dump($sql); $dbw->query($sql); }
public function getOldRecords($datestamp) { $dbr = $this->getTitusDB(); $rows = DatabaseHelper::batchSelect(TitusDB::TITUS_HISTORICAL_TABLE_NAME, '*', array('ti_datestamp' => $datestamp), __METHOD__, array(), 2000, $dbr); return $rows; }