private function addCoreDBData() { if ($this->db->getType() == 'oracle') { # Insert 0 user to prevent FK violations # Anonymous user if (!$this->db->selectField('user', '1', ['user_id' => 0])) { $this->db->insert('user', ['user_id' => 0, 'user_name' => 'Anonymous'], __METHOD__, ['IGNORE']); } # Insert 0 page to prevent FK violations # Blank page if (!$this->db->selectField('page', '1', ['page_id' => 0])) { $this->db->insert('page', ['page_id' => 0, 'page_namespace' => 0, 'page_title' => ' ', 'page_restrictions' => null, 'page_is_redirect' => 0, 'page_is_new' => 0, 'page_random' => 0, 'page_touched' => $this->db->timestamp(), 'page_latest' => 0, 'page_len' => 0], __METHOD__, ['IGNORE']); } } User::resetIdByNameCache(); // Make sysop user $user = static::getTestSysop()->getUser(); // Make 1 page with 1 revision $page = WikiPage::factory(Title::newFromText('UTPage')); if ($page->getId() == 0) { $page->doEditContent(new WikitextContent('UTContent'), 'UTPageSummary', EDIT_NEW, false, $user); // doEditContent() probably started the session via // User::loadFromSession(). Close it now. if (session_id() !== '') { session_write_close(); session_id(''); } } }
protected function doCollationUpdate() { global $wgCategoryCollation; if ($this->db->selectField('categorylinks', 'COUNT(*)', 'cl_collation != ' . $this->db->addQuotes($wgCategoryCollation), __METHOD__) == 0) { $this->output("...collations up-to-date.\n"); return; } $task = new UpdateCollation(); $task->execute(); }
/** * Update CategoryLinks collation */ protected function doCollationUpdate() { global $wgCategoryCollation; if ($this->db->selectField('categorylinks', 'COUNT(*)', 'cl_collation != ' . $this->db->addQuotes($wgCategoryCollation), __METHOD__) == 0) { $this->output("...collations up-to-date.\n"); return; } $this->output("Updating category collations..."); $task = $this->maintenance->runChild('UpdateCollation'); $task->execute(); $this->output("...done.\n"); }
function buildTable($table, $key, $callback) { $count = $this->dbw->selectField($table, 'count(*)', '', __METHOD__); $this->init($count, $table); $this->output("Processing {$table}...\n"); $result = wfGetDB(DB_SLAVE)->select($table, '*', array(), __METHOD__); foreach ($result as $row) { $update = call_user_func($callback, $row, null); if ($update) { $this->progress(1); } else { $this->progress(0); } } $this->output("Finished {$table}... {$this->updated} of {$this->processed} rows updated\n"); }
/** * Copy all rows from $srcTable to $dstTable */ function sync($srcTable, $dstTable) { $batchSize = 1000; $minTs = $this->dbw->selectField($srcTable, 'MIN(log_timestamp)', false, __METHOD__); $minTsUnix = wfTimestamp(TS_UNIX, $minTs); $numRowsCopied = 0; while (true) { $maxTs = $this->dbw->selectField($srcTable, 'MAX(log_timestamp)', false, __METHOD__); $copyPos = $this->dbw->selectField($dstTable, 'MAX(log_timestamp)', false, __METHOD__); $maxTsUnix = wfTimestamp(TS_UNIX, $maxTs); $copyPosUnix = wfTimestamp(TS_UNIX, $copyPos); if ($copyPos === null) { $percent = 0; } else { $percent = ($copyPosUnix - $minTsUnix) / ($maxTsUnix - $minTsUnix) * 100; } printf("%s %.2f%%\n", $copyPos, $percent); # Handle all entries with timestamp equal to $copyPos if ($copyPos !== null) { $numRowsCopied += $this->copyExactMatch($srcTable, $dstTable, $copyPos); } # Now copy a batch of rows if ($copyPos === null) { $conds = false; } else { $conds = array('log_timestamp > ' . $this->dbw->addQuotes($copyPos)); } $srcRes = $this->dbw->select($srcTable, '*', $conds, __METHOD__, array('LIMIT' => $batchSize, 'ORDER BY' => 'log_timestamp')); if (!$srcRes->numRows()) { # All done break; } $batch = array(); foreach ($srcRes as $srcRow) { $batch[] = (array) $srcRow; } $this->dbw->insert($dstTable, $batch, __METHOD__); $numRowsCopied += count($batch); wfWaitForSlaves(); } echo "Copied {$numRowsCopied} rows\n"; }
/** * Update the query cache as needed * * @param DatabaseBase $dbw * @param int $days How many days user must be idle before he is considered inactive * @param int $window Maximum time range of new data to scan (in seconds) * @return int|bool UNIX timestamp the cache is now up-to-date as of (false on error) */ protected static function doQueryCacheUpdate(DatabaseBase $dbw, $days, $window) { $lockKey = wfWikiID() . '-activeusers'; if (!$dbw->lock($lockKey, __METHOD__, 1)) { return false; // exclusive update (avoids duplicate entries) } $now = time(); $cTime = $dbw->selectField('querycache_info', 'qci_timestamp', array('qci_type' => 'activeusers')); $cTimeUnix = $cTime ? wfTimestamp(TS_UNIX, $cTime) : 1; // Pick the date range to fetch from. This is normally from the last // update to till the present time, but has a limited window for sanity. // If the window is limited, multiple runs are need to fully populate it. $sTimestamp = max($cTimeUnix, $now - $days * 86400); $eTimestamp = min($sTimestamp + $window, $now); // Get all the users active since the last update $res = $dbw->select(array('recentchanges'), array('rc_user_text', 'lastedittime' => 'MAX(rc_timestamp)'), array('rc_user > 0', 'rc_type != ' . $dbw->addQuotes(RC_EXTERNAL), 'rc_log_type IS NULL OR rc_log_type != ' . $dbw->addQuotes('newusers'), 'rc_timestamp >= ' . $dbw->addQuotes($dbw->timestamp($sTimestamp)), 'rc_timestamp <= ' . $dbw->addQuotes($dbw->timestamp($eTimestamp))), __METHOD__, array('GROUP BY' => array('rc_user_text'), 'ORDER BY' => 'NULL')); $names = array(); foreach ($res as $row) { $names[$row->rc_user_text] = $row->lastedittime; } // Rotate out users that have not edited in too long (according to old data set) $dbw->delete('querycachetwo', array('qcc_type' => 'activeusers', 'qcc_value < ' . $dbw->addQuotes($now - $days * 86400)), __METHOD__); // Find which of the recently active users are already accounted for if (count($names)) { $res = $dbw->select('querycachetwo', array('user_name' => 'qcc_title'), array('qcc_type' => 'activeusers', 'qcc_namespace' => NS_USER, 'qcc_title' => array_keys($names)), __METHOD__); foreach ($res as $row) { unset($names[$row->user_name]); } } // Insert the users that need to be added to the list (which their last edit time if (count($names)) { $newRows = array(); foreach ($names as $name => $lastEditTime) { $newRows[] = array('qcc_type' => 'activeusers', 'qcc_namespace' => NS_USER, 'qcc_title' => $name, 'qcc_value' => wfTimestamp(TS_UNIX, $lastEditTime), 'qcc_namespacetwo' => 0, 'qcc_titletwo' => ''); } foreach (array_chunk($newRows, 500) as $rowBatch) { $dbw->insert('querycachetwo', $rowBatch, __METHOD__); if (!$dbw->trxLevel()) { wfWaitForSlaves(); } } } // Touch the data freshness timestamp $dbw->replace('querycache_info', array('qci_type'), array('qci_type' => 'activeusers', 'qci_timestamp' => $dbw->timestamp($eTimestamp)), __METHOD__); $dbw->unlock($lockKey, __METHOD__); return $eTimestamp; }
/** * Create some initial DB entries for important built-in properties. Having the DB contents predefined * allows us to safe DB calls when certain data is needed. At the same time, the entries in the DB * make sure that DB-based functions work as with all other properties. */ protected function setupPredefinedProperties($verbose, DatabaseBase $db) { global $wgDBtype; $this->reportProgress("Setting up internal property indices ...\n", $verbose); // Check if we already have this structure $borderiw = $db->selectField(SMWSQLStore3::ID_TABLE, 'smw_iw', 'smw_id=' . $db->addQuotes(\SMWSql3SmwIds::FXD_PROP_BORDER_ID)); if ($borderiw != SMW_SQL3_SMWBORDERIW) { $this->reportProgress(" ... allocating space for internal properties ...\n", $verbose); $this->store->smwIds->moveSMWPageID(\SMWSql3SmwIds::FXD_PROP_BORDER_ID); // make sure position 50 is empty $db->insert(SMWSQLStore3::ID_TABLE, array('smw_id' => \SMWSql3SmwIds::FXD_PROP_BORDER_ID, 'smw_title' => '', 'smw_namespace' => 0, 'smw_iw' => SMW_SQL3_SMWBORDERIW, 'smw_subobject' => '', 'smw_sortkey' => ''), 'SMW::setup'); // put dummy "border element" on index 50 $this->reportProgress(' ', $verbose); for ($i = 0; $i < \SMWSql3SmwIds::FXD_PROP_BORDER_ID; $i++) { // make way for built-in ids $this->store->smwIds->moveSMWPageID($i); $this->reportProgress('.', $verbose); } $this->reportProgress(" done.\n", $verbose); } else { $this->reportProgress(" ... space for internal properties already allocated.\n", $verbose); } // now write actual properties; do that each time, it is cheap enough and we can update sortkeys by current language $this->reportProgress(" ... writing entries for internal properties ...", $verbose); foreach (SMWSql3SmwIds::$special_ids as $prop => $id) { $p = new SMWDIProperty($prop); $db->replace(SMWSQLStore3::ID_TABLE, array('smw_id'), array('smw_id' => $id, 'smw_title' => $p->getKey(), 'smw_namespace' => SMW_NS_PROPERTY, 'smw_iw' => $this->store->smwIds->getPropertyInterwiki($p), 'smw_subobject' => '', 'smw_sortkey' => $p->getLabel()), 'SMW::setup'); } $this->reportProgress(" done.\n", $verbose); if ($wgDBtype == 'postgres') { $sequenceIndex = SMWSQLStore3::ID_TABLE . '_smw_id_seq'; $this->reportProgress(" ... updating {$sequenceIndex} sequence accordingly.\n", $verbose); $max = $db->selectField(SMWSQLStore3::ID_TABLE, 'max(smw_id)', array(), __METHOD__); $max += 1; $db->query("ALTER SEQUENCE {$sequenceIndex} RESTART WITH {$max}", __METHOD__); } $this->reportProgress("Internal properties initialized successfully.\n", $verbose); }
/** * Get previous revision Id for this page_id * This is used to populate rev_parent_id on save * * @param DatabaseBase $db * @return int */ private function getPreviousRevisionId($db) { if ($this->mPage === null) { return 0; } # Use page_latest if ID is not given if (!$this->mId) { $prevId = $db->selectField('page', 'page_latest', array('page_id' => $this->mPage), __METHOD__); } else { $prevId = $db->selectField('revision', 'rev_id', array('rev_page' => $this->mPage, 'rev_id < ' . $this->mId), __METHOD__, array('ORDER BY' => 'rev_id DESC')); } return intval($prevId); }
/** * Tries to get the revision text for a revision id. * Export transformations are applied if the content model can is given or can be * determined from the database. * * Upon errors, retries (Up to $this->maxFailures tries each call). * If still no good revision get could be found even after this retrying, "" is returned. * If no good revision text could be returned for * $this->maxConsecutiveFailedTextRetrievals consecutive calls to getText, MWException * is thrown. * * @param string $id The revision id to get the text for * @param string|bool|null $model The content model used to determine * applicable export transformations. * If $model is null, it will be determined from the database. * @param string|null $format The content format used when applying export transformations. * * @throws MWException * @return string The revision text for $id, or "" */ function getText($id, $model = null, $format = null) { global $wgContentHandlerUseDB; $prefetchNotTried = true; // Whether or not we already tried to get the text via prefetch. $text = false; // The candidate for a good text. false if no proper value. $failures = 0; // The number of times, this invocation of getText already failed. // The number of times getText failed without yielding a good text in between. static $consecutiveFailedTextRetrievals = 0; $this->fetchCount++; // To allow to simply return on success and do not have to worry about book keeping, // we assume, this fetch works (possible after some retries). Nevertheless, we koop // the old value, so we can restore it, if problems occur (See after the while loop). $oldConsecutiveFailedTextRetrievals = $consecutiveFailedTextRetrievals; $consecutiveFailedTextRetrievals = 0; if ($model === null && $wgContentHandlerUseDB) { $row = $this->db->selectRow('revision', array('rev_content_model', 'rev_content_format'), array('rev_id' => $this->thisRev), __METHOD__); if ($row) { $model = $row->rev_content_model; $format = $row->rev_content_format; } } if ($model === null || $model === '') { $model = false; } while ($failures < $this->maxFailures) { // As soon as we found a good text for the $id, we will return immediately. // Hence, if we make it past the try catch block, we know that we did not // find a good text. try { // Step 1: Get some text (or reuse from previous iteratuon if checking // for plausibility failed) // Trying to get prefetch, if it has not been tried before if ($text === false && isset($this->prefetch) && $prefetchNotTried) { $prefetchNotTried = false; $tryIsPrefetch = true; $text = $this->prefetch->prefetch(intval($this->thisPage), intval($this->thisRev)); if ($text === null) { $text = false; } if (is_string($text) && $model !== false) { // Apply export transformation to text coming from an old dump. // The purpose of this transformation is to convert up from legacy // formats, which may still be used in the older dump that is used // for pre-fetching. Applying the transformation again should not // interfere with content that is already in the correct form. $text = $this->exportTransform($text, $model, $format); } } if ($text === false) { // Fallback to asking the database $tryIsPrefetch = false; if ($this->spawn) { $text = $this->getTextSpawned($id); } else { $text = $this->getTextDb($id); } if ($text !== false && $model !== false) { // Apply export transformation to text coming from the database. // Prefetched text should already have transformations applied. $text = $this->exportTransform($text, $model, $format); } // No more checks for texts from DB for now. // If we received something that is not false, // We treat it as good text, regardless of whether it actually is or is not if ($text !== false) { return $text; } } if ($text === false) { throw new MWException("Generic error while obtaining text for id " . $id); } // We received a good candidate for the text of $id via some method // Step 2: Checking for plausibility and return the text if it is // plausible $revID = intval($this->thisRev); if (!isset($this->db)) { throw new MWException("No database available"); } if ($model !== CONTENT_MODEL_WIKITEXT) { $revLength = strlen($text); } else { $revLength = $this->db->selectField('revision', 'rev_len', array('rev_id' => $revID)); } if (strlen($text) == $revLength) { if ($tryIsPrefetch) { $this->prefetchCount++; } return $text; } $text = false; throw new MWException("Received text is unplausible for id " . $id); } catch (Exception $e) { $msg = "getting/checking text " . $id . " failed (" . $e->getMessage() . ")"; if ($failures + 1 < $this->maxFailures) { $msg .= " (Will retry " . ($this->maxFailures - $failures - 1) . " more times)"; } $this->progress($msg); } // Something went wrong; we did not a text that was plausible :( $failures++; // A failure in a prefetch hit does not warrant resetting db connection etc. if (!$tryIsPrefetch) { // After backing off for some time, we try to reboot the whole process as // much as possible to not carry over failures from one part to the other // parts sleep($this->failureTimeout); try { $this->rotateDb(); if ($this->spawn) { $this->closeSpawn(); $this->openSpawn(); } } catch (Exception $e) { $this->progress("Rebooting getText infrastructure failed (" . $e->getMessage() . ")" . " Trying to continue anyways"); } } } // Retirieving a good text for $id failed (at least) maxFailures times. // We abort for this $id. // Restoring the consecutive failures, and maybe aborting, if the dump // is too broken. $consecutiveFailedTextRetrievals = $oldConsecutiveFailedTextRetrievals + 1; if ($consecutiveFailedTextRetrievals > $this->maxConsecutiveFailedTextRetrievals) { throw new MWException("Graceful storage failure"); } return ""; }
public static function getResultsCount() { $row = self::$dbr->selectField(array_map([self::$dbr, 'tableName'], self::$aTables), ['count(distinct(page_id)) as count'], self::$aWhere, __METHOD__); return $row ? (int) $row : 0; }