/** * loads property pair occurrence probability table from given csv file */ function execute() { if (substr($this->getOption('file'), 0, 2) === "--") { $this->error("The --file option requires a file as an argument.\n", true); } $path = $this->getOption('file'); $fullPath = realpath($path); $fullPath = str_replace('\\', '/', $fullPath); if (!file_exists($fullPath)) { $this->error("Cant find {$path} \n", true); } $tableName = 'wbs_propertypairs'; wfWaitForSlaves(); $lb = wfGetLB(); $this->clearTable($lb, $tableName); $this->output("loading new entries from file\n"); $importContext = $this->createImportContext($lb, $tableName, $fullPath, $this->isQuiet()); $importStrategy = new BasicImporter(); try { $success = $importStrategy->importFromCsvFileToDb($importContext); } catch (UnexpectedValueException $e) { $this->error("Import failed: " . $e->getMessage()); exit; } if (!$success) { $this->error("Failed to run import to db"); } $this->output("... Done loading\n"); }
/** * Add an update to the deferred list * @param DeferrableUpdate $update Some object that implements doUpdate() */ public static function addUpdate(DeferrableUpdate $update) { global $wgCommandLineMode; array_push(self::$updates, $update); if (self::$forceDeferral) { return; } // CLI scripts may forget to periodically flush these updates, // so try to handle that rather than OOMing and losing them. // Try to run the updates as soon as there is no local transaction. static $waitingOnTrx = false; // de-duplicate callback if ($wgCommandLineMode && !$waitingOnTrx) { $lb = wfGetLB(); $dbw = $lb->getAnyOpenConnection($lb->getWriterIndex()); // Do the update as soon as there is no transaction if ($dbw && $dbw->trxLevel()) { $waitingOnTrx = true; $dbw->onTransactionIdle(function () use(&$waitingOnTrx) { DeferredUpdates::doUpdates(); $waitingOnTrx = false; }); } else { self::doUpdates(); } } }
/** * Execute checks for all requested clusters */ private function testClusters() { foreach ($this->clusters as $clusterName) { $this->current = "{$clusterName}: "; $fullHealth = false; $operational = false; try { $databaseName = $this->getClusterDatabase($clusterName); $loadBalancer = wfGetLB($databaseName); $serverCount = $loadBalancer->getServerCount(); $roles = ['master' => [], 'slave' => []]; for ($i = 0; $i < $serverCount; $i++) { $serverName = $loadBalancer->getServerName($i); $this->current = "{$clusterName}: {$serverName}: "; $role = $i === 0 ? 'master' : 'slave'; $roles[$role][$serverName] = $this->testHost($databaseName, $loadBalancer, $i); } if ($serverCount == 1) { $fullHealth = $operational = reset($roles['master']); } else { // full health = no host raised any issue $fullHealth = !$this->occursInArray(false, $roles['master']) && !$this->occursInArray(false, $roles['slave']); // operational = at least one master and one slave are working correctly $operational = $this->occursInArray(true, $roles['master']) && $this->occursInArray(true, $roles['slave']); } } catch (DBError $e) { $this->addError($e->getMessage()); } $this->status[$clusterName] = $fullHealth ? 'ok' : ($operational ? 'warning' : 'critical'); } }
public function execute() { if ($this->hasOption('r')) { $lb = wfGetLB(); echo 'time '; $serverCount = $lb->getServerCount(); for ($i = 1; $i < $serverCount; $i++) { $hostname = $lb->getServerName($i); printf("%-12s ", $hostname); } echo "\n"; while (1) { $lb->clearLagTimeCache(); $lags = $lb->getLagTimes(); unset($lags[0]); echo gmdate('H:i:s') . ' '; foreach ($lags as $lag) { printf("%-12s ", $lag === false ? 'false' : $lag); } echo "\n"; sleep(5); } } else { $lb = wfGetLB(); $lags = $lb->getLagTimes(); foreach ($lags as $i => $lag) { $name = $lb->getServerName($i); $this->output(sprintf("%-20s %s\n", $name, $lag === false ? 'false' : $lag)); } } }
/** * Constructor * * @param bool $withTransaction Whether this update should be wrapped in a * transaction (default: true). A transaction is only started if no * transaction is already in progress, see beginTransaction() for details. */ public function __construct($withTransaction = true) { parent::__construct(); $this->mDb = wfGetLB()->getLazyConnectionRef(DB_MASTER); $this->mWithTransaction = $withTransaction; $this->mHasTransaction = false; }
public function execute() { $userName = '******'; // <- targer username $user = new CentralAuthUser( $userName ); if ( !$user->exists() ) { echo "Cannot unsuppress non-existent user {$userName}!\n"; exit( 0 ); } $userName = $user->getName(); // sanity $wikis = $user->listAttached(); // wikis with attached accounts foreach ( $wikis as $wiki ) { $lb = wfGetLB( $wiki ); $dbw = $lb->getConnection( DB_MASTER, array(), $wiki ); # Get local ID like $user->localUserData( $wiki ) does $localUserId = $dbw->selectField( 'user', 'user_id', array( 'user_name' => $userName ), __METHOD__ ); $delUserBit = Revision::DELETED_USER; $hiddenCount = $dbw->selectField( 'revision', 'COUNT(*)', array( 'rev_user' => $localUserId, "rev_deleted & $delUserBit != 0" ), __METHOD__ ); echo "$hiddenCount edits have the username hidden on \"$wiki\"\n"; # Unsuppress username on edits if ( $hiddenCount > 0 ) { echo "Unsuppressed edits of attached account (local id $localUserId) on \"$wiki\"..."; IPBlockForm::unsuppressUserName( $userName, $localUserId, $dbw ); echo "done!\n\n"; } $lb->reuseConnection( $dbw ); // not really needed # Don't lag too bad wfWaitForSlaves( 5 ); } }
/** * Special page "deleted user contributions". * Shows a list of the deleted contributions of a user. * * @param string $par (optional) user name of the user for which to show the contributions */ function execute($par) { $this->setHeaders(); $this->outputHeader(); $user = $this->getUser(); if (!$this->userCanExecute($user)) { $this->displayRestrictionError(); return; } $request = $this->getRequest(); $out = $this->getOutput(); $out->setPageTitle($this->msg('deletedcontributions-title')); $options = []; if ($par !== null) { $target = $par; } else { $target = $request->getVal('target'); } if (!strlen($target)) { $out->addHTML($this->getForm('')); return; } $options['limit'] = $request->getInt('limit', $this->getConfig()->get('QueryPageDefaultLimit')); $options['target'] = $target; $userObj = User::newFromName($target, false); if (!$userObj) { $out->addHTML($this->getForm('')); return; } $this->getSkin()->setRelevantUser($userObj); $target = $userObj->getName(); $out->addSubtitle($this->getSubTitle($userObj)); $ns = $request->getVal('namespace', null); if ($ns !== null && $ns !== '') { $options['namespace'] = intval($ns); } else { $options['namespace'] = ''; } $out->addHTML($this->getForm($options)); $pager = new DeletedContribsPager($this->getContext(), $target, $options['namespace']); if (!$pager->getNumRows()) { $out->addWikiMsg('nocontribs'); return; } # Show a message about replica DB lag, if applicable $lag = wfGetLB()->safeGetLag($pager->getDatabase()); if ($lag > 0) { $out->showLagWarning($lag); } $out->addHTML('<p>' . $pager->getNavigationBar() . '</p>' . $pager->getBody() . '<p>' . $pager->getNavigationBar() . '</p>'); # If there were contributions, and it was a valid user or IP, show # the appropriate "footer" message - WHOIS tools, etc. if ($target != 'newbies') { $message = IP::isIPAddress($target) ? 'sp-contributions-footer-anon' : 'sp-contributions-footer'; if (!$this->msg($message)->isDisabled()) { $out->wrapWikiMsg("<div class='mw-contributions-footer'>\n\$1\n</div>", [$message, $target]); } } }
/** * @dataProvider provideChangeExists */ public function testChangeExists($expected, array $changeData) { $connectionManager = new ConsistentReadConnectionManager(wfGetLB()); $detector = new RecentChangesDuplicateDetector($connectionManager); $this->initRecentChanges(); $change = $this->newChange($changeData); $this->assertEquals($expected, $detector->changeExists($change), 'changeExists()'); }
function tearDown() { if (!is_null($this->db)) { wfGetLB()->closeConnecton($this->db); } unset($this->db); unset($this->search); }
private function addTestChanges() { $changeStore = new SqlChangeStore(wfGetLB()); $change = new EntityChange($this->getChangeRowData('20150101000005')); $changeStore->saveChange($change); $change = new EntityChange($this->getChangeRowData('20150101000300')); $changeStore->saveChange($change); }
protected function setUp() { $this->tablesUsed[] = EntityUsageTable::DEFAULT_TABLE_NAME; $this->tablesUsed[] = 'page'; parent::setUp(); $this->sqlUsageTracker = new SqlUsageTracker(new BasicEntityIdParser(), new ConsistentReadConnectionManager(wfGetLB())); $this->trackerTester = new UsageTrackerContractTester($this->sqlUsageTracker, array($this, 'getUsages')); $this->lookupTester = new UsageLookupContractTester($this->sqlUsageTracker, array($this, 'putUsages')); }
function execute() { global $wgConf; foreach ( $wgConf->getLocalDatabases() as $wiki ) { $lb = wfGetLB( $wiki ); $db = $lb->getConnection( DB_MASTER, array(), $wiki ); $count = intval( $db->selectField( 'job', 'COUNT(*)', '', __METHOD__ ) ); $this->output( "$wiki $count\n" ); $lb->reuseConnection( $db ); } }
/** * Check if the maximum lag of database slaves is higher that $maxLag, and * if it's the case, output an error message * * @param $maxLag int: maximum lag allowed for the request, as supplied by * the client * @return bool true if the request can continue */ function checkMaxLag($maxLag) { list($host, $lag) = wfGetLB()->getMaxLag(); if ($lag > $maxLag) { wfMaxlagError($host, $lag, $maxLag); return false; } else { return true; } }
/** * Static wrapper for EntityUsageTableBuilder::fillUsageTable * * @param DatabaseUpdater $dbUpdater * @param string $table */ public static function fillSubscriptionTable(DatabaseUpdater $dbUpdater, $table) { $primer = new ChangesSubscriptionTableBuilder(wfGetLB(), $table, 1000); $reporter = new ObservableMessageReporter(); $reporter->registerReporterCallback(function ($msg) use($dbUpdater) { $dbUpdater->output("\t{$msg}\n"); }); $primer->setProgressReporter($reporter); $primer->fillSubscriptionTable(); }
/** * @see EntityLookupTest::newEntityLoader() * * @return array array( EntityStore, EntityLookup ) */ protected function createStoreAndLookup() { // make sure the term index is empty to avoid conflicts. $wikibaseRepo = WikibaseRepo::getDefaultInstance(); $wikibaseRepo->getStore()->getTermIndex()->clear(); //NOTE: we want to test integration of WikiPageEntityRevisionLookup and WikiPageEntityStore here! $contentCodec = $wikibaseRepo->getEntityContentDataCodec(); $lookup = new WikiPageEntityRevisionLookup($contentCodec, new WikiPageEntityMetaDataLookup($this->getEntityIdParser(), false), false); $store = new WikiPageEntityStore(new EntityContentFactory($wikibaseRepo->getContentModelMappings()), new SqlIdGenerator(wfGetLB())); return array($store, $lookup); }
/** * Static wrapper for EntityUsageTableBuilder::fillUsageTable * * @param DatabaseUpdater $dbUpdater */ public static function fillUsageTable(DatabaseUpdater $dbUpdater) { $idParser = WikibaseClient::getDefaultInstance()->getEntityIdParser(); $primer = new EntityUsageTableBuilder($idParser, wfGetLB()); $reporter = new ObservableMessageReporter(); $reporter->registerReporterCallback(function ($msg) use($dbUpdater) { $dbUpdater->output("\t{$msg}\n"); }); $primer->setProgressReporter($reporter); $primer->fillUsageTable(); }
/** * Wrapper function for wfGetDB * * @param $db int Index of the connection to get * @param $groups mixed Query groups. * @param $wiki string|bool The wiki ID, or false for the current wiki * @return DatabaseBase */ public static function getDB($db, $groups = array(), $wiki = false) { global $wgEchoCluster; // Use the external db defined for Echo if ($wgEchoCluster) { $lb = wfGetLBFactory()->getExternalLB($wgEchoCluster, $wiki); } else { $lb = wfGetLB($wiki); } return $lb->getConnection($db, $groups, $wiki); }
public function syncDBs() { $lb = wfGetLB(); // bug 27975 - Don't try to wait for slaves if there are none // Prevents permission error when getting master position if ($lb->getServerCount() > 1) { $dbw = $lb->getConnection(DB_MASTER); $pos = $dbw->getMasterPos(); $lb->waitForAll($pos); } }
function run() { if (!empty($this->params['mainDbMasterPos'])) { wfGetLB()->waitFor($this->params['mainDbMasterPos']); } if (!empty($this->params['echoDbMasterPos'])) { global $wgEchoCluster; wfGetLBFactory()->getExternalLB($wgEchoCluster)->waitFor($this->params['echoDbMasterPos']); } EchoNotificationController::notify($this->event, false); return true; }
/** * Update the special pages localization cache */ public function rebuildLocalizationCache() { global $IP, $wgSpecialPageCacheUpdates, $wgQueryPages, $wgQueryCacheLimit, $wgDisableQueryPageUpdate; $dbw = wfGetDB(DB_MASTER); foreach ($wgSpecialPageCacheUpdates as $special => $call) { if (!is_callable($call)) { throw new \InvalidArgumentException("Uncallable function '{$call}' for special page {$special}"); } $start = wfTime(); call_user_func($call, $dbw); $end = wfTime(); $this->info(sprintf("%-30s completed in %.2fs", $special, $end - $start)); // Wait for the slave to catch up wfWaitForSlaves(); } // This is needed to initialise $wgQueryPages require_once "{$IP}/includes/QueryPage.php"; $disabledPages = $wgDisableQueryPageUpdate ? array_flip($wgDisableQueryPageUpdate) : []; foreach ($wgQueryPages as $page) { list($class, $special) = $page; $limit = isset($page[2]) ? $page[2] : $wgQueryCacheLimit; $queryPage = $this->getQueryPage($special, $class); if (array_key_exists($special, $disabledPages)) { // skip disabled pages $this->info(sprintf("%-30s disabled", $special)); continue; } if (!$queryPage->isExpensive()) { // don't bother with cheap pages $this->info(sprintf("%-30s skipped", $special)); continue; } $start = wfTime(); $num = $queryPage->recache($limit); $end = wfTime(); if ($num === false) { throw new \DBError($dbw, "database error"); } $this->info(sprintf("%-30s updated %d rows in %.2fs", $special, $num, $end - $start)); if (wfGetLB()->pingAll()) { // commit the changes if all connections are still open $dbw->commit(); } else { // Reopen any connections that have closed $count = 6; do { sleep(10); } while ($count-- > 0 && !wfGetLB()->pingAll()); } // Wait for the slave to catch up wfWaitForSlaves(); } }
public function syncDBs() { // FIXME: Copied from updateCollation.php, should be centralized somewhere $lb = wfGetLB(); // bug 27975 - Don't try to wait for slaves if there are none // Prevents permission error when getting master position if ($lb->getServerCount() > 1) { $dbw = $lb->getConnection(DB_MASTER); $pos = $dbw->getMasterPos(); $lb->waitForAll($pos); } }
function clearBlobs() { global $wgConf; echo "Clearing blobs...\n"; foreach ( $wgConf->getLocalDatabases() as $wiki ) { $lb = wfGetLB( $wiki ); $db = $lb->getConnection( DB_MASTER, array(), $wiki ); $db->query( "TRUNCATE TABLE " . $db->tableName( 'msg_resource' ), __METHOD__ ); $db->query( "TRUNCATE TABLE " . $db->tableName( 'msg_resource_links' ), __METHOD__ ); $lb->reuseConnection( $db ); } echo "Done.\n"; }
public function testGetSubscriptions() { $subscriptions = array(array('cs_subscriber_id' => 'enwiki', 'cs_entity_id' => 'P1'), array('cs_subscriber_id' => 'enwiki', 'cs_entity_id' => 'Q3'), array('cs_subscriber_id' => 'enwiki', 'cs_entity_id' => 'Q7'), array('cs_subscriber_id' => 'dewiki', 'cs_entity_id' => 'Q2')); $this->insertSubscriptions($subscriptions); $lookup = new SqlSubscriptionLookup(wfGetLB()); $subscriptions = $lookup->getSubscriptions('enwiki', array(new PropertyId('P1'), new ItemId('Q2'), new ItemId('Q7'), new PropertyId('P3'))); $actual = array_map(function (EntityId $id) { return $id->getSerialization(); }, $subscriptions); sort($actual); $expected = array('P1', 'Q7'); $this->assertEquals($expected, $actual); }
/** * @see LoggedUpdateMaintenance::doDBUpdates * * @return boolean */ public function doDBUpdates() { if (!defined('WBC_VERSION')) { $this->output("You need to have Wikibase enabled in order to use this maintenance script!\n\n"); exit; } $startPage = (int) $this->getOption('start-page', 0); $reporter = new ObservableMessageReporter(); $reporter->registerReporterCallback(array($this, 'report')); $builder = new EntityUsageTableBuilder(WikibaseClient::getDefaultInstance()->getEntityIdParser(), wfGetLB(), $this->mBatchSize); $builder->setProgressReporter($reporter); $builder->setExceptionHandler(new ReportingExceptionHandler($reporter)); $builder->fillUsageTable($startPage); return true; }
public function execute() { global $wgAllDBsAreLocalhost; if ($wgAllDBsAreLocalhost) { $host = 'localhost'; } elseif ($this->hasOption('group')) { $db = $this->getDB(DB_REPLICA, $this->getOption('group')); $host = $db->getServer(); } else { $lb = wfGetLB(); $i = $lb->getReaderIndex(); $host = $lb->getServerName($i); } $this->output("{$host}\n"); }
public function testLoadByRevisionId() { if (!defined('WB_VERSION')) { $this->markTestSkipped("Skipping because WikibaseClient doesn't have a local wb_changes table."); } list($expected) = $this->getEntityChanges(); $expected->setField('revision_id', 342342); $expected->setField('id', null); // Null the id as we save the same changes multiple times $changeStore = new SqlChangeStore(wfGetLB()); $changeStore->saveChange($expected); $lookup = new ChangeLookup(array('wikibase-item~remove' => 'Wikibase\\EntityChange'), wfWikiID()); $change = $lookup->loadByRevisionId(342342); $this->assertChangesEqual(array($expected), array($change)); }
public function testGetEntityIds() { $db = wfGetDB(DB_MASTER); $title11 = $this->makeTitle(11); $title22 = $this->makeTitle(22); $title99 = $this->makeTitle(99); $q11 = new ItemId('Q11'); $q22 = new ItemId('Q22'); $this->insertPageProps($db, 11, $q11); $this->insertPageProps($db, 22, $q22); $expected = array(11 => $q11, 22 => $q22); $lookup = new PagePropsEntityIdLookup(wfGetLB(), new BasicEntityIdParser()); $actual = $lookup->getEntityIds(array($title22, $title99, $title11)); ksort($actual); $this->assertEquals($expected, $actual); }
/** * Run a refreshLinks2 job * @return boolean success */ function run() { global $wgUpdateRowsPerJob; $linkCache = LinkCache::singleton(); $linkCache->clear(); if (is_null($this->title)) { $this->error = "refreshLinks2: Invalid title"; return false; } // Back compat for pre-r94435 jobs $table = isset($this->params['table']) ? $this->params['table'] : 'templatelinks'; // Avoid slave lag when fetching templates. // When the outermost job is run, we know that the caller that enqueued it must have // committed the relevant changes to the DB by now. At that point, record the master // position and pass it along as the job recursively breaks into smaller range jobs. // Hopefully, when leaf jobs are popped, the slaves will have reached that position. if (isset($this->params['masterPos'])) { $masterPos = $this->params['masterPos']; } elseif (wfGetLB()->getServerCount() > 1) { $masterPos = wfGetLB()->getMasterPos(); } else { $masterPos = false; } $tbc = $this->title->getBacklinkCache(); $jobs = array(); // jobs to insert if (isset($this->params['start']) && isset($this->params['end'])) { # This is a partition job to trigger the insertion of leaf jobs... $jobs = array_merge($jobs, $this->getSingleTitleJobs($table, $masterPos)); } else { # This is a base job to trigger the insertion of partitioned jobs... if ($tbc->getNumLinks($table, $wgUpdateRowsPerJob + 1) <= $wgUpdateRowsPerJob) { # Just directly insert the single per-title jobs $jobs = array_merge($jobs, $this->getSingleTitleJobs($table, $masterPos)); } else { # Insert the partition jobs to make per-title jobs foreach ($tbc->partition($table, $wgUpdateRowsPerJob) as $batch) { list($start, $end) = $batch; $jobs[] = new RefreshLinksJob2($this->title, array('table' => $table, 'start' => $start, 'end' => $end, 'masterPos' => $masterPos) + $this->getRootJobParams()); } } } if (count($jobs)) { JobQueueGroup::singleton()->push($jobs); } return true; }
public function run() { $page = WikiPage::newFromID($this->params['pageId'], WikiPage::READ_LATEST); if (!$page) { $this->setLastError("Could not find page #{$this->params['pageId']}"); return false; // deleted? } $dbw = wfGetDB(DB_MASTER); // Use a named lock so that jobs for this page see each others' changes $lockKey = "CategoryMembershipUpdates:{$page->getId()}"; $scopedLock = $dbw->getScopedLockAndFlush($lockKey, __METHOD__, 10); if (!$scopedLock) { $this->setLastError("Could not acquire lock '{$lockKey}'"); return false; } $dbr = wfGetDB(DB_SLAVE, ['recentchanges']); // Wait till the slave is caught up so that jobs for this page see each others' changes if (!wfGetLB()->safeWaitForMasterPos($dbr)) { $this->setLastError("Timed out while waiting for slave to catch up"); return false; } // Clear any stale REPEATABLE-READ snapshot $dbr->commit(__METHOD__, 'flush'); $cutoffUnix = wfTimestamp(TS_UNIX, $this->params['revTimestamp']); // Using ENQUEUE_FUDGE_SEC handles jobs inserted out of revision order due to the delay // between COMMIT and actual enqueueing of the CategoryMembershipChangeJob job. $cutoffUnix -= self::ENQUEUE_FUDGE_SEC; // Get the newest revision that has a SRC_CATEGORIZE row... $row = $dbr->selectRow(['revision', 'recentchanges'], ['rev_timestamp', 'rev_id'], ['rev_page' => $page->getId(), 'rev_timestamp >= ' . $dbr->addQuotes($dbr->timestamp($cutoffUnix))], __METHOD__, ['ORDER BY' => 'rev_timestamp DESC, rev_id DESC'], ['recentchanges' => ['INNER JOIN', ['rc_this_oldid = rev_id', 'rc_source' => RecentChange::SRC_CATEGORIZE, 'rc_cur_id = rev_page', 'rc_timestamp >= rev_timestamp']]]); // Only consider revisions newer than any such revision if ($row) { $cutoffUnix = wfTimestamp(TS_UNIX, $row->rev_timestamp); $lastRevId = (int) $row->rev_id; } else { $lastRevId = 0; } // Find revisions to this page made around and after this revision which lack category // notifications in recent changes. This lets jobs pick up were the last one left off. $encCutoff = $dbr->addQuotes($dbr->timestamp($cutoffUnix)); $res = $dbr->select('revision', Revision::selectFields(), ['rev_page' => $page->getId(), "rev_timestamp > {$encCutoff}" . " OR (rev_timestamp = {$encCutoff} AND rev_id > {$lastRevId})"], __METHOD__, ['ORDER BY' => 'rev_timestamp ASC, rev_id ASC']); // Apply all category updates in revision timestamp order foreach ($res as $row) { $this->notifyUpdatesForRevision($page, Revision::newFromRow($row)); } return true; }
public function execute() { $this->slaveIndexes = array(); for ($i = 1; $i < wfGetLB()->getServerCount(); $i++) { if (wfGetLB()->isNonZeroLoad($i)) { $this->slaveIndexes[] = $i; } } if ($this->hasArg()) { $this->desyncFixPage($this->getArg()); } else { $corrupt = $this->findPageLatestCorruption(); foreach ($corrupt as $id => $dummy) { $this->desyncFixPage($id); } } }