/** * @return MessageReporter */ private function newMessageReporter() { $reporter = new ObservableMessageReporter(); $self = $this; // evil PHP 5.3 ;) $reporter->registerReporterCallback(function ($message) use($self) { $self->messages[] = $message; }); return $reporter; }
/** * Static wrapper for EntityUsageTableBuilder::fillUsageTable * * @param DatabaseUpdater $dbUpdater * @param string $table */ public static function fillSubscriptionTable(DatabaseUpdater $dbUpdater, $table) { $primer = new ChangesSubscriptionTableBuilder(wfGetLB(), $table, 1000); $reporter = new ObservableMessageReporter(); $reporter->registerReporterCallback(function ($msg) use($dbUpdater) { $dbUpdater->output("\t{$msg}\n"); }); $primer->setProgressReporter($reporter); $primer->fillSubscriptionTable(); }
/** * Static wrapper for EntityUsageTableBuilder::fillUsageTable * * @param DatabaseUpdater $dbUpdater */ public static function fillUsageTable(DatabaseUpdater $dbUpdater) { $idParser = WikibaseClient::getDefaultInstance()->getEntityIdParser(); $primer = new EntityUsageTableBuilder($idParser, wfGetLB()); $reporter = new ObservableMessageReporter(); $reporter->registerReporterCallback(function ($msg) use($dbUpdater) { $dbUpdater->output("\t{$msg}\n"); }); $primer->setProgressReporter($reporter); $primer->fillUsageTable(); }
/** * @see LoggedUpdateMaintenance::doDBUpdates * * @return boolean */ public function doDBUpdates() { if (!defined('WBC_VERSION')) { $this->output("You need to have Wikibase enabled in order to use this maintenance script!\n\n"); exit; } $startPage = (int) $this->getOption('start-page', 0); $reporter = new ObservableMessageReporter(); $reporter->registerReporterCallback(array($this, 'report')); $builder = new EntityUsageTableBuilder(WikibaseClient::getDefaultInstance()->getEntityIdParser(), wfGetLB(), $this->mBatchSize); $builder->setProgressReporter($reporter); $builder->setExceptionHandler(new ReportingExceptionHandler($reporter)); $builder->fillUsageTable($startPage); return true; }
/** * @see LoggedUpdateMaintenance::doDBUpdates * * @return bool */ public function doDBUpdates() { if (!defined('WB_VERSION')) { $this->output("You need to have Wikibase enabled in order to use this maintenance script!\n\n"); exit; } $batchSize = (int) $this->getOption('batch-size', 100); $rebuildAll = $this->getOption('rebuild-all', false); $reporter = new ObservableMessageReporter(); $reporter->registerReporterCallback(array($this, 'report')); $wikibaseRepo = WikibaseRepo::getDefaultInstance(); $builder = new EntityPerPageBuilder($wikibaseRepo->getStore()->newEntityPerPage(), $wikibaseRepo->getEntityIdParser(), $wikibaseRepo->getEntityNamespaceLookup(), $wikibaseRepo->getContentModelMappings()); $builder->setReporter($reporter); $builder->setBatchSize($batchSize); $builder->setRebuildAll($rebuildAll); $builder->rebuild(); return true; }
/** * @see LoggedUpdateMaintenance::doDBUpdates * * @return bool */ public function doDBUpdates() { if (!defined('WB_VERSION')) { $this->output("You need to have Wikibase enabled in order to use this maintenance script!\n\n"); exit; } $reporter = new ObservableMessageReporter(); $reporter->registerReporterCallback(array($this, 'report')); $table = WikibaseRepo::getDefaultInstance()->getStore()->getTermIndex(); $builder = new TermSearchKeyBuilder($table); $builder->setReporter($reporter); $builder->setBatchSize((int) $this->getOption('batch-size', 100)); $builder->setRebuildAll(!$this->getOption('only-missing', false)); $builder->setFromId((int) $this->getOption('start-row', 1)); $n = $builder->rebuildSearchKey(); $this->output("Done. Updated {$n} search keys.\n"); return true; }
/** * @see LoggedUpdateMaintenance::doDBUpdates * * @throws EntityIdParsingException * @return bool */ public function doDBUpdates() { if (!defined('WB_VERSION')) { $this->error("You need to have Wikibase enabled in order to use this maintenance script!", 1); } $idParser = WikibaseRepo::getDefaultInstance()->getEntityIdParser(); $startItemOption = $this->getOption('start-item'); $startItem = $startItemOption === null ? null : $idParser->parse($startItemOption); if ($startItem !== null && !$startItem instanceof ItemId) { throw new EntityIdParsingException('Not an Item ID: ' . $startItemOption); } $verbose = (bool) $this->getOption('verbose', false); $reporter = new ObservableMessageReporter(); $reporter->registerReporterCallback(array($this, 'report')); $builder = new ChangesSubscriptionTableBuilder(wfGetLB(), 'wb_changes_subscription', $this->mBatchSize, $verbose ? 'verbose' : 'standard'); $builder->setProgressReporter($reporter); $builder->setExceptionHandler(new ReportingExceptionHandler($reporter)); $builder->fillSubscriptionTable($startItem); return true; }
/** * @see LoggedUpdateMaintenance::doDBUpdates * * @return bool */ public function doDBUpdates() { if (!defined('WB_VERSION')) { $this->output("You need to have Wikibase enabled in order to use this maintenance script!\n\n"); exit; } $reporter = new ObservableMessageReporter(); $reporter->registerReporterCallback(array($this, 'report')); $table = new PropertyInfoTable(false); $wikibaseRepo = WikibaseRepo::getDefaultInstance(); $entityLookup = $wikibaseRepo->getEntityLookup(); $propertyInfoBuilder = $wikibaseRepo->newPropertyInfoBuilder(); $builder = new PropertyInfoTableBuilder($table, $entityLookup, $propertyInfoBuilder); $builder->setReporter($reporter); $builder->setBatchSize((int) $this->getOption('batch-size', 100)); $builder->setRebuildAll($this->getOption('rebuild-all', false)); $builder->setFromId((int) $this->getOption('start-row', 1)); $n = $builder->rebuildPropertyInfo(); $this->output("Done. Updated {$n} property info entries.\n"); return true; }
/** * @see Maintenance::execute */ public function execute() { if (!defined('WB_VERSION')) { $this->output("You need to have Wikibase enabled in order to use this maintenance script!\n\n"); exit; } $batchSize = (int) $this->getOption('batch-size', 100); $reporter = new ObservableMessageReporter(); $reporter->registerReporterCallback(array($this, 'report')); $siteLinkTable = new SiteLinkTable('wb_items_per_site', false); $wikibaseRepo = WikibaseRepo::getDefaultInstance(); // Use an uncached EntityLookup here to avoid memory leaks $entityLookup = $wikibaseRepo->getEntityLookup('uncached'); $store = $wikibaseRepo->getStore(); $builder = new ItemsPerSiteBuilder($siteLinkTable, $entityLookup, $store->getEntityPrefetcher()); $builder->setReporter($reporter); $builder->setBatchSize($batchSize); $entityPerPage = $store->newEntityPerPage(); $stream = new EntityPerPageIdPager($entityPerPage, 'item'); // Now <s>kill</s> fix the table $builder->rebuild($stream); }
/** * @see LoggedUpdateMaintenance::doDBUpdates * * @throws EntityIdParsingException * @return bool */ public function execute() { if (!defined('WBC_VERSION')) { $this->error('You need to have WikibaseClient enabled in order to use this maintenance script!', 1); } $wikibaseClient = WikibaseClient::getDefaultInstance(); $settings = $wikibaseClient->getSettings(); $repoDB = $settings->getSetting('repoDatabase'); $clientId = $settings->getSetting('siteGlobalID'); $idParser = $wikibaseClient->getEntityIdParser(); $startItemOption = $this->getOption('start-item'); $startItem = $startItemOption === null ? null : $idParser->parse($startItemOption); $reporter = new ObservableMessageReporter(); $reporter->registerReporterCallback(array($this, 'report')); $updater = new BulkSubscriptionUpdater(new ConsistentReadConnectionManager(wfGetLB()), new ConsistentReadConnectionManager(wfGetLB($repoDB), $repoDB), $clientId, $repoDB, $this->mBatchSize); $updater->setProgressReporter($reporter); $updater->setExceptionHandler(new ReportingExceptionHandler($reporter)); if ($this->getOption('purge')) { $updater->purgeSubscriptions($startItem); } $updater->updateSubscriptions($startItem); return true; }
/** * Initializes members from command line options and configuration settings. * * @param ChangeLookup $changeLookup * @param SettingsArray $settings * * @return ChangeDispatcher * @throws MWException */ private function newChangeDispatcher(ChangeLookup $changeLookup, SettingsArray $settings) { $repoDB = $settings->getSetting('changesDatabase'); $clientWikis = $settings->getSetting('localClientDatabases'); $batchChunkFactor = $settings->getSetting('dispatchBatchChunkFactor'); $batchCacheFactor = $settings->getSetting('dispatchBatchCacheFactor'); $subscriptionLookupMode = $settings->getSetting('subscriptionLookupMode'); $batchSize = (int) $this->getOption('batch-size', 1000); $maxChunks = (int) $this->getOption('max-chunks', 15); $dispatchInterval = (int) $this->getOption('dispatch-interval', 60); $lockGraceInterval = (int) $this->getOption('lock-grace-interval', 60); $randomness = (int) $this->getOption('randomness', 10); $this->verbose = $this->getOption('verbose', false); $cacheChunkSize = $batchSize * $batchChunkFactor; $cacheSize = $cacheChunkSize * $batchCacheFactor; $changesCache = new ChunkCache($changeLookup, $cacheChunkSize, $cacheSize); // make sure we have a mapping from siteId to database name in clientWikis: foreach ($clientWikis as $siteID => $dbName) { if (is_int($siteID)) { unset($clientWikis[$siteID]); $clientWikis[$dbName] = $dbName; } } if (empty($clientWikis)) { throw new MWException("No client wikis configured! Please set \$wgWBRepoSettings['localClientDatabases']."); } $reporter = new ObservableMessageReporter(); $self = $this; // PHP 5.3... $reporter->registerReporterCallback(function ($message) use($self) { $self->log($message); }); $coordinator = new SqlChangeDispatchCoordinator($repoDB, $clientWikis); $coordinator->setMessageReporter($reporter); $coordinator->setBatchSize($batchSize); $coordinator->setDispatchInterval($dispatchInterval); $coordinator->setLockGraceInterval($lockGraceInterval); $coordinator->setRandomness($randomness); $notificationSender = new JobQueueChangeNotificationSender($repoDB, $clientWikis); $subscriptionLookup = $this->getSubscriptionLookup($repoDB, $subscriptionLookupMode); $dispatcher = new ChangeDispatcher($coordinator, $notificationSender, $changesCache, $subscriptionLookup); $dispatcher->setMessageReporter($reporter); $dispatcher->setExceptionHandler(new ReportingExceptionHandler($reporter)); $dispatcher->setBatchSize($batchSize); $dispatcher->setMaxChunks($maxChunks); $dispatcher->setBatchChunkFactor($batchChunkFactor); $dispatcher->setVerbose($this->verbose); return $dispatcher; }
/** * @return MessageReporter */ private function newMessageReporter() { $reporter = new ObservableMessageReporter(); $reporter->registerReporterCallback(array($this, 'log')); return $reporter; }
/** * Do the actual work. All child classes will need to implement this */ public function execute() { //TODO: more validation for options $entityType = $this->getOption('entity-type'); $shardingFactor = (int) $this->getOption('sharding-factor', 1); $shard = (int) $this->getOption('shard', 0); $batchSize = (int) $this->getOption('batch-size', 100); $limit = (int) $this->getOption('limit', 0); //TODO: Allow injection of an OutputStream for logging $this->openLogFile($this->getOption('log', 'php://stderr')); $outFile = $this->getOption('output', 'php://stdout'); if ($outFile === '-') { $outFile = 'php://stdout'; } $output = fopen($outFile, 'w'); //TODO: Allow injection of an OutputStream if (!$output) { throw new MWException('Failed to open ' . $outFile . '!'); } if ($this->hasOption('list-file')) { $this->logMessage("Dumping entities listed in " . $this->getOption('list-file')); } if ($entityType) { $this->logMessage("Dumping entities of type {$entityType}"); } if ($shardingFactor) { $this->logMessage("Dumping shard {$shard}/{$shardingFactor}"); } $dumper = $this->createDumper($output); $dumper->setLimit($limit); $progressReporter = new ObservableMessageReporter(); $progressReporter->registerReporterCallback(array($this, 'logMessage')); $dumper->setProgressReporter($progressReporter); $exceptionReporter = new ReportingExceptionHandler($progressReporter); $dumper->setExceptionHandler($exceptionReporter); //NOTE: we filter for $entityType twice: filtering in the DB is efficient, // but filtering in the dumper is needed when working from a list file. $dumper->setShardingFilter($shardingFactor, $shard); $dumper->setEntityTypeFilter($entityType); $dumper->setBatchSize($batchSize); $idStream = $this->makeIdStream($entityType, $exceptionReporter); \MediaWiki\suppressWarnings(); $dumper->generateDump($idStream); \MediaWiki\restoreWarnings(); if ($idStream instanceof Disposable) { // close stream / free resources $idStream->dispose(); } $this->closeLogFile(); }
/** * Initializes members from command line options and configuration settings. * * @param string[] $clientWikis A mapping of client wiki site IDs to logical database names. * @param ChangeLookup $changeLookup * @param SettingsArray $settings * * @return ChangeDispatcher */ private function newChangeDispatcher(array $clientWikis, ChangeLookup $changeLookup, SettingsArray $settings) { $repoID = wfWikiID(); $repoDB = $settings->getSetting('changesDatabase'); $batchChunkFactor = $settings->getSetting('dispatchBatchChunkFactor'); $batchCacheFactor = $settings->getSetting('dispatchBatchCacheFactor'); $batchSize = (int) $this->getOption('batch-size', 1000); $maxChunks = (int) $this->getOption('max-chunks', 15); $dispatchInterval = (int) $this->getOption('dispatch-interval', 60); $lockGraceInterval = (int) $this->getOption('lock-grace-interval', 60); $randomness = (int) $this->getOption('randomness', 10); $this->verbose = $this->getOption('verbose', false); $cacheChunkSize = $batchSize * $batchChunkFactor; $cacheSize = $cacheChunkSize * $batchCacheFactor; $changesCache = new ChunkCache($changeLookup, $cacheChunkSize, $cacheSize); $reporter = new ObservableMessageReporter(); $self = $this; // PHP 5.3... $reporter->registerReporterCallback(function ($message) use($self) { $self->log($message); }); $coordinator = new SqlChangeDispatchCoordinator($repoDB, $repoID); $coordinator->setMessageReporter($reporter); $coordinator->setBatchSize($batchSize); $coordinator->setDispatchInterval($dispatchInterval); $coordinator->setLockGraceInterval($lockGraceInterval); $coordinator->setRandomness($randomness); $notificationSender = new JobQueueChangeNotificationSender($repoDB, $clientWikis); $subscriptionLookup = new SqlSubscriptionLookup(wfGetLB()); $dispatcher = new ChangeDispatcher($coordinator, $notificationSender, $changesCache, $subscriptionLookup); $dispatcher->setMessageReporter($reporter); $dispatcher->setExceptionHandler(new ReportingExceptionHandler($reporter)); $dispatcher->setBatchSize($batchSize); $dispatcher->setMaxChunks($maxChunks); $dispatcher->setBatchChunkFactor($batchChunkFactor); $dispatcher->setVerbose($this->verbose); return $dispatcher; }
/** * Wrapper for invoking PropertyInfoTableBuilder from DatabaseUpdater * during a database update. * * @param DatabaseUpdater $updater */ public static function rebuildPropertyInfo(DatabaseUpdater $updater) { $reporter = new ObservableMessageReporter(); $reporter->registerReporterCallback(function ($msg) use($updater) { $updater->output("..." . $msg . "\n"); }); $table = new PropertyInfoTable(false); $wikibaseRepo = WikibaseRepo::getDefaultInstance(); $contentCodec = $wikibaseRepo->getEntityContentDataCodec(); $propertyInfoBuilder = $wikibaseRepo->newPropertyInfoBuilder(); $wikiPageEntityLookup = new WikiPageEntityRevisionLookup($contentCodec, new WikiPageEntityMetaDataLookup($wikibaseRepo->getEntityIdParser()), false); $cachingEntityLookup = new CachingEntityRevisionLookup($wikiPageEntityLookup, new HashBagOStuff()); $entityLookup = new RevisionBasedEntityLookup($cachingEntityLookup); $builder = new PropertyInfoTableBuilder($table, $entityLookup, $propertyInfoBuilder); $builder->setReporter($reporter); $builder->setUseTransactions(false); $updater->output('Populating ' . $table->getTableName() . "\n"); $builder->rebuildPropertyInfo(); }