/**
  * Static wrapper for EntityUsageTableBuilder::fillUsageTable
  *
  * @param DatabaseUpdater $dbUpdater
  * @param string $table
  */
 public static function fillSubscriptionTable(DatabaseUpdater $dbUpdater, $table)
 {
     $primer = new ChangesSubscriptionTableBuilder(wfGetLB(), $table, 1000);
     $reporter = new ObservableMessageReporter();
     $reporter->registerReporterCallback(function ($msg) use($dbUpdater) {
         $dbUpdater->output("\t{$msg}\n");
     });
     $primer->setProgressReporter($reporter);
     $primer->fillSubscriptionTable();
 }
 /**
  * @return MessageReporter
  */
 private function newMessageReporter()
 {
     $reporter = new ObservableMessageReporter();
     $self = $this;
     // evil PHP 5.3 ;)
     $reporter->registerReporterCallback(function ($message) use($self) {
         $self->messages[] = $message;
     });
     return $reporter;
 }
 /**
  * Static wrapper for EntityUsageTableBuilder::fillUsageTable
  *
  * @param DatabaseUpdater $dbUpdater
  */
 public static function fillUsageTable(DatabaseUpdater $dbUpdater)
 {
     $idParser = WikibaseClient::getDefaultInstance()->getEntityIdParser();
     $primer = new EntityUsageTableBuilder($idParser, wfGetLB());
     $reporter = new ObservableMessageReporter();
     $reporter->registerReporterCallback(function ($msg) use($dbUpdater) {
         $dbUpdater->output("\t{$msg}\n");
     });
     $primer->setProgressReporter($reporter);
     $primer->fillUsageTable();
 }
 /**
  * @see LoggedUpdateMaintenance::doDBUpdates
  *
  * @return boolean
  */
 public function doDBUpdates()
 {
     if (!defined('WBC_VERSION')) {
         $this->output("You need to have Wikibase enabled in order to use this maintenance script!\n\n");
         exit;
     }
     $startPage = (int) $this->getOption('start-page', 0);
     $reporter = new ObservableMessageReporter();
     $reporter->registerReporterCallback(array($this, 'report'));
     $builder = new EntityUsageTableBuilder(WikibaseClient::getDefaultInstance()->getEntityIdParser(), wfGetLB(), $this->mBatchSize);
     $builder->setProgressReporter($reporter);
     $builder->setExceptionHandler(new ReportingExceptionHandler($reporter));
     $builder->fillUsageTable($startPage);
     return true;
 }
 public function testReportMessage()
 {
     $observedMessages = array();
     $observer = function ($message) use(&$observedMessages) {
         $observedMessages[] = $message;
     };
     $outerReporter = new ObservableMessageReporter();
     $innerReporter = new ObservableMessageReporter();
     $outerReporter->registerMessageReporter($innerReporter);
     $innerReporter->registerReporterCallback($observer);
     $innerReporter->reportMessage("one");
     $this->assertCount(1, $observedMessages);
     $innerReporter->reportMessage("two");
     $this->assertCount(2, $observedMessages);
 }
 /**
  * @see LoggedUpdateMaintenance::doDBUpdates
  *
  * @return bool
  */
 public function doDBUpdates()
 {
     if (!defined('WB_VERSION')) {
         $this->output("You need to have Wikibase enabled in order to use this maintenance script!\n\n");
         exit;
     }
     $batchSize = (int) $this->getOption('batch-size', 100);
     $rebuildAll = $this->getOption('rebuild-all', false);
     $reporter = new ObservableMessageReporter();
     $reporter->registerReporterCallback(array($this, 'report'));
     $wikibaseRepo = WikibaseRepo::getDefaultInstance();
     $builder = new EntityPerPageBuilder($wikibaseRepo->getStore()->newEntityPerPage(), $wikibaseRepo->getEntityIdParser(), $wikibaseRepo->getEntityNamespaceLookup(), $wikibaseRepo->getContentModelMappings());
     $builder->setReporter($reporter);
     $builder->setBatchSize($batchSize);
     $builder->setRebuildAll($rebuildAll);
     $builder->rebuild();
     return true;
 }
 /**
  * @see LoggedUpdateMaintenance::doDBUpdates
  *
  * @return bool
  */
 public function doDBUpdates()
 {
     if (!defined('WB_VERSION')) {
         $this->output("You need to have Wikibase enabled in order to use this maintenance script!\n\n");
         exit;
     }
     $reporter = new ObservableMessageReporter();
     $reporter->registerReporterCallback(array($this, 'report'));
     $table = WikibaseRepo::getDefaultInstance()->getStore()->getTermIndex();
     $builder = new TermSearchKeyBuilder($table);
     $builder->setReporter($reporter);
     $builder->setBatchSize((int) $this->getOption('batch-size', 100));
     $builder->setRebuildAll(!$this->getOption('only-missing', false));
     $builder->setFromId((int) $this->getOption('start-row', 1));
     $n = $builder->rebuildSearchKey();
     $this->output("Done. Updated {$n} search keys.\n");
     return true;
 }
 /**
  * @see LoggedUpdateMaintenance::doDBUpdates
  *
  * @throws EntityIdParsingException
  * @return bool
  */
 public function doDBUpdates()
 {
     if (!defined('WB_VERSION')) {
         $this->error("You need to have Wikibase enabled in order to use this maintenance script!", 1);
     }
     $idParser = WikibaseRepo::getDefaultInstance()->getEntityIdParser();
     $startItemOption = $this->getOption('start-item');
     $startItem = $startItemOption === null ? null : $idParser->parse($startItemOption);
     if ($startItem !== null && !$startItem instanceof ItemId) {
         throw new EntityIdParsingException('Not an Item ID: ' . $startItemOption);
     }
     $verbose = (bool) $this->getOption('verbose', false);
     $reporter = new ObservableMessageReporter();
     $reporter->registerReporterCallback(array($this, 'report'));
     $builder = new ChangesSubscriptionTableBuilder(wfGetLB(), 'wb_changes_subscription', $this->mBatchSize, $verbose ? 'verbose' : 'standard');
     $builder->setProgressReporter($reporter);
     $builder->setExceptionHandler(new ReportingExceptionHandler($reporter));
     $builder->fillSubscriptionTable($startItem);
     return true;
 }
 /**
  * @see LoggedUpdateMaintenance::doDBUpdates
  *
  * @return bool
  */
 public function doDBUpdates()
 {
     if (!defined('WB_VERSION')) {
         $this->output("You need to have Wikibase enabled in order to use this maintenance script!\n\n");
         exit;
     }
     $reporter = new ObservableMessageReporter();
     $reporter->registerReporterCallback(array($this, 'report'));
     $table = new PropertyInfoTable(false);
     $wikibaseRepo = WikibaseRepo::getDefaultInstance();
     $entityLookup = $wikibaseRepo->getEntityLookup();
     $propertyInfoBuilder = $wikibaseRepo->newPropertyInfoBuilder();
     $builder = new PropertyInfoTableBuilder($table, $entityLookup, $propertyInfoBuilder);
     $builder->setReporter($reporter);
     $builder->setBatchSize((int) $this->getOption('batch-size', 100));
     $builder->setRebuildAll($this->getOption('rebuild-all', false));
     $builder->setFromId((int) $this->getOption('start-row', 1));
     $n = $builder->rebuildPropertyInfo();
     $this->output("Done. Updated {$n} property info entries.\n");
     return true;
 }
Пример #10
0
 /**
  * @see Maintenance::execute
  */
 public function execute()
 {
     if (!defined('WB_VERSION')) {
         $this->output("You need to have Wikibase enabled in order to use this maintenance script!\n\n");
         exit;
     }
     $batchSize = (int) $this->getOption('batch-size', 100);
     $reporter = new ObservableMessageReporter();
     $reporter->registerReporterCallback(array($this, 'report'));
     $siteLinkTable = new SiteLinkTable('wb_items_per_site', false);
     $wikibaseRepo = WikibaseRepo::getDefaultInstance();
     // Use an uncached EntityLookup here to avoid memory leaks
     $entityLookup = $wikibaseRepo->getEntityLookup('uncached');
     $store = $wikibaseRepo->getStore();
     $builder = new ItemsPerSiteBuilder($siteLinkTable, $entityLookup, $store->getEntityPrefetcher());
     $builder->setReporter($reporter);
     $builder->setBatchSize($batchSize);
     $entityPerPage = $store->newEntityPerPage();
     $stream = new EntityPerPageIdPager($entityPerPage, 'item');
     // Now <s>kill</s> fix the table
     $builder->rebuild($stream);
 }
 /**
  * @see LoggedUpdateMaintenance::doDBUpdates
  *
  * @throws EntityIdParsingException
  * @return bool
  */
 public function execute()
 {
     if (!defined('WBC_VERSION')) {
         $this->error('You need to have WikibaseClient enabled in order to use this maintenance script!', 1);
     }
     $wikibaseClient = WikibaseClient::getDefaultInstance();
     $settings = $wikibaseClient->getSettings();
     $repoDB = $settings->getSetting('repoDatabase');
     $clientId = $settings->getSetting('siteGlobalID');
     $idParser = $wikibaseClient->getEntityIdParser();
     $startItemOption = $this->getOption('start-item');
     $startItem = $startItemOption === null ? null : $idParser->parse($startItemOption);
     $reporter = new ObservableMessageReporter();
     $reporter->registerReporterCallback(array($this, 'report'));
     $updater = new BulkSubscriptionUpdater(new ConsistentReadConnectionManager(wfGetLB()), new ConsistentReadConnectionManager(wfGetLB($repoDB), $repoDB), $clientId, $repoDB, $this->mBatchSize);
     $updater->setProgressReporter($reporter);
     $updater->setExceptionHandler(new ReportingExceptionHandler($reporter));
     if ($this->getOption('purge')) {
         $updater->purgeSubscriptions($startItem);
     }
     $updater->updateSubscriptions($startItem);
     return true;
 }
Пример #12
0
 /**
  * Initializes members from command line options and configuration settings.
  *
  * @param ChangeLookup $changeLookup
  * @param SettingsArray $settings
  *
  * @return ChangeDispatcher
  * @throws MWException
  */
 private function newChangeDispatcher(ChangeLookup $changeLookup, SettingsArray $settings)
 {
     $repoDB = $settings->getSetting('changesDatabase');
     $clientWikis = $settings->getSetting('localClientDatabases');
     $batchChunkFactor = $settings->getSetting('dispatchBatchChunkFactor');
     $batchCacheFactor = $settings->getSetting('dispatchBatchCacheFactor');
     $subscriptionLookupMode = $settings->getSetting('subscriptionLookupMode');
     $batchSize = (int) $this->getOption('batch-size', 1000);
     $maxChunks = (int) $this->getOption('max-chunks', 15);
     $dispatchInterval = (int) $this->getOption('dispatch-interval', 60);
     $lockGraceInterval = (int) $this->getOption('lock-grace-interval', 60);
     $randomness = (int) $this->getOption('randomness', 10);
     $this->verbose = $this->getOption('verbose', false);
     $cacheChunkSize = $batchSize * $batchChunkFactor;
     $cacheSize = $cacheChunkSize * $batchCacheFactor;
     $changesCache = new ChunkCache($changeLookup, $cacheChunkSize, $cacheSize);
     // make sure we have a mapping from siteId to database name in clientWikis:
     foreach ($clientWikis as $siteID => $dbName) {
         if (is_int($siteID)) {
             unset($clientWikis[$siteID]);
             $clientWikis[$dbName] = $dbName;
         }
     }
     if (empty($clientWikis)) {
         throw new MWException("No client wikis configured! Please set \$wgWBRepoSettings['localClientDatabases'].");
     }
     $reporter = new ObservableMessageReporter();
     $self = $this;
     // PHP 5.3...
     $reporter->registerReporterCallback(function ($message) use($self) {
         $self->log($message);
     });
     $coordinator = new SqlChangeDispatchCoordinator($repoDB, $clientWikis);
     $coordinator->setMessageReporter($reporter);
     $coordinator->setBatchSize($batchSize);
     $coordinator->setDispatchInterval($dispatchInterval);
     $coordinator->setLockGraceInterval($lockGraceInterval);
     $coordinator->setRandomness($randomness);
     $notificationSender = new JobQueueChangeNotificationSender($repoDB, $clientWikis);
     $subscriptionLookup = $this->getSubscriptionLookup($repoDB, $subscriptionLookupMode);
     $dispatcher = new ChangeDispatcher($coordinator, $notificationSender, $changesCache, $subscriptionLookup);
     $dispatcher->setMessageReporter($reporter);
     $dispatcher->setExceptionHandler(new ReportingExceptionHandler($reporter));
     $dispatcher->setBatchSize($batchSize);
     $dispatcher->setMaxChunks($maxChunks);
     $dispatcher->setBatchChunkFactor($batchChunkFactor);
     $dispatcher->setVerbose($this->verbose);
     return $dispatcher;
 }
Пример #13
0
 /**
  * @return MessageReporter
  */
 private function newMessageReporter()
 {
     $reporter = new ObservableMessageReporter();
     $reporter->registerReporterCallback(array($this, 'log'));
     return $reporter;
 }
Пример #14
0
 /**
  * Do the actual work. All child classes will need to implement this
  */
 public function execute()
 {
     //TODO: more validation for options
     $entityType = $this->getOption('entity-type');
     $shardingFactor = (int) $this->getOption('sharding-factor', 1);
     $shard = (int) $this->getOption('shard', 0);
     $batchSize = (int) $this->getOption('batch-size', 100);
     $limit = (int) $this->getOption('limit', 0);
     //TODO: Allow injection of an OutputStream for logging
     $this->openLogFile($this->getOption('log', 'php://stderr'));
     $outFile = $this->getOption('output', 'php://stdout');
     if ($outFile === '-') {
         $outFile = 'php://stdout';
     }
     $output = fopen($outFile, 'w');
     //TODO: Allow injection of an OutputStream
     if (!$output) {
         throw new MWException('Failed to open ' . $outFile . '!');
     }
     if ($this->hasOption('list-file')) {
         $this->logMessage("Dumping entities listed in " . $this->getOption('list-file'));
     }
     if ($entityType) {
         $this->logMessage("Dumping entities of type {$entityType}");
     }
     if ($shardingFactor) {
         $this->logMessage("Dumping shard {$shard}/{$shardingFactor}");
     }
     $dumper = $this->createDumper($output);
     $dumper->setLimit($limit);
     $progressReporter = new ObservableMessageReporter();
     $progressReporter->registerReporterCallback(array($this, 'logMessage'));
     $dumper->setProgressReporter($progressReporter);
     $exceptionReporter = new ReportingExceptionHandler($progressReporter);
     $dumper->setExceptionHandler($exceptionReporter);
     //NOTE: we filter for $entityType twice: filtering in the DB is efficient,
     //      but filtering in the dumper is needed when working from a list file.
     $dumper->setShardingFilter($shardingFactor, $shard);
     $dumper->setEntityTypeFilter($entityType);
     $dumper->setBatchSize($batchSize);
     $idStream = $this->makeIdStream($entityType, $exceptionReporter);
     \MediaWiki\suppressWarnings();
     $dumper->generateDump($idStream);
     \MediaWiki\restoreWarnings();
     if ($idStream instanceof Disposable) {
         // close stream / free resources
         $idStream->dispose();
     }
     $this->closeLogFile();
 }
 /**
  * Initializes members from command line options and configuration settings.
  *
  * @param string[] $clientWikis A mapping of client wiki site IDs to logical database names.
  * @param ChangeLookup $changeLookup
  * @param SettingsArray $settings
  *
  * @return ChangeDispatcher
  */
 private function newChangeDispatcher(array $clientWikis, ChangeLookup $changeLookup, SettingsArray $settings)
 {
     $repoID = wfWikiID();
     $repoDB = $settings->getSetting('changesDatabase');
     $batchChunkFactor = $settings->getSetting('dispatchBatchChunkFactor');
     $batchCacheFactor = $settings->getSetting('dispatchBatchCacheFactor');
     $batchSize = (int) $this->getOption('batch-size', 1000);
     $maxChunks = (int) $this->getOption('max-chunks', 15);
     $dispatchInterval = (int) $this->getOption('dispatch-interval', 60);
     $lockGraceInterval = (int) $this->getOption('lock-grace-interval', 60);
     $randomness = (int) $this->getOption('randomness', 10);
     $this->verbose = $this->getOption('verbose', false);
     $cacheChunkSize = $batchSize * $batchChunkFactor;
     $cacheSize = $cacheChunkSize * $batchCacheFactor;
     $changesCache = new ChunkCache($changeLookup, $cacheChunkSize, $cacheSize);
     $reporter = new ObservableMessageReporter();
     $self = $this;
     // PHP 5.3...
     $reporter->registerReporterCallback(function ($message) use($self) {
         $self->log($message);
     });
     $coordinator = new SqlChangeDispatchCoordinator($repoDB, $repoID);
     $coordinator->setMessageReporter($reporter);
     $coordinator->setBatchSize($batchSize);
     $coordinator->setDispatchInterval($dispatchInterval);
     $coordinator->setLockGraceInterval($lockGraceInterval);
     $coordinator->setRandomness($randomness);
     $notificationSender = new JobQueueChangeNotificationSender($repoDB, $clientWikis);
     $subscriptionLookup = new SqlSubscriptionLookup(wfGetLB());
     $dispatcher = new ChangeDispatcher($coordinator, $notificationSender, $changesCache, $subscriptionLookup);
     $dispatcher->setMessageReporter($reporter);
     $dispatcher->setExceptionHandler(new ReportingExceptionHandler($reporter));
     $dispatcher->setBatchSize($batchSize);
     $dispatcher->setMaxChunks($maxChunks);
     $dispatcher->setBatchChunkFactor($batchChunkFactor);
     $dispatcher->setVerbose($this->verbose);
     return $dispatcher;
 }
 /**
  * Wrapper for invoking PropertyInfoTableBuilder from DatabaseUpdater
  * during a database update.
  *
  * @param DatabaseUpdater $updater
  */
 public static function rebuildPropertyInfo(DatabaseUpdater $updater)
 {
     $reporter = new ObservableMessageReporter();
     $reporter->registerReporterCallback(function ($msg) use($updater) {
         $updater->output("..." . $msg . "\n");
     });
     $table = new PropertyInfoTable(false);
     $wikibaseRepo = WikibaseRepo::getDefaultInstance();
     $contentCodec = $wikibaseRepo->getEntityContentDataCodec();
     $propertyInfoBuilder = $wikibaseRepo->newPropertyInfoBuilder();
     $wikiPageEntityLookup = new WikiPageEntityRevisionLookup($contentCodec, new WikiPageEntityMetaDataLookup($wikibaseRepo->getEntityIdParser()), false);
     $cachingEntityLookup = new CachingEntityRevisionLookup($wikiPageEntityLookup, new HashBagOStuff());
     $entityLookup = new RevisionBasedEntityLookup($cachingEntityLookup);
     $builder = new PropertyInfoTableBuilder($table, $entityLookup, $propertyInfoBuilder);
     $builder->setReporter($reporter);
     $builder->setUseTransactions(false);
     $updater->output('Populating ' . $table->getTableName() . "\n");
     $builder->rebuildPropertyInfo();
 }