/**
* Static wrapper for EntityUsageTableBuilder::fillUsageTable
*
* @param DatabaseUpdater $dbUpdater
* @param string $table
*/
public static function fillSubscriptionTable(DatabaseUpdater $dbUpdater, $table)
{
$primer = new ChangesSubscriptionTableBuilder(wfGetLB(), $table, 1000);
$reporter = new ObservableMessageReporter();
$reporter->registerReporterCallback(function ($msg) use($dbUpdater) {
$dbUpdater->output("\t{$msg}\n");
});
$primer->setProgressReporter($reporter);
$primer->fillSubscriptionTable();
}
/**
* @return MessageReporter
*/
private function newMessageReporter()
{
$reporter = new ObservableMessageReporter();
$self = $this;
// evil PHP 5.3 ;)
$reporter->registerReporterCallback(function ($message) use($self) {
$self->messages[] = $message;
});
return $reporter;
}
/**
* Static wrapper for EntityUsageTableBuilder::fillUsageTable
*
* @param DatabaseUpdater $dbUpdater
*/
public static function fillUsageTable(DatabaseUpdater $dbUpdater)
{
$idParser = WikibaseClient::getDefaultInstance()->getEntityIdParser();
$primer = new EntityUsageTableBuilder($idParser, wfGetLB());
$reporter = new ObservableMessageReporter();
$reporter->registerReporterCallback(function ($msg) use($dbUpdater) {
$dbUpdater->output("\t{$msg}\n");
});
$primer->setProgressReporter($reporter);
$primer->fillUsageTable();
}
/**
* @see LoggedUpdateMaintenance::doDBUpdates
*
* @return boolean
*/
public function doDBUpdates()
{
if (!defined('WBC_VERSION')) {
$this->output("You need to have Wikibase enabled in order to use this maintenance script!\n\n");
exit;
}
$startPage = (int) $this->getOption('start-page', 0);
$reporter = new ObservableMessageReporter();
$reporter->registerReporterCallback(array($this, 'report'));
$builder = new EntityUsageTableBuilder(WikibaseClient::getDefaultInstance()->getEntityIdParser(), wfGetLB(), $this->mBatchSize);
$builder->setProgressReporter($reporter);
$builder->setExceptionHandler(new ReportingExceptionHandler($reporter));
$builder->fillUsageTable($startPage);
return true;
}
public function testReportMessage()
{
$observedMessages = array();
$observer = function ($message) use(&$observedMessages) {
$observedMessages[] = $message;
};
$outerReporter = new ObservableMessageReporter();
$innerReporter = new ObservableMessageReporter();
$outerReporter->registerMessageReporter($innerReporter);
$innerReporter->registerReporterCallback($observer);
$innerReporter->reportMessage("one");
$this->assertCount(1, $observedMessages);
$innerReporter->reportMessage("two");
$this->assertCount(2, $observedMessages);
}
/**
* @see LoggedUpdateMaintenance::doDBUpdates
*
* @return bool
*/
public function doDBUpdates()
{
if (!defined('WB_VERSION')) {
$this->output("You need to have Wikibase enabled in order to use this maintenance script!\n\n");
exit;
}
$batchSize = (int) $this->getOption('batch-size', 100);
$rebuildAll = $this->getOption('rebuild-all', false);
$reporter = new ObservableMessageReporter();
$reporter->registerReporterCallback(array($this, 'report'));
$wikibaseRepo = WikibaseRepo::getDefaultInstance();
$builder = new EntityPerPageBuilder($wikibaseRepo->getStore()->newEntityPerPage(), $wikibaseRepo->getEntityIdParser(), $wikibaseRepo->getEntityNamespaceLookup(), $wikibaseRepo->getContentModelMappings());
$builder->setReporter($reporter);
$builder->setBatchSize($batchSize);
$builder->setRebuildAll($rebuildAll);
$builder->rebuild();
return true;
}
/**
* @see LoggedUpdateMaintenance::doDBUpdates
*
* @return bool
*/
public function doDBUpdates()
{
if (!defined('WB_VERSION')) {
$this->output("You need to have Wikibase enabled in order to use this maintenance script!\n\n");
exit;
}
$reporter = new ObservableMessageReporter();
$reporter->registerReporterCallback(array($this, 'report'));
$table = WikibaseRepo::getDefaultInstance()->getStore()->getTermIndex();
$builder = new TermSearchKeyBuilder($table);
$builder->setReporter($reporter);
$builder->setBatchSize((int) $this->getOption('batch-size', 100));
$builder->setRebuildAll(!$this->getOption('only-missing', false));
$builder->setFromId((int) $this->getOption('start-row', 1));
$n = $builder->rebuildSearchKey();
$this->output("Done. Updated {$n} search keys.\n");
return true;
}
/**
* @see LoggedUpdateMaintenance::doDBUpdates
*
* @throws EntityIdParsingException
* @return bool
*/
public function doDBUpdates()
{
if (!defined('WB_VERSION')) {
$this->error("You need to have Wikibase enabled in order to use this maintenance script!", 1);
}
$idParser = WikibaseRepo::getDefaultInstance()->getEntityIdParser();
$startItemOption = $this->getOption('start-item');
$startItem = $startItemOption === null ? null : $idParser->parse($startItemOption);
if ($startItem !== null && !$startItem instanceof ItemId) {
throw new EntityIdParsingException('Not an Item ID: ' . $startItemOption);
}
$verbose = (bool) $this->getOption('verbose', false);
$reporter = new ObservableMessageReporter();
$reporter->registerReporterCallback(array($this, 'report'));
$builder = new ChangesSubscriptionTableBuilder(wfGetLB(), 'wb_changes_subscription', $this->mBatchSize, $verbose ? 'verbose' : 'standard');
$builder->setProgressReporter($reporter);
$builder->setExceptionHandler(new ReportingExceptionHandler($reporter));
$builder->fillSubscriptionTable($startItem);
return true;
}
/**
* @see LoggedUpdateMaintenance::doDBUpdates
*
* @return bool
*/
public function doDBUpdates()
{
if (!defined('WB_VERSION')) {
$this->output("You need to have Wikibase enabled in order to use this maintenance script!\n\n");
exit;
}
$reporter = new ObservableMessageReporter();
$reporter->registerReporterCallback(array($this, 'report'));
$table = new PropertyInfoTable(false);
$wikibaseRepo = WikibaseRepo::getDefaultInstance();
$entityLookup = $wikibaseRepo->getEntityLookup();
$propertyInfoBuilder = $wikibaseRepo->newPropertyInfoBuilder();
$builder = new PropertyInfoTableBuilder($table, $entityLookup, $propertyInfoBuilder);
$builder->setReporter($reporter);
$builder->setBatchSize((int) $this->getOption('batch-size', 100));
$builder->setRebuildAll($this->getOption('rebuild-all', false));
$builder->setFromId((int) $this->getOption('start-row', 1));
$n = $builder->rebuildPropertyInfo();
$this->output("Done. Updated {$n} property info entries.\n");
return true;
}
/**
* @see Maintenance::execute
*/
public function execute()
{
if (!defined('WB_VERSION')) {
$this->output("You need to have Wikibase enabled in order to use this maintenance script!\n\n");
exit;
}
$batchSize = (int) $this->getOption('batch-size', 100);
$reporter = new ObservableMessageReporter();
$reporter->registerReporterCallback(array($this, 'report'));
$siteLinkTable = new SiteLinkTable('wb_items_per_site', false);
$wikibaseRepo = WikibaseRepo::getDefaultInstance();
// Use an uncached EntityLookup here to avoid memory leaks
$entityLookup = $wikibaseRepo->getEntityLookup('uncached');
$store = $wikibaseRepo->getStore();
$builder = new ItemsPerSiteBuilder($siteLinkTable, $entityLookup, $store->getEntityPrefetcher());
$builder->setReporter($reporter);
$builder->setBatchSize($batchSize);
$entityPerPage = $store->newEntityPerPage();
$stream = new EntityPerPageIdPager($entityPerPage, 'item');
// Now <s>kill</s> fix the table
$builder->rebuild($stream);
}
/**
* @see LoggedUpdateMaintenance::doDBUpdates
*
* @throws EntityIdParsingException
* @return bool
*/
public function execute()
{
if (!defined('WBC_VERSION')) {
$this->error('You need to have WikibaseClient enabled in order to use this maintenance script!', 1);
}
$wikibaseClient = WikibaseClient::getDefaultInstance();
$settings = $wikibaseClient->getSettings();
$repoDB = $settings->getSetting('repoDatabase');
$clientId = $settings->getSetting('siteGlobalID');
$idParser = $wikibaseClient->getEntityIdParser();
$startItemOption = $this->getOption('start-item');
$startItem = $startItemOption === null ? null : $idParser->parse($startItemOption);
$reporter = new ObservableMessageReporter();
$reporter->registerReporterCallback(array($this, 'report'));
$updater = new BulkSubscriptionUpdater(new ConsistentReadConnectionManager(wfGetLB()), new ConsistentReadConnectionManager(wfGetLB($repoDB), $repoDB), $clientId, $repoDB, $this->mBatchSize);
$updater->setProgressReporter($reporter);
$updater->setExceptionHandler(new ReportingExceptionHandler($reporter));
if ($this->getOption('purge')) {
$updater->purgeSubscriptions($startItem);
}
$updater->updateSubscriptions($startItem);
return true;
}
/**
* Initializes members from command line options and configuration settings.
*
* @param ChangeLookup $changeLookup
* @param SettingsArray $settings
*
* @return ChangeDispatcher
* @throws MWException
*/
private function newChangeDispatcher(ChangeLookup $changeLookup, SettingsArray $settings)
{
$repoDB = $settings->getSetting('changesDatabase');
$clientWikis = $settings->getSetting('localClientDatabases');
$batchChunkFactor = $settings->getSetting('dispatchBatchChunkFactor');
$batchCacheFactor = $settings->getSetting('dispatchBatchCacheFactor');
$subscriptionLookupMode = $settings->getSetting('subscriptionLookupMode');
$batchSize = (int) $this->getOption('batch-size', 1000);
$maxChunks = (int) $this->getOption('max-chunks', 15);
$dispatchInterval = (int) $this->getOption('dispatch-interval', 60);
$lockGraceInterval = (int) $this->getOption('lock-grace-interval', 60);
$randomness = (int) $this->getOption('randomness', 10);
$this->verbose = $this->getOption('verbose', false);
$cacheChunkSize = $batchSize * $batchChunkFactor;
$cacheSize = $cacheChunkSize * $batchCacheFactor;
$changesCache = new ChunkCache($changeLookup, $cacheChunkSize, $cacheSize);
// make sure we have a mapping from siteId to database name in clientWikis:
foreach ($clientWikis as $siteID => $dbName) {
if (is_int($siteID)) {
unset($clientWikis[$siteID]);
$clientWikis[$dbName] = $dbName;
}
}
if (empty($clientWikis)) {
throw new MWException("No client wikis configured! Please set \$wgWBRepoSettings['localClientDatabases'].");
}
$reporter = new ObservableMessageReporter();
$self = $this;
// PHP 5.3...
$reporter->registerReporterCallback(function ($message) use($self) {
$self->log($message);
});
$coordinator = new SqlChangeDispatchCoordinator($repoDB, $clientWikis);
$coordinator->setMessageReporter($reporter);
$coordinator->setBatchSize($batchSize);
$coordinator->setDispatchInterval($dispatchInterval);
$coordinator->setLockGraceInterval($lockGraceInterval);
$coordinator->setRandomness($randomness);
$notificationSender = new JobQueueChangeNotificationSender($repoDB, $clientWikis);
$subscriptionLookup = $this->getSubscriptionLookup($repoDB, $subscriptionLookupMode);
$dispatcher = new ChangeDispatcher($coordinator, $notificationSender, $changesCache, $subscriptionLookup);
$dispatcher->setMessageReporter($reporter);
$dispatcher->setExceptionHandler(new ReportingExceptionHandler($reporter));
$dispatcher->setBatchSize($batchSize);
$dispatcher->setMaxChunks($maxChunks);
$dispatcher->setBatchChunkFactor($batchChunkFactor);
$dispatcher->setVerbose($this->verbose);
return $dispatcher;
}
/**
* @return MessageReporter
*/
private function newMessageReporter()
{
$reporter = new ObservableMessageReporter();
$reporter->registerReporterCallback(array($this, 'log'));
return $reporter;
}
/**
* Do the actual work. All child classes will need to implement this
*/
public function execute()
{
//TODO: more validation for options
$entityType = $this->getOption('entity-type');
$shardingFactor = (int) $this->getOption('sharding-factor', 1);
$shard = (int) $this->getOption('shard', 0);
$batchSize = (int) $this->getOption('batch-size', 100);
$limit = (int) $this->getOption('limit', 0);
//TODO: Allow injection of an OutputStream for logging
$this->openLogFile($this->getOption('log', 'php://stderr'));
$outFile = $this->getOption('output', 'php://stdout');
if ($outFile === '-') {
$outFile = 'php://stdout';
}
$output = fopen($outFile, 'w');
//TODO: Allow injection of an OutputStream
if (!$output) {
throw new MWException('Failed to open ' . $outFile . '!');
}
if ($this->hasOption('list-file')) {
$this->logMessage("Dumping entities listed in " . $this->getOption('list-file'));
}
if ($entityType) {
$this->logMessage("Dumping entities of type {$entityType}");
}
if ($shardingFactor) {
$this->logMessage("Dumping shard {$shard}/{$shardingFactor}");
}
$dumper = $this->createDumper($output);
$dumper->setLimit($limit);
$progressReporter = new ObservableMessageReporter();
$progressReporter->registerReporterCallback(array($this, 'logMessage'));
$dumper->setProgressReporter($progressReporter);
$exceptionReporter = new ReportingExceptionHandler($progressReporter);
$dumper->setExceptionHandler($exceptionReporter);
//NOTE: we filter for $entityType twice: filtering in the DB is efficient,
// but filtering in the dumper is needed when working from a list file.
$dumper->setShardingFilter($shardingFactor, $shard);
$dumper->setEntityTypeFilter($entityType);
$dumper->setBatchSize($batchSize);
$idStream = $this->makeIdStream($entityType, $exceptionReporter);
\MediaWiki\suppressWarnings();
$dumper->generateDump($idStream);
\MediaWiki\restoreWarnings();
if ($idStream instanceof Disposable) {
// close stream / free resources
$idStream->dispose();
}
$this->closeLogFile();
}
/**
* Initializes members from command line options and configuration settings.
*
* @param string[] $clientWikis A mapping of client wiki site IDs to logical database names.
* @param ChangeLookup $changeLookup
* @param SettingsArray $settings
*
* @return ChangeDispatcher
*/
private function newChangeDispatcher(array $clientWikis, ChangeLookup $changeLookup, SettingsArray $settings)
{
$repoID = wfWikiID();
$repoDB = $settings->getSetting('changesDatabase');
$batchChunkFactor = $settings->getSetting('dispatchBatchChunkFactor');
$batchCacheFactor = $settings->getSetting('dispatchBatchCacheFactor');
$batchSize = (int) $this->getOption('batch-size', 1000);
$maxChunks = (int) $this->getOption('max-chunks', 15);
$dispatchInterval = (int) $this->getOption('dispatch-interval', 60);
$lockGraceInterval = (int) $this->getOption('lock-grace-interval', 60);
$randomness = (int) $this->getOption('randomness', 10);
$this->verbose = $this->getOption('verbose', false);
$cacheChunkSize = $batchSize * $batchChunkFactor;
$cacheSize = $cacheChunkSize * $batchCacheFactor;
$changesCache = new ChunkCache($changeLookup, $cacheChunkSize, $cacheSize);
$reporter = new ObservableMessageReporter();
$self = $this;
// PHP 5.3...
$reporter->registerReporterCallback(function ($message) use($self) {
$self->log($message);
});
$coordinator = new SqlChangeDispatchCoordinator($repoDB, $repoID);
$coordinator->setMessageReporter($reporter);
$coordinator->setBatchSize($batchSize);
$coordinator->setDispatchInterval($dispatchInterval);
$coordinator->setLockGraceInterval($lockGraceInterval);
$coordinator->setRandomness($randomness);
$notificationSender = new JobQueueChangeNotificationSender($repoDB, $clientWikis);
$subscriptionLookup = new SqlSubscriptionLookup(wfGetLB());
$dispatcher = new ChangeDispatcher($coordinator, $notificationSender, $changesCache, $subscriptionLookup);
$dispatcher->setMessageReporter($reporter);
$dispatcher->setExceptionHandler(new ReportingExceptionHandler($reporter));
$dispatcher->setBatchSize($batchSize);
$dispatcher->setMaxChunks($maxChunks);
$dispatcher->setBatchChunkFactor($batchChunkFactor);
$dispatcher->setVerbose($this->verbose);
return $dispatcher;
}
/**
* Wrapper for invoking PropertyInfoTableBuilder from DatabaseUpdater
* during a database update.
*
* @param DatabaseUpdater $updater
*/
public static function rebuildPropertyInfo(DatabaseUpdater $updater)
{
$reporter = new ObservableMessageReporter();
$reporter->registerReporterCallback(function ($msg) use($updater) {
$updater->output("..." . $msg . "\n");
});
$table = new PropertyInfoTable(false);
$wikibaseRepo = WikibaseRepo::getDefaultInstance();
$contentCodec = $wikibaseRepo->getEntityContentDataCodec();
$propertyInfoBuilder = $wikibaseRepo->newPropertyInfoBuilder();
$wikiPageEntityLookup = new WikiPageEntityRevisionLookup($contentCodec, new WikiPageEntityMetaDataLookup($wikibaseRepo->getEntityIdParser()), false);
$cachingEntityLookup = new CachingEntityRevisionLookup($wikiPageEntityLookup, new HashBagOStuff());
$entityLookup = new RevisionBasedEntityLookup($cachingEntityLookup);
$builder = new PropertyInfoTableBuilder($table, $entityLookup, $propertyInfoBuilder);
$builder->setReporter($reporter);
$builder->setUseTransactions(false);
$updater->output('Populating ' . $table->getTableName() . "\n");
$builder->rebuildPropertyInfo();
}