public function testScript()
{
$dumpScript = new DumpRdf();
$mockRepo = new MockRepository();
$mockEntityPerPage = new MockEntityPerPage();
$snakList = new SnakList();
$snakList->addSnak(new PropertySomeValueSnak(new PropertyId('P12')));
$snakList->addSnak(new PropertyValueSnak(new PropertyId('P12'), new StringValue('stringVal')));
/** @var Entity[] $testEntities */
$testEntities = array(new Item(new ItemId('Q1')), new Property(new PropertyId('P1'), null, 'string'), new Property(new PropertyId('P12'), null, 'string', new StatementList(array(new Statement(new PropertySomeValueSnak(new PropertyId('P999')), null, null, 'GUID1')))), new Item(new ItemId('Q2'), new Fingerprint(new TermList(array(new Term('en', 'en-label'), new Term('de', 'de-label'))), new TermList(array(new Term('fr', 'en-desc'), new Term('de', 'de-desc'))), new AliasGroupList(array(new AliasGroup('en', array('ali1', 'ali2')), new AliasGroup('dv', array('ali11', 'ali22'))))), new SiteLinkList(array(new SiteLink('enwiki', 'Berlin'), new SiteLink('dewiki', 'England', array(new ItemId('Q1'))))), new StatementList(array(new Statement(new PropertySomeValueSnak(new PropertyId('P12')), null, null, 'GUID1'), new Statement(new PropertySomeValueSnak(new PropertyId('P12')), $snakList, new ReferenceList(array(new Reference(array(new PropertyValueSnak(new PropertyId('P12'), new StringValue('refSnakVal')), new PropertyNoValueSnak(new PropertyId('P12')))))), 'GUID2')))));
foreach ($testEntities as $key => $testEntity) {
$mockRepo->putEntity($testEntity, $key, '20000101000000');
$mockEntityPerPage->addEntityPage($testEntity->getId(), $key);
}
// Note: We are testing with the actual RDF bindings, so we can check for actual RDF output.
$rdfBuilder = WikibaseRepo::getDefaultInstance()->getValueSnakRdfBuilderFactory();
$dumpScript->setServices($mockEntityPerPage, new NullEntityPrefetcher(), MockSiteStore::newFromTestSites(), $this->getMockPropertyDataTypeLookup(), $rdfBuilder, $mockRepo, 'fooUri');
$logFileName = tempnam(sys_get_temp_dir(), "Wikibase-DumpRdfTest");
$outFileName = tempnam(sys_get_temp_dir(), "Wikibase-DumpRdfTest");
$dumpScript->loadParamsAndArgs(null, array('log' => $logFileName, 'output' => $outFileName, 'format' => 'n-triples'));
$dumpScript->execute();
$expectedLog = file_get_contents(__DIR__ . '/../data/maintenance/dumpRdf-log.txt');
$expectedOut = file_get_contents(__DIR__ . '/../data/maintenance/dumpRdf-out.txt');
$actualOut = file_get_contents($outFileName);
$actualOut = preg_replace('/<http:\\/\\/wikiba.se\\/ontology-beta#Dump> <http:\\/\\/schema.org\\/dateModified> "[^"]+"/', "<http://wikiba.se/ontology-beta#Dump> <http://schema.org/dateModified> \"2015-01-01T00:00:00Z\"", $actualOut);
$this->assertEquals($this->fixLineEndings($expectedLog), $this->fixLineEndings(file_get_contents($logFileName)));
$this->assertEquals($this->fixLineEndings($expectedOut), $this->fixLineEndings($actualOut));
}
private function copyQualifiers(SnakList $qualifiers)
{
$newQualifiers = new SnakList();
foreach ($qualifiers as $qualifier) {
$newQualifiers->addSnak($this->copySnak($qualifier));
}
return $newQualifiers;
}
public function testScript()
{
$dumpScript = new DumpJson();
$mockRepo = new MockRepository();
$mockEntityPerPage = new MockEntityPerPage();
$snakList = new SnakList();
$snakList->addSnak(new PropertySomeValueSnak(new PropertyId('P12')));
$snakList->addSnak(new PropertyValueSnak(new PropertyId('P12'), new StringValue('stringVal')));
/** @var Entity[] $testEntities */
$testEntities = array(new Item(new ItemId('Q1')), new Property(new PropertyId('P1'), null, 'string'), new Property(new PropertyId('P12'), null, 'string', new StatementList(array(new Statement(new PropertySomeValueSnak(new PropertyId('P999')), null, null, 'GUID1')))), new Item(new ItemId('Q2'), new Fingerprint(new TermList(array(new Term('en', 'en-label'), new Term('de', 'de-label'))), new TermList(array(new Term('fr', 'en-desc'), new Term('de', 'de-desc'))), new AliasGroupList(array(new AliasGroup('en', array('ali1', 'ali2')), new AliasGroup('dv', array('ali11', 'ali22'))))), new SiteLinkList(array(new SiteLink('enwiki', 'Berlin'), new SiteLink('dewiki', 'England', array(new ItemId('Q1'))))), new StatementList(array(new Statement(new PropertySomeValueSnak(new PropertyId('P12')), null, null, 'GUID1'), new Statement(new PropertySomeValueSnak(new PropertyId('P12')), $snakList, new ReferenceList(array(new Reference(array(new PropertyValueSnak(new PropertyId('P12'), new StringValue('refSnakVal')), new PropertyNoValueSnak(new PropertyId('P12')))))), 'GUID2')))));
foreach ($testEntities as $key => $testEntity) {
$mockRepo->putEntity($testEntity);
$mockEntityPerPage->addEntityPage($testEntity->getId(), $key);
}
$dumpScript->setServices($mockEntityPerPage, new NullEntityPrefetcher(), $this->getMockPropertyDataTypeLookup(), $mockRepo);
$logFileName = tempnam(sys_get_temp_dir(), "Wikibase-DumpJsonTest");
$outFileName = tempnam(sys_get_temp_dir(), "Wikibase-DumpJsonTest");
$dumpScript->loadParamsAndArgs(null, array('log' => $logFileName, 'output' => $outFileName));
$dumpScript->execute();
$expectedLog = file_get_contents(__DIR__ . '/../data/maintenance/dumpJson-log.txt');
$expectedOut = file_get_contents(__DIR__ . '/../data/maintenance/dumpJson-out.txt');
$this->assertEquals($this->fixLineEndings($expectedLog), $this->fixLineEndings(file_get_contents($logFileName)));
$this->assertEquals($this->fixLineEndings($expectedOut), $this->fixLineEndings(file_get_contents($outFileName)));
}
public function testFullEntityGetEntityResponse()
{
$item = new Item(new ItemId('Q123098'));
//Basic
$item->setLabel('de', 'foo-de');
$item->setLabel('qu', 'foo-qu');
$item->setAliases('en', array('bar', 'baz'));
$item->setAliases('de-formal', array('bar', 'baz'));
$item->setDescription('en', 'en-desc');
$item->setDescription('pt', 'ptDesc');
$item->getSiteLinkList()->addNewSiteLink('enwiki', 'Berlin', array(new ItemId('Q333')));
$item->getSiteLinkList()->addNewSiteLink('zh_classicalwiki', 'User:Addshore', array());
$snak = new PropertyValueSnak(65, new StringValue('snakStringValue'));
$qualifiers = new SnakList();
$qualifiers->addSnak(new PropertyValueSnak(65, new StringValue('string!')));
$qualifiers->addSnak(new PropertySomeValueSnak(65));
$references = new ReferenceList();
$references->addNewReference(array(new PropertySomeValueSnak(65), new PropertySomeValueSnak(68)));
$guid = 'imaguid';
$item->getStatements()->addNewStatement($snak, $qualifiers, $references, $guid);
$entityLookup = new MockRepository();
$entityLookup->putEntity($item);
$entityAccessor = $this->getEntityAccessor($entityLookup, null, 'qug');
$expected = array('id' => 'Q123098', 'type' => 'item', 'labels' => array('de' => array('language' => 'de', 'value' => 'foo-de')), 'descriptions' => array('en' => array('language' => 'en', 'value' => 'en-desc')), 'aliases' => array('en' => array(1 => array('language' => 'en', 'value' => 'bar'), 2 => array('language' => 'en', 'value' => 'baz'))), 'claims' => array('P65' => array(1 => array('id' => 'imaguid', 'type' => 'statement', 'mainsnak' => array('snaktype' => 'value', 'property' => 'P65', 'datatype' => 'structured-cat', 'datavalue' => array('value' => 'snakStringValue', 'type' => 'string')), 'qualifiers' => array('P65' => array(1 => array('hash' => 'e95e866e7fa1c18bd06dae9b712cb99545107eb8', 'snaktype' => 'value', 'property' => 'P65', 'datavalue' => array('value' => 'string!', 'type' => 'string'), 'datatype' => 'structured-cat'), 2 => array('hash' => '210b00274bf03247a89de918f15b12142ebf9e56', 'snaktype' => 'somevalue', 'property' => 'P65', 'datatype' => 'structured-cat'))), 'rank' => 'normal', 'qualifiers-order' => array(1 => 'P65'), 'references' => array(1 => array('hash' => 'bdc5f7185904d6d3219e13b7443571dda8c4bee8', 'snaks' => array('P65' => array(1 => array('snaktype' => 'somevalue', 'property' => 'P65', 'datatype' => 'structured-cat')), 'P68' => array(1 => array('snaktype' => 'somevalue', 'property' => 'P68', 'datatype' => 'structured-cat'))), 'snaks-order' => array(1 => 'P65', 2 => 'P68')))))), 'sitelinks' => array('enwiki' => array('site' => 'enwiki', 'title' => 'Berlin', 'badges' => array(1 => 'Q333')), 'zh_classicalwiki' => array('site' => 'zh_classicalwiki', 'title' => 'User:Addshore', 'badges' => array())), 'schemaVersion' => 2);
$this->assertEquals($expected, $entityAccessor->getEntity('Q123098'));
}
/**
* @dataProvider provideTestAddEntityRevision
*/
public function testAddEntityRevision($addMetaData, array $expected)
{
$result = $this->getDefaultResult();
$item = new Item(new ItemId('Q123098'));
//Basic
$item->setLabel('de', 'foo');
$item->setLabel('zh_classical', 'Longer Label');
$item->setAliases('en', array('bar', 'baz'));
$item->setAliases('zh', array('????????'));
$item->setDescription('pt', 'ptDesc');
$item->setDescription('pl', 'Longer Description For An Item');
$item->getSiteLinkList()->addNewSiteLink('enwiki', 'Berlin', array(new ItemId('Q333')));
$item->getSiteLinkList()->addNewSiteLink('zh_classicalwiki', 'User:Addshore', array());
$snak = new PropertyValueSnak(new PropertyId('P65'), new StringValue('snakStringValue'));
$qualifiers = new SnakList();
$qualifiers->addSnak(new PropertyValueSnak(new PropertyId('P65'), new StringValue('string!')));
$qualifiers->addSnak(new PropertySomeValueSnak(new PropertyId('P65')));
$references = new ReferenceList();
$referenceSnaks = new SnakList();
$referenceSnaks->addSnak(new PropertySomeValueSnak(new PropertyId('P65')));
$referenceSnaks->addSnak(new PropertySomeValueSnak(new PropertyId('P68')));
$references->addReference(new Reference($referenceSnaks));
$guid = 'imaguid';
$item->getStatements()->addNewStatement($snak, $qualifiers, $references, $guid);
$entityRevision = new EntityRevision($item, 33, '20131126202923');
$resultBuilder = $this->getResultBuilder($result, $addMetaData);
$resultBuilder->addEntityRevision('Q1230000', $entityRevision);
$data = $result->getResultData();
$this->assertEquals($expected, $data);
}
/**
* @param string $tsvStatement
* @return FullStatement
*/
public function parseTsvStatement($tsvStatement)
{
$parts = explode("\t", $tsvStatement);
$partsCount = count($parts);
if ($partsCount < 3 || $partsCount % 2 != 1) {
throw new InvalidArgumentException("Invalid statement: {$tsvStatement}");
}
list($subject, $property, $object) = $parts;
$qualifiers = new SnakList();
$reference = new SnakList();
for ($i = 3; $i < $partsCount; $i += 2) {
switch ($parts[$i]) {
case 'P':
$qualifiers->addSnak(new PropertyValueSnak($this->entityIdParser->parse($parts[$i]), $this->parseDataValue($parts[$i + 1])));
break;
case 'S':
$reference->addSnak(new PropertyValueSnak($this->entityIdParser->parse(str_replace('S', 'P', $parts[$i])), $this->parseDataValue($parts[$i + 1])));
break;
default:
throw new InvalidArgumentException("Invalid statement: {$tsvStatement}");
}
}
return new FullStatement($this->entityIdParser->parse($subject), new Statement(new PropertyValueSnak($this->entityIdParser->parse($property), $this->parseDataValue($object)), $qualifiers, new ReferenceList($reference->isEmpty() ? [new Reference($reference)] : [])));
}
/**
* @dataProvider instanceProvider
* @param SnakList $array
*/
public function testAddSnak(SnakList $array)
{
$elementCount = $array->count();
$elements = $this->elementInstancesProvider();
$element = array_shift($elements);
$element = $element[0][0];
if (!$array->hasSnak($element)) {
++$elementCount;
}
$this->assertEquals(!$array->hasSnak($element), $array->addSnak($element));
$this->assertEquals($elementCount, $array->count());
$this->assertFalse($array->addSnak($element));
$this->assertEquals($elementCount, $array->count());
}
/**
* @since 0.4
*
* @param SnakList $qualifiers
* @param Summary $summary
*
* @throws ChangeOpException
*/
protected function setQualifier(SnakList $qualifiers, Summary $summary = null)
{
if (!$qualifiers->hasSnakHash($this->snakHash)) {
throw new ChangeOpException("Qualifier with hash {$this->snakHash} does not exist");
}
if ($qualifiers->hasSnak($this->snak)) {
throw new ChangeOpException('The statement has already a qualifier with hash ' . $this->snak->getHash());
}
$qualifiers->removeSnakHash($this->snakHash);
$qualifiers->addSnak($this->snak);
$this->updateSummary($summary, 'update', '', $this->getSnakSummaryArgs($this->snak));
}
protected function execute(InputInterface $input, OutputInterface $output)
{
// Get options
$user = $input->getOption('user');
$userDetails = $this->appConfig->offsetGet('users.' . $user);
if ($userDetails === null) {
throw new RuntimeException('User not found in config');
}
$items = $input->getOption('item');
if (empty($items)) {
$output->writeln('Running SPARQL query to find items to check');
$queryBuilder = new QueryBuilder(array('prov' => 'http://www.w3.org/ns/prov#', 'wd' => 'http://www.wikidata.org/entity/', 'wikibase' => 'http://wikiba.se/ontology#', 'prv' => 'http://www.wikidata.org/prop/reference/value/'));
$itemIds = $this->sparqlQueryRunner->getItemIdsFromQuery($queryBuilder->select('?item')->where('?ref', 'prv:P813', '?value')->also('?value', 'wikibase:timeCalendarModel', 'wd:Q1985786')->also('?st', 'prov:wasDerivedFrom', '?ref')->also('?item', '?pred', '?st')->limit(10000)->__toString());
} else {
/** @var ItemId[] $itemIds */
$itemIds = array();
foreach (array_unique($items) as $itemIdString) {
$itemIds[] = new ItemId($itemIdString);
}
}
$itemIds = array_unique($itemIds);
$output->writeln('Running for ' . count($itemIds) . ' items');
// Log in to Wikidata
$loggedIn = $this->wikibaseApi->login(new ApiUser($userDetails['username'], $userDetails['password']));
if (!$loggedIn) {
$output->writeln('Failed to log in to wikidata wiki');
return -1;
}
$itemLookup = $this->wikibaseFactory->newItemLookup();
foreach ($itemIds as $itemId) {
$output->write($itemId->getSerialization() . ' ');
$item = $itemLookup->getItemForId($itemId);
foreach ($item->getStatements()->getIterator() as $statement) {
foreach ($statement->getReferences() as $reference) {
/** @var Reference $reference */
foreach ($reference->getSnaks()->getIterator() as $snak) {
if ($snak instanceof PropertyValueSnak) {
if ($snak->getPropertyId()->getSerialization() == 'P813') {
/** @var TimeValue $dataValue */
$dataValue = $snak->getDataValue();
// We can assume ALL retrieval dates should be Gregorian!
if ($dataValue->getCalendarModel() === TimeValue::CALENDAR_JULIAN) {
$oldRefHash = $reference->getHash();
$statementGuid = $statement->getGuid();
$snakList = $reference->getSnaks();
$snakList = new SnakList($snakList->getArrayCopy());
$snakList->removeSnak($snak);
$fixedTimestamp = $this->getFixedTimestamp($dataValue->getTime());
if ($fixedTimestamp) {
$snakList->addSnak(new PropertyValueSnak(new PropertyId('P813'), new TimeValue($fixedTimestamp, $dataValue->getTimezone(), $dataValue->getBefore(), $dataValue->getAfter(), $dataValue->getPrecision(), TimeValue::CALENDAR_GREGORIAN)));
$editSummary = 'Fix reference retrieval date';
$output->write('.');
} else {
//TODO optionally remove rather than always doing so?
$editSummary = 'Removing bad reference retrieval date';
$output->write('x');
}
try {
$this->wikibaseFactory->newReferenceSetter()->set(new Reference($snakList), $statementGuid, $oldRefHash, new EditInfo($editSummary));
} catch (UsageException $e) {
$output->writeln('');
$output->write($e->getMessage());
}
}
}
}
}
}
}
$output->writeln('');
}
return 0;
}