/**
* Run the ApplyOperation job
* @throws \Exception
*/
public function perform()
{
try {
$this->debugLog("[JOBID " . $this->job->payload['id'] . "] ApplyOperation::perform() start");
$timer = new \Tripod\Timer();
$timer->start();
$this->validateArgs();
$statsConfig = array();
if (isset($this->args['statsConfig'])) {
$statsConfig['statsConfig'] = $this->args['statsConfig'];
}
// set the config to what is received
\Tripod\Mongo\Config::setConfig($this->args[self::TRIPOD_CONFIG_KEY]);
$this->getStat()->increment(MONGO_QUEUE_APPLY_OPERATION_JOB . '.' . SUBJECT_COUNT, count($this->args[self::SUBJECTS_KEY]));
foreach ($this->args[self::SUBJECTS_KEY] as $subject) {
$opTimer = new \Tripod\Timer();
$opTimer->start();
$impactedSubject = $this->createImpactedSubject($subject);
$impactedSubject->update();
$opTimer->stop();
// stat time taken to perform operation for the given subject
$this->getStat()->timer(MONGO_QUEUE_APPLY_OPERATION . '.' . $subject['operation'], $opTimer->result());
}
$timer->stop();
// stat time taken to process job, from time it was picked up
$this->getStat()->timer(MONGO_QUEUE_APPLY_OPERATION_SUCCESS, $timer->result());
$this->debugLog("[JOBID " . $this->job->payload['id'] . "] ApplyOperation::perform() done in {$timer->result()}ms");
} catch (\Exception $e) {
$this->getStat()->increment(MONGO_QUEUE_APPLY_OPERATION_FAIL);
$this->errorLog("Caught exception in " . get_class($this) . ": " . $e->getMessage());
throw $e;
}
}
/**
* Note: Current version of this test tried to create 1000 objects within 6000ms which is reasonable at this time.
* Any change to this class if make it a more a big number it should be validated and tested to ensure performance impact.
*
* Create some instances of Config to see what amount of time is taken in creating instance and processing in constructor.
*/
public function testCreateMongoTripodConfigObject()
{
$testStartTime = microtime();
//Let's try to create 1000 objects to see how much time they take.
for ($i = 0; $i < self::BENCHMARK_OBJECT_CREATE_ITERATIONS; $i++) {
\Tripod\Mongo\Config::setConfig($this->config);
$instance = \Tripod\Mongo\Config::getInstance();
}
$testEndTime = microtime();
$this->assertLessThan(self::BENCHMARK_OBJECT_CREATE_TIME, $this->getTimeDifference($testStartTime, $testEndTime), "It should always take less than " . self::BENCHMARK_OBJECT_CREATE_TIME . "ms to create " . self::BENCHMARK_OBJECT_CREATE_ITERATIONS . " objects of Config class");
}
protected function setUp()
{
date_default_timezone_set('Europe/London');
$config = json_decode(file_get_contents($this->getConfigLocation()), true);
if (getenv('TRIPOD_DATASOURCE_RS1_CONFIG')) {
$config['data_sources']['rs1'] = json_decode(getenv('TRIPOD_DATASOURCE_RS1_CONFIG'), true);
}
if (getenv('TRIPOD_DATASOURCE_RS2_CONFIG')) {
$config['data_sources']['rs2'] = json_decode(getenv('TRIPOD_DATASOURCE_RS2_CONFIG'), true);
}
\Tripod\Mongo\Config::setConfig($config);
$className = get_class($this);
$testName = $this->getName();
echo "\nTest: {$className}->{$testName}\n";
// make sure log statements don't go to stdout during tests...
$log = new \Monolog\Logger("unittest");
$log->pushHandler(new \Monolog\Handler\NullHandler());
\Tripod\Mongo\DriverBase::$logger = $log;
}
\Tripod\Mongo\Config::getInstance()->setMongoCursorTimeout(-1);
print "Generating {$tableId}";
$tripod = new \Tripod\Mongo\Driver($tableSpec['from'], $storeName, array('stat' => $stat));
$tTables = $tripod->getTripodTables();
if ($id) {
print " for {$id}....\n";
$tTables->generateTableRows($tableId, $id);
} else {
print " for all tables....\n";
$tTables->generateTableRows($tableId, null, null, $queue);
}
}
}
$t = new \Tripod\Timer();
$t->start();
\Tripod\Mongo\Config::setConfig(json_decode(file_get_contents($configLocation), true));
if (isset($options['s']) || isset($options['storename'])) {
$storeName = isset($options['s']) ? $options['s'] : $options['storename'];
} else {
$storeName = null;
}
if (isset($options['t']) || isset($options['spec'])) {
$tableId = isset($options['t']) ? $options['t'] : $options['spec'];
} else {
$tableId = null;
}
if (isset($options['i']) || isset($options['id'])) {
$id = isset($options['i']) ? $options['i'] : $options['id'];
} else {
$id = null;
}
$loader->loadTriplesAbout($subject, $triples, $storeName, $podName);
} catch (Exception $e) {
print "Exception for subject {$subject} failed with message: " . $e->getMessage() . "\n";
$errors[] = $subject;
}
}
$timer = new \Tripod\Timer();
$timer->start();
if ($argc != 4) {
echo "usage: ./loadTriples.php storename podname tripodConfig.json < ntriplesdata\n";
die;
}
array_shift($argv);
$storeName = $argv[0];
$podName = $argv[1];
\Tripod\Mongo\Config::setConfig(json_decode(file_get_contents($argv[2]), true));
$i = 0;
$currentSubject = "";
$triples = array();
$errors = array();
// array of subjects that failed to insert, even after retry...
$loader = new \Tripod\Mongo\TriplesUtil();
while (($line = fgets(STDIN)) !== false) {
$i++;
if ($i % 250000 == 0) {
print "Memory: " . memory_get_usage() . "\n";
}
$line = rtrim($line);
$parts = preg_split("/\\s/", $line);
$subject = trim($parts[0], '><');
if (empty($currentSubject)) {
/**
* Run the DiscoverImpactedSubjects job
* @throws \Exception
*/
public function perform()
{
try {
$this->debugLog("[JOBID " . $this->job->payload['id'] . "] DiscoverImpactedSubjects::perform() start");
$timer = new \Tripod\Timer();
$timer->start();
$this->validateArgs();
// set the config to what is received
\Tripod\Mongo\Config::setConfig($this->args[self::TRIPOD_CONFIG_KEY]);
$statsConfig = array();
if (isset($this->args['statsConfig'])) {
$statsConfig['statsConfig'] = $this->args['statsConfig'];
}
$tripod = $this->getTripod($this->args[self::STORE_NAME_KEY], $this->args[self::POD_NAME_KEY], $statsConfig);
$operations = $this->args[self::OPERATIONS_KEY];
$subjectsAndPredicatesOfChange = $this->args[self::CHANGES_KEY];
$subjectCount = 0;
foreach ($operations as $op) {
/** @var \Tripod\Mongo\Composites\IComposite $composite */
$composite = $tripod->getComposite($op);
$modifiedSubjects = $composite->getImpactedSubjects($subjectsAndPredicatesOfChange, $this->args[self::CONTEXT_ALIAS_KEY]);
if (!empty($modifiedSubjects)) {
/* @var $subject \Tripod\Mongo\ImpactedSubject */
foreach ($modifiedSubjects as $subject) {
$subjectCount++;
$subjectTimer = new \Tripod\Timer();
$subjectTimer->start();
if (isset($this->args[self::QUEUE_KEY]) || count($subject->getSpecTypes()) == 0) {
$queueName = isset($this->args[self::QUEUE_KEY]) ? $this->args[self::QUEUE_KEY] : Config::getApplyQueueName();
$this->addSubjectToQueue($subject, $queueName);
} else {
$specsGroupedByQueue = array();
foreach ($subject->getSpecTypes() as $specType) {
$spec = null;
switch ($subject->getOperation()) {
case OP_VIEWS:
$spec = Config::getInstance()->getViewSpecification($this->args[self::STORE_NAME_KEY], $specType);
break;
case OP_TABLES:
$spec = Config::getInstance()->getTableSpecification($this->args[self::STORE_NAME_KEY], $specType);
break;
case OP_SEARCH:
$spec = Config::getInstance()->getSearchDocumentSpecification($this->args[self::STORE_NAME_KEY], $specType);
break;
}
if (!$spec || !isset($spec['queue'])) {
if (!$spec) {
$spec = array();
}
$spec['queue'] = Config::getApplyQueueName();
}
if (!isset($specsGroupedByQueue[$spec['queue']])) {
$specsGroupedByQueue[$spec['queue']] = array();
}
$specsGroupedByQueue[$spec['queue']][] = $specType;
}
foreach ($specsGroupedByQueue as $queueName => $specs) {
$queuedSubject = new \Tripod\Mongo\ImpactedSubject($subject->getResourceId(), $subject->getOperation(), $subject->getStoreName(), $subject->getPodName(), $specs);
$this->addSubjectToQueue($queuedSubject, $queueName);
}
}
$subjectTimer->stop();
// stat time taken to discover impacted subjects for the given subject of change
$this->getStat()->timer(MONGO_QUEUE_DISCOVER_SUBJECT, $subjectTimer->result());
}
if (!empty($this->subjectsGroupedByQueue)) {
foreach ($this->subjectsGroupedByQueue as $queueName => $subjects) {
$this->getApplyOperation()->createJob($subjects, $queueName, $statsConfig);
}
$this->subjectsGroupedByQueue = array();
}
}
}
// stat time taken to process item, from time it was created (queued)
$timer->stop();
$this->getStat()->timer(MONGO_QUEUE_DISCOVER_SUCCESS, $timer->result());
$this->debugLog("[JOBID " . $this->job->payload['id'] . "] DiscoverImpactedSubjects::perform() done in {$timer->result()}ms");
$this->getStat()->increment(MONGO_QUEUE_DISCOVER_JOB . '.' . SUBJECT_COUNT, $subjectCount);
} catch (\Exception $e) {
$this->getStat()->increment(MONGO_QUEUE_DISCOVER_FAIL);
$this->errorLog("Caught exception in " . get_class($this) . ": " . $e->getMessage());
throw $e;
}
}
$tripod = new \Tripod\Mongo\Driver($podName, $storeName, $tripodOptions);
$oldGraph = $tripod->describeResource(base64_decode($encodedFqUri));
$tripod->saveChanges($oldGraph, new \Tripod\ExtendedGraph());
});
$app->post('/', function ($storeName, $podName) use($app, $tripodOptions) {
$tripodOptions['statsConfig'] = getStat($app, $tripodOptions);
$tripod = new \Tripod\Mongo\Driver($podName, $storeName, $tripodOptions);
$rawGraphData = $app->request()->getBody();
$graph = new \Tripod\Mongo\MongoGraph();
$graph->add_rdf($rawGraphData);
$tripod->saveChanges(new \Tripod\ExtendedGraph(), $graph);
});
});
$app->group('/change', function () use($app, $tripodOptions) {
$app->post('/', function ($storeName, $podName) use($app, $tripodOptions) {
\Tripod\Mongo\Config::setConfig(json_decode(file_get_contents('./config/tripod-config-' . $storeName . '.json'), true));
$app->response()->setStatus(500);
$tripodOptions['statsConfig'] = getStat($app, $tripodOptions);
$tripod = new \Tripod\Mongo\Driver($podName, $storeName, $tripodOptions);
$rawChangeData = $app->request()->post('data');
if ($rawChangeData) {
$changeData = json_decode($rawChangeData, true);
$from = new \Tripod\Mongo\MongoGraph();
$to = new \Tripod\Mongo\MongoGraph();
if (isset($changeData['originalCBDs'])) {
foreach ($changeData['originalCBDs'] as $change) {
if (is_array($change) && isset($change[_ID_KEY])) {
$from->add_tripod_array($change);
}
}
}
<?php
require_once dirname(__FILE__) . '/common.inc.php';
require_once dirname(dirname(dirname(__FILE__))) . '/src/tripod.inc.php';
if ($argc != 2) {
echo "usage: ./BSONToQuads.php tripodConfig.json < bsondata\n";
echo " When exporting bson data from Mongo use: \n";
echo " mongoexport -d <dbname> -c <collectionName> > bsondata.txt \n";
die;
}
array_shift($argv);
$config = json_decode(file_get_contents($argv[0]), true);
\Tripod\Mongo\Config::setConfig($config);
$tu = new \Tripod\Mongo\TriplesUtil();
while (($line = fgets(STDIN)) !== false) {
$line = rtrim($line);
$doc = json_decode($line, true);
$context = $doc['_id']['c'];
$graph = new \Tripod\Mongo\MongoGraph();
$graph->add_tripod_array($doc);
echo $graph->to_nquads($context);
}
public function testManuallySpecifiedQueueWillOverrideQueuesDefinedInConfig()
{
$config = \Tripod\Mongo\Config::getConfig();
// Create a bunch of specs on various queues
$tableSpecs = array(array("_id" => "t_resource", "type" => "acorn:Resource", "from" => "CBD_testing", "ensureIndexes" => array("value.isbn" => 1), "fields" => array(array("fieldName" => "type", "predicates" => array("rdf:type")), array("fieldName" => "isbn", "predicates" => array("bibo:isbn13"))), "joins" => array("dct:isVersionOf" => array("fields" => array(array("fieldName" => "isbn13", "predicates" => array("bibo:isbn13")))))), array("_id" => "t_source_count", "type" => "acorn:Resource", "from" => "CBD_testing", "to_data_source" => "rs2", "queue" => "counts_and_other_non_essentials", "fields" => array(array("fieldName" => "type", "predicates" => array("rdf:type"))), "joins" => array("dct:isVersionOf" => array("fields" => array(array("fieldName" => "isbn13", "predicates" => array("bibo:isbn13"))))), "counts" => array(array("fieldName" => "source_count", "property" => "dct:isVersionOf"), array("fieldName" => "random_predicate_count", "property" => "dct:randomPredicate"))), array("_id" => "t_source_count_regex", "type" => "acorn:Resource", "from" => "CBD_testing", "queue" => "counts_and_other_non_essentials", "fields" => array(array("fieldName" => "type", "predicates" => array("rdf:type"))), "joins" => array("dct:isVersionOf" => array("fields" => array(array("fieldName" => "isbn13", "predicates" => array("bibo:isbn13"))))), "counts" => array(array("fieldName" => "source_count", "property" => "dct:isVersionOf"), array("fieldName" => "regex_source_count", "property" => "dct:isVersionOf", "regex" => "/foobar/"))), array("_id" => "t_join_source_count_regex", "type" => "acorn:Resource", "from" => "CBD_testing", "queue" => "MOST_IMPORTANT_QUEUE_EVER", "joins" => array("acorn:jacsUri" => array("counts" => array(array("fieldName" => "titles_count", "property" => "dct:title"))))));
$config['stores']['tripod_php_testing']['table_specifications'] = $tableSpecs;
\Tripod\Mongo\Config::setConfig($config);
/** @var \Tripod\Mongo\Jobs\DiscoverImpactedSubjects|PHPUnit_Framework_MockObject_MockObject $discoverImpactedSubjects */
$discoverImpactedSubjects = $this->getMockBuilder('\\Tripod\\Mongo\\Jobs\\DiscoverImpactedSubjects')->setMethods(array('getTripod', 'getApplyOperation'))->getMock();
$this->setArgs();
$args = $this->args;
$args['operations'] = array(OP_TABLES);
$args['queue'] = 'TRIPOD_TESTING_QUEUE_' . uniqid();
$discoverImpactedSubjects->args = $args;
$discoverImpactedSubjects->job->payload['id'] = uniqid();
$tripod = $this->getMockBuilder('\\Tripod\\Mongo\\Driver')->setMethods(array('getComposite'))->setConstructorArgs(array('CBD_testing', 'tripod_php_testing'))->getMock();
$tables = $this->getMockBuilder('\\Tripod\\Mongo\\Composites\\Tables')->setMethods(array('getImpactedSubjects'))->setConstructorArgs(array('tripod_php_testing', \Tripod\Mongo\Config::getInstance()->getCollectionForCBD('tripod_php_testing', 'CBD_testing'), 'http://talisaspire.com/'))->getMock();
$applyOperation = $this->getMockBuilder('\\Tripod\\Mongo\\Jobs\\ApplyOperation')->setMethods(array('createJob'))->getMock();
$tableSubjects = array(new \Tripod\Mongo\ImpactedSubject(array(_ID_RESOURCE => 'http://example.com/resources/foo2', _ID_CONTEXT => $this->args['contextAlias']), OP_TABLES, $this->args['storeName'], $this->args['podName'], array('t_resource', 't_source_count', 't_source_count_regex', 't_join_source_count_regex')), new \Tripod\Mongo\ImpactedSubject(array(_ID_RESOURCE => 'http://example.com/resources/foo3', _ID_CONTEXT => $this->args['contextAlias']), OP_TABLES, $this->args['storeName'], $this->args['podName'], array('t_distinct')));
$tables->expects($this->once())->method('getImpactedSubjects')->with($this->args['changes'], $this->args['contextAlias'])->will($this->returnValue($tableSubjects));
$tripod->expects($this->once())->method('getComposite')->with(OP_TABLES)->will($this->returnValue($tables));
$discoverImpactedSubjects->expects($this->once())->method('getTripod')->will($this->returnValue($tripod));
$discoverImpactedSubjects->expects($this->once())->method('getApplyOperation')->will($this->returnValue($applyOperation));
$applyOperation->expects($this->once())->method('createJob')->withConsecutive(array($tableSubjects, $args['queue']));
$discoverImpactedSubjects->perform();
}
public function testNestConditionalInArithmeticFunction()
{
$tableSpec = array("_id" => "t_arithmetic_with_nested_conditional", "type" => array("bibo:Book", "bibo:Document"), "from" => "CBD_testing", "fields" => array(array("fieldName" => "x", "predicates" => array("foo:wibble"))), "computed_fields" => array(array("fieldName" => "foobar", "value" => array("arithmetic" => array(array("conditional" => array("if" => array('$x'), "then" => '$x', "else" => 100)), "*", 3)))));
$oldConfig = \Tripod\Mongo\Config::getConfig();
$newConfig = \Tripod\Mongo\Config::getConfig();
$newConfig['stores']['tripod_php_testing']['table_specifications'][] = $tableSpec;
\Tripod\Mongo\Config::setConfig($newConfig);
\Tripod\Mongo\Config::getInstance();
$this->tripod = new \Tripod\Mongo\Driver('CBD_testing', 'tripod_php_testing');
$this->loadResourceDataViaTripod();
$this->tripod->generateTableRows('t_arithmetic_with_nested_conditional');
$collection = \Tripod\Mongo\Config::getInstance()->getCollectionForTable('tripod_php_testing', 't_arithmetic_with_nested_conditional');
$tableDoc = $collection->findOne(array('_id.type' => 't_arithmetic_with_nested_conditional'));
$this->assertEquals(300, $tableDoc['value']['foobar']);
\Tripod\Mongo\Config::setConfig($oldConfig);
\Tripod\Mongo\Config::getInstance();
$collection->drop();
}
public function testRemoveTableSpecDoesNotAffectInvalidation()
{
foreach (\Tripod\Mongo\Config::getInstance()->getTableSpecifications($this->tripod->getStoreName()) as $specId => $spec) {
$this->generateTableRows($specId);
}
$context = 'http://talisaspire.com/';
$uri = "http://talisaspire.com/works/4d101f63c10a6";
$collection = \Tripod\Mongo\Config::getInstance()->getCollectionForTable('tripod_php_testing', 't_resource');
$this->assertGreaterThan(0, $collection->count(array('_id.type' => 't_resource', 'value._impactIndex' => array(_ID_RESOURCE => $uri, _ID_CONTEXT => $context))));
$config = \Tripod\Mongo\Config::getConfig();
unset($config['stores']['tripod_php_testing']['table_specifications'][0]);
\Tripod\Mongo\Config::setConfig($config);
/** @var PHPUnit_Framework_MockObject_MockObject|\Tripod\Mongo\Driver $mockTripod */
$mockTripod = $this->getMockBuilder('\\Tripod\\Mongo\\Driver')->setMethods(array('getComposite'))->setConstructorArgs(array('CBD_testing', 'tripod_php_testing', array('defaultContext' => $context, OP_ASYNC => array(OP_VIEWS => true, OP_TABLES => false, OP_SEARCH => true))))->getMock();
$mockTables = $this->getMockBuilder('\\Tripod\\Mongo\\Composites\\Tables')->setMethods(array('update'))->setConstructorArgs(array('tripod_php_testing', \Tripod\Mongo\Config::getInstance()->getCollectionForCBD('tripod_php_testing', 'CBD_testing'), $context))->getMock();
$labeller = new \Tripod\Mongo\Labeller();
$mockTripod->expects($this->once())->method('getComposite')->with(OP_TABLES)->will($this->returnValue($mockTables));
$mockTables->expects($this->never())->method('update');
$originalGraph = $mockTripod->describeResource($uri);
$updatedGraph = $originalGraph->get_subject_subgraph($uri);
$updatedGraph->add_literal_triple($uri, $labeller->qname_to_uri('dct:description'), 'Physics textbook');
$mockTripod->saveChanges($originalGraph, $updatedGraph);
// The table row should still be there, even if the tablespec no longer exists
$this->assertGreaterThan(0, $collection->count(array('_id.type' => 't_resource', 'value._impactIndex' => array(_ID_RESOURCE => $uri, _ID_CONTEXT => $context))));
}
public function testSaveChangesWithInvalidCardinality()
{
$this->setExpectedException('\\Tripod\\Exceptions\\CardinalityException', "Cardinality failed on http://foo/bar/1 for 'rdf:type' - should only have 1 value and has: http://foo/bar#Class1, http://foo/bar#Class2");
$config = array();
$config['namespaces'] = array('rdf' => 'http://www.w3.org/1999/02/22-rdf-syntax-ns#');
$config["defaultContext"] = "http://talisaspire.com/";
$config["transaction_log"] = array("database" => "transactions", "collection" => "transaction_log", "data_source" => "tlog");
$config["data_sources"] = array("db" => array("type" => "mongo", "connection" => "mongodb://localhost"), "tlog" => array("type" => "mongo", "connection" => "mongodb://*****:*****@localhost:27018"));
$config["stores"] = array("tripod_php_testing" => array("data_source" => "db", "pods" => array("CBD_testing" => array("cardinality" => array("rdf:type" => 1)))));
$config['queue'] = array("database" => "queue", "collection" => "q_queue", "data_source" => "db");
// Override the config defined in base test class as we need specific config here.
\Tripod\Mongo\Config::setConfig($config);
$tripod = new \Tripod\Mongo\Driver('CBD_testing', 'tripod_php_testing', array('defaultContext' => 'http://talisaspire.com/'));
$oldGraph = new \Tripod\ExtendedGraph();
$newGraph = new \Tripod\ExtendedGraph();
$newGraph->add_resource_triple('http://foo/bar/1', 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type', 'http://foo/bar#Class1');
$newGraph->add_resource_triple('http://foo/bar/1', 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type', 'http://foo/bar#Class2');
$tripod->saveChanges($oldGraph, $newGraph, "http://talisaspire.com/");
}
public function testArithmeticSpecValidationInvalidNestedOperator()
{
$newConfig = \Tripod\Mongo\Config::getConfig();
\Tripod\Mongo\Config::setValidationLevel(\Tripod\Mongo\Config::VALIDATE_MAX);
$arithmeticFunction = array('fieldName' => 'fooBar', 'value' => array('arithmetic' => array(array(101, '#', 100), '*', 3)));
$newConfig['stores']['tripod_php_testing']['table_specifications'][0]['computed_fields'] = array($arithmeticFunction);
\Tripod\Mongo\Config::setConfig($newConfig);
$this->setExpectedException('\\Tripod\\Exceptions\\ConfigException', "Invalid arithmetic operator '#' in computed arithmetic spec");
\Tripod\Mongo\Config::getInstance();
}
