/** * @param \Tripod\Mongo\ImpactedSubject[] $subjects * @param string|null $queueName * @param array $otherData */ public function createJob(array $subjects, $queueName = null, $otherData = array()) { if (!$queueName) { $queueName = \Tripod\Mongo\Config::getApplyQueueName(); } elseif (strpos($queueName, \Tripod\Mongo\Config::getApplyQueueName()) === false) { $queueName = \Tripod\Mongo\Config::getApplyQueueName() . '::' . $queueName; } $data = array(self::SUBJECTS_KEY => array_map(function (\Tripod\Mongo\ImpactedSubject $subject) { return $subject->toArray(); }, $subjects), self::TRIPOD_CONFIG_KEY => \Tripod\Mongo\Config::getConfig()); $this->submitJob($queueName, get_class($this), array_merge($otherData, $data)); }
/** * Run the DiscoverImpactedSubjects job * @throws \Exception */ public function perform() { try { $this->debugLog("[JOBID " . $this->job->payload['id'] . "] DiscoverImpactedSubjects::perform() start"); $timer = new \Tripod\Timer(); $timer->start(); $this->validateArgs(); // set the config to what is received \Tripod\Mongo\Config::setConfig($this->args[self::TRIPOD_CONFIG_KEY]); $statsConfig = array(); if (isset($this->args['statsConfig'])) { $statsConfig['statsConfig'] = $this->args['statsConfig']; } $tripod = $this->getTripod($this->args[self::STORE_NAME_KEY], $this->args[self::POD_NAME_KEY], $statsConfig); $operations = $this->args[self::OPERATIONS_KEY]; $subjectsAndPredicatesOfChange = $this->args[self::CHANGES_KEY]; $subjectCount = 0; foreach ($operations as $op) { /** @var \Tripod\Mongo\Composites\IComposite $composite */ $composite = $tripod->getComposite($op); $modifiedSubjects = $composite->getImpactedSubjects($subjectsAndPredicatesOfChange, $this->args[self::CONTEXT_ALIAS_KEY]); if (!empty($modifiedSubjects)) { /* @var $subject \Tripod\Mongo\ImpactedSubject */ foreach ($modifiedSubjects as $subject) { $subjectCount++; $subjectTimer = new \Tripod\Timer(); $subjectTimer->start(); if (isset($this->args[self::QUEUE_KEY]) || count($subject->getSpecTypes()) == 0) { $queueName = isset($this->args[self::QUEUE_KEY]) ? $this->args[self::QUEUE_KEY] : Config::getApplyQueueName(); $this->addSubjectToQueue($subject, $queueName); } else { $specsGroupedByQueue = array(); foreach ($subject->getSpecTypes() as $specType) { $spec = null; switch ($subject->getOperation()) { case OP_VIEWS: $spec = Config::getInstance()->getViewSpecification($this->args[self::STORE_NAME_KEY], $specType); break; case OP_TABLES: $spec = Config::getInstance()->getTableSpecification($this->args[self::STORE_NAME_KEY], $specType); break; case OP_SEARCH: $spec = Config::getInstance()->getSearchDocumentSpecification($this->args[self::STORE_NAME_KEY], $specType); break; } if (!$spec || !isset($spec['queue'])) { if (!$spec) { $spec = array(); } $spec['queue'] = Config::getApplyQueueName(); } if (!isset($specsGroupedByQueue[$spec['queue']])) { $specsGroupedByQueue[$spec['queue']] = array(); } $specsGroupedByQueue[$spec['queue']][] = $specType; } foreach ($specsGroupedByQueue as $queueName => $specs) { $queuedSubject = new \Tripod\Mongo\ImpactedSubject($subject->getResourceId(), $subject->getOperation(), $subject->getStoreName(), $subject->getPodName(), $specs); $this->addSubjectToQueue($queuedSubject, $queueName); } } $subjectTimer->stop(); // stat time taken to discover impacted subjects for the given subject of change $this->getStat()->timer(MONGO_QUEUE_DISCOVER_SUBJECT, $subjectTimer->result()); } if (!empty($this->subjectsGroupedByQueue)) { foreach ($this->subjectsGroupedByQueue as $queueName => $subjects) { $this->getApplyOperation()->createJob($subjects, $queueName, $statsConfig); } $this->subjectsGroupedByQueue = array(); } } } // stat time taken to process item, from time it was created (queued) $timer->stop(); $this->getStat()->timer(MONGO_QUEUE_DISCOVER_SUCCESS, $timer->result()); $this->debugLog("[JOBID " . $this->job->payload['id'] . "] DiscoverImpactedSubjects::perform() done in {$timer->result()}ms"); $this->getStat()->increment(MONGO_QUEUE_DISCOVER_JOB . '.' . SUBJECT_COUNT, $subjectCount); } catch (\Exception $e) { $this->getStat()->increment(MONGO_QUEUE_DISCOVER_FAIL); $this->errorLog("Caught exception in " . get_class($this) . ": " . $e->getMessage()); throw $e; } }
public function testCreateJobSpecifyQueue() { $impactedSubject = new \Tripod\Mongo\ImpactedSubject(array(_ID_RESOURCE => "http://example.com/1", _ID_CONTEXT => "http://talisaspire.com/"), OP_VIEWS, 'tripod_php_testing', 'CBD_testing', array()); $jobData = array('subjects' => array($impactedSubject->toArray()), 'tripodConfig' => \Tripod\Mongo\Config::getConfig()); /** @var \Tripod\Mongo\Jobs\ApplyOperation|PHPUnit_Framework_MockObject_MockObject $applyOperation */ $applyOperation = $this->getMockBuilder('\\Tripod\\Mongo\\Jobs\\ApplyOperation')->setMethods(array('submitJob'))->setMockClassName('MockApplyOperation')->getMock(); $queueName = \Tripod\Mongo\Config::getApplyQueueName() . '::TRIPOD_TESTING_QUEUE_' . uniqid(); $applyOperation->expects($this->once())->method('submitJob')->with($queueName, 'MockApplyOperation', $jobData); $applyOperation->createJob(array($impactedSubject), $queueName); }
public function testDiscoverOperationWillSubmitApplyOperationForDistinctQueues() { $config = \Tripod\Mongo\Config::getConfig(); // Create a bunch of specs on various queues $tableSpecs = array(array("_id" => "t_resource", "type" => "acorn:Resource", "from" => "CBD_testing", "ensureIndexes" => array("value.isbn" => 1), "fields" => array(array("fieldName" => "type", "predicates" => array("rdf:type")), array("fieldName" => "isbn", "predicates" => array("bibo:isbn13"))), "joins" => array("dct:isVersionOf" => array("fields" => array(array("fieldName" => "isbn13", "predicates" => array("bibo:isbn13")))))), array("_id" => "t_source_count", "type" => "acorn:Resource", "from" => "CBD_testing", "to_data_source" => "rs2", "queue" => "counts_and_other_non_essentials", "fields" => array(array("fieldName" => "type", "predicates" => array("rdf:type"))), "joins" => array("dct:isVersionOf" => array("fields" => array(array("fieldName" => "isbn13", "predicates" => array("bibo:isbn13"))))), "counts" => array(array("fieldName" => "source_count", "property" => "dct:isVersionOf"), array("fieldName" => "random_predicate_count", "property" => "dct:randomPredicate"))), array("_id" => "t_source_count_regex", "type" => "acorn:Resource", "from" => "CBD_testing", "queue" => "counts_and_other_non_essentials", "fields" => array(array("fieldName" => "type", "predicates" => array("rdf:type"))), "joins" => array("dct:isVersionOf" => array("fields" => array(array("fieldName" => "isbn13", "predicates" => array("bibo:isbn13"))))), "counts" => array(array("fieldName" => "source_count", "property" => "dct:isVersionOf"), array("fieldName" => "regex_source_count", "property" => "dct:isVersionOf", "regex" => "/foobar/"))), array("_id" => "t_join_source_count_regex", "type" => "acorn:Resource", "from" => "CBD_testing", "queue" => "MOST_IMPORTANT_QUEUE_EVER", "joins" => array("acorn:jacsUri" => array("counts" => array(array("fieldName" => "titles_count", "property" => "dct:title")))))); $config['stores']['tripod_php_testing']['table_specifications'] = $tableSpecs; \Tripod\Mongo\Config::setConfig($config); /** @var \Tripod\Mongo\Jobs\DiscoverImpactedSubjects|PHPUnit_Framework_MockObject_MockObject $discoverImpactedSubjects */ $discoverImpactedSubjects = $this->getMockBuilder('\\Tripod\\Mongo\\Jobs\\DiscoverImpactedSubjects')->setMethods(array('getTripod', 'getApplyOperation'))->getMock(); $this->setArgs(); $args = $this->args; $args['operations'] = array(OP_TABLES); $discoverImpactedSubjects->args = $args; $discoverImpactedSubjects->job->payload['id'] = uniqid(); $tripod = $this->getMockBuilder('\\Tripod\\Mongo\\Driver')->setMethods(array('getComposite'))->setConstructorArgs(array('CBD_testing', 'tripod_php_testing'))->getMock(); $tables = $this->getMockBuilder('\\Tripod\\Mongo\\Composites\\Tables')->setMethods(array('getImpactedSubjects'))->setConstructorArgs(array('tripod_php_testing', \Tripod\Mongo\Config::getInstance()->getCollectionForCBD('tripod_php_testing', 'CBD_testing'), 'http://talisaspire.com/'))->getMock(); $applyOperation = $this->getMockBuilder('\\Tripod\\Mongo\\Jobs\\ApplyOperation')->setMethods(array('createJob'))->getMock(); $tableSubject1 = new \Tripod\Mongo\ImpactedSubject(array(_ID_RESOURCE => 'http://example.com/resources/foo2', _ID_CONTEXT => $this->args['contextAlias']), OP_TABLES, $this->args['storeName'], $this->args['podName'], array('t_resource', 't_source_count', 't_source_count_regex', 't_join_source_count_regex')); $tableSubject2 = new \Tripod\Mongo\ImpactedSubject(array(_ID_RESOURCE => 'http://example.com/resources/foo3', _ID_CONTEXT => $this->args['contextAlias']), OP_TABLES, $this->args['storeName'], $this->args['podName'], array('t_resource', 't_source_count')); $queuedTable1 = array(new \Tripod\Mongo\ImpactedSubject(array(_ID_RESOURCE => 'http://example.com/resources/foo2', _ID_CONTEXT => $this->args['contextAlias']), OP_TABLES, $this->args['storeName'], $this->args['podName'], array('t_resource')), new \Tripod\Mongo\ImpactedSubject(array(_ID_RESOURCE => 'http://example.com/resources/foo3', _ID_CONTEXT => $this->args['contextAlias']), OP_TABLES, $this->args['storeName'], $this->args['podName'], array('t_resource'))); $queuedTable2 = array(new \Tripod\Mongo\ImpactedSubject(array(_ID_RESOURCE => 'http://example.com/resources/foo2', _ID_CONTEXT => $this->args['contextAlias']), OP_TABLES, $this->args['storeName'], $this->args['podName'], array('t_source_count', 't_source_count_regex')), new \Tripod\Mongo\ImpactedSubject(array(_ID_RESOURCE => 'http://example.com/resources/foo3', _ID_CONTEXT => $this->args['contextAlias']), OP_TABLES, $this->args['storeName'], $this->args['podName'], array('t_source_count'))); $queuedTable3 = array(new \Tripod\Mongo\ImpactedSubject(array(_ID_RESOURCE => 'http://example.com/resources/foo2', _ID_CONTEXT => $this->args['contextAlias']), OP_TABLES, $this->args['storeName'], $this->args['podName'], array('t_join_source_count_regex'))); $tables->expects($this->once())->method('getImpactedSubjects')->with($this->args['changes'], $this->args['contextAlias'])->will($this->returnValue(array($tableSubject1, $tableSubject2))); $tripod->expects($this->once())->method('getComposite')->with(OP_TABLES)->will($this->returnValue($tables)); $discoverImpactedSubjects->expects($this->once())->method('getTripod')->will($this->returnValue($tripod)); $discoverImpactedSubjects->expects($this->exactly(3))->method('getApplyOperation')->will($this->returnValue($applyOperation)); $applyOperation->expects($this->exactly(3))->method('createJob')->withConsecutive(array($queuedTable1, \Tripod\Mongo\Config::getApplyQueueName()), array($queuedTable2, "counts_and_other_non_essentials"), array($queuedTable3, "MOST_IMPORTANT_QUEUE_EVER")); $discoverImpactedSubjects->perform(); }