public function testGetMongoDateWithParam() { $config = \Tripod\Mongo\Config::getInstance(); $updatedAt = (new \Tripod\Mongo\DateUtil())->getMongoDate(); $_id = array('r' => 'http://talisaspire.com/resources/testEtag' . microtime(false), 'c' => 'http://talisaspire.com/'); $doc = array('_id' => $_id, 'dct:title' => array('l' => 'etag'), '_version' => 0, '_cts' => $updatedAt, '_uts' => $updatedAt); $config->getCollectionForCBD('tripod_php_testing', 'CBD_testing')->insertOne($doc, array("w" => 1)); $time = floor(microtime(true) * 1000); $date = \Tripod\Mongo\DateUtil::getMongoDate($time); $this->assertInstanceOf('\\MongoDB\\BSON\\UTCDateTime', $date); $this->assertEquals(13, strlen($date->__toString())); $this->assertEquals($time, $date->__toString()); }
/** * @param string $storeName * @param string $podName * @param string|null $fromDate only transactions after this specified date will be replayed. This must be a datetime string i.e. '2010-01-15 00:00:00' * @param string|null $toDate only transactions after this specified date will be replayed. This must be a datetime string i.e. '2010-01-15 00:00:00' * @return Cursor * @throws \InvalidArgumentException */ public function getCompletedTransactions($storeName = null, $podName = null, $fromDate = null, $toDate = null) { $query = array(); $query['status'] = 'completed'; if (!empty($storeName) && !empty($podName)) { $query['dbName'] = $storeName; $query['collectionName'] = $podName; } if (!empty($fromDate)) { $q = array(); $q['$gte'] = \Tripod\Mongo\DateUtil::getMongoDate(strtotime($fromDate) * 1000); if (!empty($toDate)) { $q['$lte'] = \Tripod\Mongo\DateUtil::getMongoDate(strtotime($toDate) * 1000); } $query['endTime'] = $q; } return $this->transaction_collection->find($query, array('sort' => array('endTime' => 1))); }
/** * Apply a specific modifier * Options you can use are * lowercase - no options * join - pass in "glue":" " to specify what to glue multiple values together with * date - no options * @param string $modifier * @param string $value * @param array $options * @throws \Exception * @return mixed */ private function applyModifier($modifier, $value, $options = array()) { try { switch ($modifier) { case 'predicates': // Used to generate a list of values - does nothing here break; case 'lowercase': if (is_array($value)) { $value = array_map('strtolower', $value); } else { $value = strtolower($value); } break; case 'join': if (is_array($value)) { $value = implode($options['glue'], $value); } break; case 'date': if (is_string($value)) { $value = \Tripod\Mongo\DateUtil::getMongoDate(strtotime($value) * 1000); } break; default: throw new \Exception("Could not apply modifier:" . $modifier); break; } } catch (\Exception $e) { throw $e; } return $value; }
public function testGenerateViewWithCountAggregate() { $expiryDate = \Tripod\Mongo\DateUtil::getMongoDate((time() + 300) * 1000); /** * @var $mockTripodViews \Tripod\Mongo\Composites\Views */ $mockTripodViews = $this->getMock('\\Tripod\\Mongo\\Composites\\Views', array('getExpirySecFromNow'), $this->viewsConstParams); $mockTripodViews->expects($this->once())->method('getExpirySecFromNow')->with(300)->will($this->returnValue(time() + 300)); $mockTripodViews->getViewForResource("http://talisaspire.com/works/4d101f63c10a6", "v_counts"); $expectedView = array("_id" => array("r" => "http://talisaspire.com/works/4d101f63c10a6", "c" => "http://talisaspire.com/", "type" => "v_counts"), "value" => array(_GRAPHS => array(array("_id" => array("r" => "http://talisaspire.com/works/4d101f63c10a6-2", "c" => "http://talisaspire.com/"), "rdf:type" => array(array("u" => "bibo:Book"), array("u" => "acorn:Work")), "acorn:resourceCount" => array("l" => "0"), "acorn:isbnCount" => array("l" => "1")), array("_id" => array("r" => "http://talisaspire.com/works/4d101f63c10a6", "c" => "http://talisaspire.com/"), "rdf:type" => array(array("u" => "bibo:Book"), array("u" => "acorn:Work")), "acorn:seeAlso" => array("u" => "http://talisaspire.com/works/4d101f63c10a6-2"), "acorn:resourceCount" => array("l" => "2"), "acorn:resourceCountAlt" => array("l" => "0"), "acorn:isbnCount" => array("l" => "2"))), _EXPIRES => $expiryDate)); $actualView = \Tripod\Mongo\Config::getInstance()->getCollectionForView('tripod_php_testing', 'v_counts')->findOne(array('_id' => array("r" => 'http://talisaspire.com/works/4d101f63c10a6', "c" => "http://talisaspire.com/", "type" => 'v_counts'))); $this->assertEquals($expectedView, $actualView); }
/** * helper method * @param string $id * @param string $subjectOfChange * @param string $startTime * @param string $endTime * @param int $_version * @return array */ protected function buildTransactionDocument($id, $subjectOfChange, $startTime, $endTime, $_version) { $transaction_template = array('_id' => "transaction_{$id}", 'changes' => array(array('_id' => array('r' => '_:cs0', 'c' => 'http://talisaspire.com/'), 'rdf:type' => array('u' => 'cs:ChangeSet'), 'cs:subjectOfChange' => array('u' => $subjectOfChange), 'cs:createdDate' => array('l' => date('c')), 'cs:creatorName' => array('l' => 'Unit Test'), 'cs:changeReason' => array('l' => 'Unit Test'), 'cs:addition' => array('u' => '_:Add1')), array('_id' => array('r' => '_:Add1', 'c' => 'http://talisaspire.com/'), 'rdf:type' => array('u' => 'rdf:Statement'), 'rdf:subject' => array('u' => $subjectOfChange), 'rdf:predicate' => array('u' => "searchterms:title"), 'rdf:object' => array('l' => "anything at all"))), 'collectionName' => 'CBD_testing', 'dbName' => 'tripod_php_testing', 'startTime' => \Tripod\Mongo\DateUtil::getMongoDate(strtotime($startTime) * 1000), 'endTime' => \Tripod\Mongo\DateUtil::getMongoDate(strtotime($endTime) * 1000), 'status' => 'completed', 'newCBDs' => array(array("_id" => array('r' => $subjectOfChange, 'c' => 'http://talisaspire.com/'), "searchterms:title" => array('l' => 'anything at all'), "_version" => $_version, "_uts" => \Tripod\Mongo\DateUtil::getMongoDate(), "_cts" => \Tripod\Mongo\DateUtil::getMongoDate())), 'originalCBDs' => array(array("_id" => array('r' => $subjectOfChange, 'c' => 'http://talisaspire.com/')))); return $transaction_template; }
/** * @param array $query * @param string $type * @param Collection|null $collection * @param array $includeProperties * @param int $cursorSize * @return MongoGraph */ protected function fetchGraph($query, $type, $collection = null, $includeProperties = array(), $cursorSize = 101) { $graph = new MongoGraph(); $t = new \Tripod\Timer(); $t->start(); if ($collection == null) { $collection = $this->collection; $collectionName = $collection->getCollectionName(); } else { $collectionName = $collection->getCollectionName(); } if (empty($includeProperties)) { $cursor = $collection->find($query); } else { $fields = array(); foreach ($includeProperties as $property) { $fields[$this->labeller->uri_to_alias($property)] = true; } $cursor = $collection->find($query, array('projection' => $fields, 'batchSize' => $cursorSize)); } $ttlExpiredResources = false; $retries = 1; $exception = null; $cursorSuccess = false; do { try { foreach ($cursor as $result) { // handle MONGO_VIEWS that have expired due to ttl. These are expired // on read (lazily) rather than on write if ($type == MONGO_VIEW && array_key_exists(_EXPIRES, $result['value'])) { // if expires < current date, regenerate view.. $currentDate = \Tripod\Mongo\DateUtil::getMongoDate(); if ($result['value'][_EXPIRES]->__toString() < $currentDate) { // regenerate! $this->generateView($result['_id']['type'], $result['_id']['r']); } } $graph->add_tripod_array($result); } $cursorSuccess = true; } catch (\Exception $e) { self::getLogger()->error("CursorException attempt " . $retries . ". Retrying...:" . $e->getMessage()); sleep(1); $retries++; $exception = $e; } } while ($retries <= Config::CONNECTION_RETRIES && $cursorSuccess === false); if ($cursorSuccess === false) { self::getLogger()->error("CursorException failed after " . $retries . " attempts (MAX:" . Config::CONNECTION_RETRIES . "): " . $e->getMessage()); throw new \Exception($exception); } if ($ttlExpiredResources) { // generate views and retry... $this->debugLog("One or more view had exceeded TTL was regenerated - request again..."); $graph = $this->fetchGraph($query, $type, $collection); } $t->stop(); $this->timingLog($type, array('duration' => $t->result(), 'query' => $query, 'collection' => $collectionName)); if ($type == MONGO_VIEW) { if (array_key_exists("_id.type", $query)) { $this->getStat()->timer("{$type}.{$query["_id.type"]}", $t->result()); } else { if (array_key_exists("_id", $query) && array_key_exists("type", $query["_id"])) { $this->getStat()->timer("{$type}.{$query["_id"]["type"]}", $t->result()); } } } else { $this->getStat()->timer("{$type}.{$collectionName}", $t->result()); } return $graph; }
/** * Given a specific $viewId, generates a single view for the $resource * @param string $viewId * @param string|null $resource * @param string|null $context * @param string|null $queueName Queue for background bulk generation * @throws \Tripod\Exceptions\ViewException * @return array */ public function generateView($viewId, $resource = null, $context = null, $queueName = null) { $contextAlias = $this->getContextAlias($context); $viewSpec = Config::getInstance()->getViewSpecification($this->storeName, $viewId); if ($viewSpec == null) { $this->debugLog("Could not find a view specification for {$resource} with viewId '{$viewId}'"); return null; } else { $t = new \Tripod\Timer(); $t->start(); $from = $this->getFromCollectionForViewSpec($viewSpec); $collection = $this->config->getCollectionForView($this->storeName, $viewId); if (!isset($viewSpec['joins'])) { throw new \Tripod\Exceptions\ViewException('Could not find any joins in view specification - usecase better served with select()'); } // ensure that the ID field, view type, and the impactIndex indexes are correctly set up $collection->createIndex(array('_id.r' => 1, '_id.c' => 1, '_id.type' => 1), array('background' => 1)); $collection->createIndex(array('_id.type' => 1), array('background' => 1)); $collection->createIndex(array('value.' . _IMPACT_INDEX => 1), array('background' => 1)); // ensure any custom view indexes if (isset($viewSpec['ensureIndexes'])) { foreach ($viewSpec['ensureIndexes'] as $ensureIndex) { $collection->createIndex($ensureIndex, array('background' => 1)); } } $types = array(); // this is used to filter the CBD table to speed up the view creation if (is_array($viewSpec["type"])) { foreach ($viewSpec["type"] as $type) { $types[] = array("rdf:type.u" => $this->labeller->qname_to_alias($type)); $types[] = array("rdf:type.u" => $this->labeller->uri_to_alias($type)); } } else { $types[] = array("rdf:type.u" => $this->labeller->qname_to_alias($viewSpec["type"])); $types[] = array("rdf:type.u" => $this->labeller->uri_to_alias($viewSpec["type"])); } $filter = array('$or' => $types); if (isset($resource)) { $resourceAlias = $this->labeller->uri_to_alias($resource); $filter["_id"] = array(_ID_RESOURCE => $resourceAlias, _ID_CONTEXT => $contextAlias); } $docs = $this->config->getCollectionForCBD($this->storeName, $from)->find($filter, array('maxTimeMS' => \Tripod\Mongo\Config::getInstance()->getMongoCursorTimeout())); foreach ($docs as $doc) { if ($queueName && !$resource) { $subject = new ImpactedSubject($doc['_id'], OP_VIEWS, $this->storeName, $from, array($viewId)); $jobOptions = array(); if ($this->stat || !empty($this->statsConfig)) { $jobOptions['statsConfig'] = $this->getStatsConfig(); } $this->getApplyOperation()->createJob(array($subject), $queueName, $jobOptions); } else { // set up ID $id = array("_id" => array(_ID_RESOURCE => $doc["_id"][_ID_RESOURCE], _ID_CONTEXT => $doc["_id"][_ID_CONTEXT], _ID_TYPE => $viewSpec['_id'])); $generatedView = $id; $value = array(); // everything must go in the value object todo: this is a hang over from map reduce days, engineer out once we have stability on new PHP method for M/R $value[_GRAPHS] = array(); $buildImpactIndex = true; if (isset($viewSpec['ttl'])) { $buildImpactIndex = false; $value[_EXPIRES] = \Tripod\Mongo\DateUtil::getMongoDate($this->getExpirySecFromNow($viewSpec['ttl']) * 1000); } else { $value[_IMPACT_INDEX] = array($doc['_id']); } $this->doJoins($doc, $viewSpec['joins'], $value, $from, $contextAlias, $buildImpactIndex); // add top level properties $value[_GRAPHS][] = $this->extractProperties($doc, $viewSpec, $from); $generatedView['value'] = $value; $collection->replaceOne($id, $generatedView, ['upsert' => true]); } } $t->stop(); $this->timingLog(MONGO_CREATE_VIEW, array('view' => $viewSpec['type'], 'duration' => $t->result(), 'filter' => $filter, 'from' => $from)); $this->getStat()->timer(MONGO_CREATE_VIEW . ".{$viewId}", $t->result()); } }
/** * This is a private method that performs exactly the same operation as Driver::lockSingleDocument, the reason this is duplicated here * is so that we can simulate the correct locking of documents as part of mocking a workflow that will lock a document correctly but not another * @param $s * @param $transaction_id * @param $contextAlias * @return array */ public function lockSingleDocumentCallback($s, $transaction_id, $contextAlias) { $lCollection = \Tripod\Mongo\Config::getInstance()->getCollectionForLocks($this->tripod->getStoreName()); $countEntriesInLocksCollection = $lCollection->count(array('_id' => array(_ID_RESOURCE => $this->labeller->uri_to_alias($s), _ID_CONTEXT => $contextAlias))); if ($countEntriesInLocksCollection > 0) { //Subject is already locked return false; } else { try { //Add a entry to locks collection for this subject, will throws exception if an entry already there $result = $lCollection->insertOne(array('_id' => array(_ID_RESOURCE => $this->labeller->uri_to_alias($s), _ID_CONTEXT => $contextAlias), _LOCKED_FOR_TRANS => $transaction_id, _LOCKED_FOR_TRANS_TS => \Tripod\Mongo\DateUtil::getMongoDate()), array("w" => 1)); if (!$result->isAcknowledged()) { throw new Exception("Failed to lock document with error message- " . $this->getLastDBError()); } } catch (Exception $e) { //Subject is already locked or unable to lock $this->debugLog(MONGO_LOCK, array('description' => 'Driver::lockSingleDocument - failed with exception', 'transaction_id' => $transaction_id, 'subject' => $s, 'exception-message' => $e->getMessage())); return false; } //Let's get original document for processing. $document = $this->getTripodCollection($this->tripod)->findOne(array('_id' => array(_ID_RESOURCE => $this->labeller->uri_to_alias($s), _ID_CONTEXT => $contextAlias))); if (empty($document)) { //if document is not there, create it try { $result = $this->getTripodCollection($this->tripod)->insertOne(array('_id' => array(_ID_RESOURCE => $this->labeller->uri_to_alias($s), _ID_CONTEXT => $contextAlias)), array("w" => 1)); if (!$result->isAcknowledged()) { throw new Exception("Failed to create new document with error message- " . $this->getLastDBError()); } $document = $this->getTripodCollection($this->tripod)->findOne(array('_id' => array(_ID_RESOURCE => $this->labeller->uri_to_alias($s), _ID_CONTEXT => $contextAlias))); } catch (\Exception $e) { $this->errorLog(MONGO_LOCK, array('description' => 'Driver::lockSingleDocument - failed when creating new document', 'transaction_id' => $transaction_id, 'subject' => $s, 'exception-message' => $e->getMessage())); return false; } } return $document; } }
/** START: getETag tests */ public function testEtagIsMicrotimeFormat() { $config = \Tripod\Mongo\Config::getInstance(); $updatedAt = \Tripod\Mongo\DateUtil::getMongoDate(); $_id = array('r' => 'http://talisaspire.com/resources/testEtag', 'c' => 'http://talisaspire.com/'); $doc = array('_id' => $_id, 'dct:title' => array('l' => 'etag'), '_version' => 0, '_cts' => $updatedAt, '_uts' => $updatedAt); $config->getCollectionForCBD('tripod_php_testing', 'CBD_testing')->insertOne($doc, array("w" => 1)); $tripod = new \Tripod\Mongo\Driver('CBD_testing', 'tripod_php_testing', array('defaultContext' => 'http://talisaspire.com/')); $this->assertRegExp('/^0.[0-9]{8} [0-9]{10}/', $tripod->getETag($_id['r'])); }
/** * Returns a count according to the $query and $groupBy conditions * @param array $query Mongo query object * @param null $groupBy * @param null $ttl acceptable time to live if you're willing to accept a cached version of this request * @return array|int */ public function getCount($query, $groupBy = null, $ttl = null) { $t = new \Tripod\Timer(); $t->start(); $id = null; $results = null; if (!empty($ttl)) { $id['query'] = $query; $id['groupBy'] = $groupBy; $this->debugLog("Looking in cache", array("id" => $id)); $candidate = $this->config->getCollectionForTTLCache($this->storeName)->findOne(array(_ID_KEY => $id)); if (!empty($candidate)) { $this->debugLog("Found candidate", array("candidate" => $candidate)); $ttlTo = \Tripod\Mongo\DateUtil::getMongoDate(((int) $candidate['created']->__toString() / 1000 + $ttl) * 1000); if ($ttlTo > \Tripod\Mongo\DateUtil::getMongoDate()) { // cache hit! $this->debugLog("Cache hit", array("id" => $id)); $results = $candidate['results']; } else { // cache miss $this->debugLog("Cache miss", array("id" => $id)); } } } if (empty($results)) { if ($groupBy) { $ops = [['$match' => $query], ['$group' => [_ID_KEY => '$' . $groupBy, 'total' => ['$sum' => 1]]]]; $cursor = $this->collection->aggregate($ops); foreach ($cursor as $doc) { if (!is_array($doc[_ID_KEY])) { $results[$doc[_ID_KEY]] = $doc['total']; } else { $results[implode(';', $doc[_ID_KEY])] = $doc['total']; } } } else { $results = $this->collection->count($query); } if (!empty($ttl)) { // add to cache $cachedResults = array(); $cachedResults[_ID_KEY] = $id; $cachedResults['results'] = $results; $cachedResults['created'] = \Tripod\Mongo\DateUtil::getMongoDate(); $this->debugLog("Adding result to cache", $cachedResults); $result = $this->config->getCollectionForTTLCache($this->storeName)->insertOne($cachedResults); if (!$result->isAcknowledged()) { $this->debugLog("Insert cache result not acknowledged"); } } } $t->stop(); $op = $groupBy ? MONGO_GROUP : MONGO_COUNT; $this->timingLog($op, array('duration' => $t->result(), 'query' => $query)); $this->getStat()->timer("{$op}.{$this->podName}", $t->result()); return $results; }
/** * @param string $subject * @param string $transaction_id */ protected function lockDocument($subject, $transaction_id) { $collection = \Tripod\Mongo\Config::getInstance()->getCollectionForLocks('tripod_php_testing'); $labeller = new \Tripod\Mongo\Labeller(); $doc = array('_id' => array(_ID_RESOURCE => $labeller->uri_to_alias($subject), _ID_CONTEXT => \Tripod\Mongo\Config::getInstance()->getDefaultContextAlias()), _LOCKED_FOR_TRANS => $transaction_id, _LOCKED_FOR_TRANS_TS => \Tripod\Mongo\DateUtil::getMongoDate()); $collection->insertOne($doc, array("w" => 1)); }
/** * @return \MongoDB\BSON\UTCDateTime */ protected function getMongoDate() { return \Tripod\Mongo\DateUtil::getMongoDate(); }