public function testGetMongoDateWithParam()
{
$config = \Tripod\Mongo\Config::getInstance();
$updatedAt = (new \Tripod\Mongo\DateUtil())->getMongoDate();
$_id = array('r' => 'http://talisaspire.com/resources/testEtag' . microtime(false), 'c' => 'http://talisaspire.com/');
$doc = array('_id' => $_id, 'dct:title' => array('l' => 'etag'), '_version' => 0, '_cts' => $updatedAt, '_uts' => $updatedAt);
$config->getCollectionForCBD('tripod_php_testing', 'CBD_testing')->insertOne($doc, array("w" => 1));
$time = floor(microtime(true) * 1000);
$date = \Tripod\Mongo\DateUtil::getMongoDate($time);
$this->assertInstanceOf('\\MongoDB\\BSON\\UTCDateTime', $date);
$this->assertEquals(13, strlen($date->__toString()));
$this->assertEquals($time, $date->__toString());
}
/**
* @param string $storeName
* @param string $podName
* @param string|null $fromDate only transactions after this specified date will be replayed. This must be a datetime string i.e. '2010-01-15 00:00:00'
* @param string|null $toDate only transactions after this specified date will be replayed. This must be a datetime string i.e. '2010-01-15 00:00:00'
* @return Cursor
* @throws \InvalidArgumentException
*/
public function getCompletedTransactions($storeName = null, $podName = null, $fromDate = null, $toDate = null)
{
$query = array();
$query['status'] = 'completed';
if (!empty($storeName) && !empty($podName)) {
$query['dbName'] = $storeName;
$query['collectionName'] = $podName;
}
if (!empty($fromDate)) {
$q = array();
$q['$gte'] = \Tripod\Mongo\DateUtil::getMongoDate(strtotime($fromDate) * 1000);
if (!empty($toDate)) {
$q['$lte'] = \Tripod\Mongo\DateUtil::getMongoDate(strtotime($toDate) * 1000);
}
$query['endTime'] = $q;
}
return $this->transaction_collection->find($query, array('sort' => array('endTime' => 1)));
}
/**
* Apply a specific modifier
* Options you can use are
* lowercase - no options
* join - pass in "glue":" " to specify what to glue multiple values together with
* date - no options
* @param string $modifier
* @param string $value
* @param array $options
* @throws \Exception
* @return mixed
*/
private function applyModifier($modifier, $value, $options = array())
{
try {
switch ($modifier) {
case 'predicates':
// Used to generate a list of values - does nothing here
break;
case 'lowercase':
if (is_array($value)) {
$value = array_map('strtolower', $value);
} else {
$value = strtolower($value);
}
break;
case 'join':
if (is_array($value)) {
$value = implode($options['glue'], $value);
}
break;
case 'date':
if (is_string($value)) {
$value = \Tripod\Mongo\DateUtil::getMongoDate(strtotime($value) * 1000);
}
break;
default:
throw new \Exception("Could not apply modifier:" . $modifier);
break;
}
} catch (\Exception $e) {
throw $e;
}
return $value;
}
public function testGenerateViewWithCountAggregate()
{
$expiryDate = \Tripod\Mongo\DateUtil::getMongoDate((time() + 300) * 1000);
/**
* @var $mockTripodViews \Tripod\Mongo\Composites\Views
*/
$mockTripodViews = $this->getMock('\\Tripod\\Mongo\\Composites\\Views', array('getExpirySecFromNow'), $this->viewsConstParams);
$mockTripodViews->expects($this->once())->method('getExpirySecFromNow')->with(300)->will($this->returnValue(time() + 300));
$mockTripodViews->getViewForResource("http://talisaspire.com/works/4d101f63c10a6", "v_counts");
$expectedView = array("_id" => array("r" => "http://talisaspire.com/works/4d101f63c10a6", "c" => "http://talisaspire.com/", "type" => "v_counts"), "value" => array(_GRAPHS => array(array("_id" => array("r" => "http://talisaspire.com/works/4d101f63c10a6-2", "c" => "http://talisaspire.com/"), "rdf:type" => array(array("u" => "bibo:Book"), array("u" => "acorn:Work")), "acorn:resourceCount" => array("l" => "0"), "acorn:isbnCount" => array("l" => "1")), array("_id" => array("r" => "http://talisaspire.com/works/4d101f63c10a6", "c" => "http://talisaspire.com/"), "rdf:type" => array(array("u" => "bibo:Book"), array("u" => "acorn:Work")), "acorn:seeAlso" => array("u" => "http://talisaspire.com/works/4d101f63c10a6-2"), "acorn:resourceCount" => array("l" => "2"), "acorn:resourceCountAlt" => array("l" => "0"), "acorn:isbnCount" => array("l" => "2"))), _EXPIRES => $expiryDate));
$actualView = \Tripod\Mongo\Config::getInstance()->getCollectionForView('tripod_php_testing', 'v_counts')->findOne(array('_id' => array("r" => 'http://talisaspire.com/works/4d101f63c10a6', "c" => "http://talisaspire.com/", "type" => 'v_counts')));
$this->assertEquals($expectedView, $actualView);
}
/**
* helper method
* @param string $id
* @param string $subjectOfChange
* @param string $startTime
* @param string $endTime
* @param int $_version
* @return array
*/
protected function buildTransactionDocument($id, $subjectOfChange, $startTime, $endTime, $_version)
{
$transaction_template = array('_id' => "transaction_{$id}", 'changes' => array(array('_id' => array('r' => '_:cs0', 'c' => 'http://talisaspire.com/'), 'rdf:type' => array('u' => 'cs:ChangeSet'), 'cs:subjectOfChange' => array('u' => $subjectOfChange), 'cs:createdDate' => array('l' => date('c')), 'cs:creatorName' => array('l' => 'Unit Test'), 'cs:changeReason' => array('l' => 'Unit Test'), 'cs:addition' => array('u' => '_:Add1')), array('_id' => array('r' => '_:Add1', 'c' => 'http://talisaspire.com/'), 'rdf:type' => array('u' => 'rdf:Statement'), 'rdf:subject' => array('u' => $subjectOfChange), 'rdf:predicate' => array('u' => "searchterms:title"), 'rdf:object' => array('l' => "anything at all"))), 'collectionName' => 'CBD_testing', 'dbName' => 'tripod_php_testing', 'startTime' => \Tripod\Mongo\DateUtil::getMongoDate(strtotime($startTime) * 1000), 'endTime' => \Tripod\Mongo\DateUtil::getMongoDate(strtotime($endTime) * 1000), 'status' => 'completed', 'newCBDs' => array(array("_id" => array('r' => $subjectOfChange, 'c' => 'http://talisaspire.com/'), "searchterms:title" => array('l' => 'anything at all'), "_version" => $_version, "_uts" => \Tripod\Mongo\DateUtil::getMongoDate(), "_cts" => \Tripod\Mongo\DateUtil::getMongoDate())), 'originalCBDs' => array(array("_id" => array('r' => $subjectOfChange, 'c' => 'http://talisaspire.com/'))));
return $transaction_template;
}
/**
* @param array $query
* @param string $type
* @param Collection|null $collection
* @param array $includeProperties
* @param int $cursorSize
* @return MongoGraph
*/
protected function fetchGraph($query, $type, $collection = null, $includeProperties = array(), $cursorSize = 101)
{
$graph = new MongoGraph();
$t = new \Tripod\Timer();
$t->start();
if ($collection == null) {
$collection = $this->collection;
$collectionName = $collection->getCollectionName();
} else {
$collectionName = $collection->getCollectionName();
}
if (empty($includeProperties)) {
$cursor = $collection->find($query);
} else {
$fields = array();
foreach ($includeProperties as $property) {
$fields[$this->labeller->uri_to_alias($property)] = true;
}
$cursor = $collection->find($query, array('projection' => $fields, 'batchSize' => $cursorSize));
}
$ttlExpiredResources = false;
$retries = 1;
$exception = null;
$cursorSuccess = false;
do {
try {
foreach ($cursor as $result) {
// handle MONGO_VIEWS that have expired due to ttl. These are expired
// on read (lazily) rather than on write
if ($type == MONGO_VIEW && array_key_exists(_EXPIRES, $result['value'])) {
// if expires < current date, regenerate view..
$currentDate = \Tripod\Mongo\DateUtil::getMongoDate();
if ($result['value'][_EXPIRES]->__toString() < $currentDate) {
// regenerate!
$this->generateView($result['_id']['type'], $result['_id']['r']);
}
}
$graph->add_tripod_array($result);
}
$cursorSuccess = true;
} catch (\Exception $e) {
self::getLogger()->error("CursorException attempt " . $retries . ". Retrying...:" . $e->getMessage());
sleep(1);
$retries++;
$exception = $e;
}
} while ($retries <= Config::CONNECTION_RETRIES && $cursorSuccess === false);
if ($cursorSuccess === false) {
self::getLogger()->error("CursorException failed after " . $retries . " attempts (MAX:" . Config::CONNECTION_RETRIES . "): " . $e->getMessage());
throw new \Exception($exception);
}
if ($ttlExpiredResources) {
// generate views and retry...
$this->debugLog("One or more view had exceeded TTL was regenerated - request again...");
$graph = $this->fetchGraph($query, $type, $collection);
}
$t->stop();
$this->timingLog($type, array('duration' => $t->result(), 'query' => $query, 'collection' => $collectionName));
if ($type == MONGO_VIEW) {
if (array_key_exists("_id.type", $query)) {
$this->getStat()->timer("{$type}.{$query["_id.type"]}", $t->result());
} else {
if (array_key_exists("_id", $query) && array_key_exists("type", $query["_id"])) {
$this->getStat()->timer("{$type}.{$query["_id"]["type"]}", $t->result());
}
}
} else {
$this->getStat()->timer("{$type}.{$collectionName}", $t->result());
}
return $graph;
}
/**
* Given a specific $viewId, generates a single view for the $resource
* @param string $viewId
* @param string|null $resource
* @param string|null $context
* @param string|null $queueName Queue for background bulk generation
* @throws \Tripod\Exceptions\ViewException
* @return array
*/
public function generateView($viewId, $resource = null, $context = null, $queueName = null)
{
$contextAlias = $this->getContextAlias($context);
$viewSpec = Config::getInstance()->getViewSpecification($this->storeName, $viewId);
if ($viewSpec == null) {
$this->debugLog("Could not find a view specification for {$resource} with viewId '{$viewId}'");
return null;
} else {
$t = new \Tripod\Timer();
$t->start();
$from = $this->getFromCollectionForViewSpec($viewSpec);
$collection = $this->config->getCollectionForView($this->storeName, $viewId);
if (!isset($viewSpec['joins'])) {
throw new \Tripod\Exceptions\ViewException('Could not find any joins in view specification - usecase better served with select()');
}
// ensure that the ID field, view type, and the impactIndex indexes are correctly set up
$collection->createIndex(array('_id.r' => 1, '_id.c' => 1, '_id.type' => 1), array('background' => 1));
$collection->createIndex(array('_id.type' => 1), array('background' => 1));
$collection->createIndex(array('value.' . _IMPACT_INDEX => 1), array('background' => 1));
// ensure any custom view indexes
if (isset($viewSpec['ensureIndexes'])) {
foreach ($viewSpec['ensureIndexes'] as $ensureIndex) {
$collection->createIndex($ensureIndex, array('background' => 1));
}
}
$types = array();
// this is used to filter the CBD table to speed up the view creation
if (is_array($viewSpec["type"])) {
foreach ($viewSpec["type"] as $type) {
$types[] = array("rdf:type.u" => $this->labeller->qname_to_alias($type));
$types[] = array("rdf:type.u" => $this->labeller->uri_to_alias($type));
}
} else {
$types[] = array("rdf:type.u" => $this->labeller->qname_to_alias($viewSpec["type"]));
$types[] = array("rdf:type.u" => $this->labeller->uri_to_alias($viewSpec["type"]));
}
$filter = array('$or' => $types);
if (isset($resource)) {
$resourceAlias = $this->labeller->uri_to_alias($resource);
$filter["_id"] = array(_ID_RESOURCE => $resourceAlias, _ID_CONTEXT => $contextAlias);
}
$docs = $this->config->getCollectionForCBD($this->storeName, $from)->find($filter, array('maxTimeMS' => \Tripod\Mongo\Config::getInstance()->getMongoCursorTimeout()));
foreach ($docs as $doc) {
if ($queueName && !$resource) {
$subject = new ImpactedSubject($doc['_id'], OP_VIEWS, $this->storeName, $from, array($viewId));
$jobOptions = array();
if ($this->stat || !empty($this->statsConfig)) {
$jobOptions['statsConfig'] = $this->getStatsConfig();
}
$this->getApplyOperation()->createJob(array($subject), $queueName, $jobOptions);
} else {
// set up ID
$id = array("_id" => array(_ID_RESOURCE => $doc["_id"][_ID_RESOURCE], _ID_CONTEXT => $doc["_id"][_ID_CONTEXT], _ID_TYPE => $viewSpec['_id']));
$generatedView = $id;
$value = array();
// everything must go in the value object todo: this is a hang over from map reduce days, engineer out once we have stability on new PHP method for M/R
$value[_GRAPHS] = array();
$buildImpactIndex = true;
if (isset($viewSpec['ttl'])) {
$buildImpactIndex = false;
$value[_EXPIRES] = \Tripod\Mongo\DateUtil::getMongoDate($this->getExpirySecFromNow($viewSpec['ttl']) * 1000);
} else {
$value[_IMPACT_INDEX] = array($doc['_id']);
}
$this->doJoins($doc, $viewSpec['joins'], $value, $from, $contextAlias, $buildImpactIndex);
// add top level properties
$value[_GRAPHS][] = $this->extractProperties($doc, $viewSpec, $from);
$generatedView['value'] = $value;
$collection->replaceOne($id, $generatedView, ['upsert' => true]);
}
}
$t->stop();
$this->timingLog(MONGO_CREATE_VIEW, array('view' => $viewSpec['type'], 'duration' => $t->result(), 'filter' => $filter, 'from' => $from));
$this->getStat()->timer(MONGO_CREATE_VIEW . ".{$viewId}", $t->result());
}
}
/**
* This is a private method that performs exactly the same operation as Driver::lockSingleDocument, the reason this is duplicated here
* is so that we can simulate the correct locking of documents as part of mocking a workflow that will lock a document correctly but not another
* @param $s
* @param $transaction_id
* @param $contextAlias
* @return array
*/
public function lockSingleDocumentCallback($s, $transaction_id, $contextAlias)
{
$lCollection = \Tripod\Mongo\Config::getInstance()->getCollectionForLocks($this->tripod->getStoreName());
$countEntriesInLocksCollection = $lCollection->count(array('_id' => array(_ID_RESOURCE => $this->labeller->uri_to_alias($s), _ID_CONTEXT => $contextAlias)));
if ($countEntriesInLocksCollection > 0) {
//Subject is already locked
return false;
} else {
try {
//Add a entry to locks collection for this subject, will throws exception if an entry already there
$result = $lCollection->insertOne(array('_id' => array(_ID_RESOURCE => $this->labeller->uri_to_alias($s), _ID_CONTEXT => $contextAlias), _LOCKED_FOR_TRANS => $transaction_id, _LOCKED_FOR_TRANS_TS => \Tripod\Mongo\DateUtil::getMongoDate()), array("w" => 1));
if (!$result->isAcknowledged()) {
throw new Exception("Failed to lock document with error message- " . $this->getLastDBError());
}
} catch (Exception $e) {
//Subject is already locked or unable to lock
$this->debugLog(MONGO_LOCK, array('description' => 'Driver::lockSingleDocument - failed with exception', 'transaction_id' => $transaction_id, 'subject' => $s, 'exception-message' => $e->getMessage()));
return false;
}
//Let's get original document for processing.
$document = $this->getTripodCollection($this->tripod)->findOne(array('_id' => array(_ID_RESOURCE => $this->labeller->uri_to_alias($s), _ID_CONTEXT => $contextAlias)));
if (empty($document)) {
//if document is not there, create it
try {
$result = $this->getTripodCollection($this->tripod)->insertOne(array('_id' => array(_ID_RESOURCE => $this->labeller->uri_to_alias($s), _ID_CONTEXT => $contextAlias)), array("w" => 1));
if (!$result->isAcknowledged()) {
throw new Exception("Failed to create new document with error message- " . $this->getLastDBError());
}
$document = $this->getTripodCollection($this->tripod)->findOne(array('_id' => array(_ID_RESOURCE => $this->labeller->uri_to_alias($s), _ID_CONTEXT => $contextAlias)));
} catch (\Exception $e) {
$this->errorLog(MONGO_LOCK, array('description' => 'Driver::lockSingleDocument - failed when creating new document', 'transaction_id' => $transaction_id, 'subject' => $s, 'exception-message' => $e->getMessage()));
return false;
}
}
return $document;
}
}
/** START: getETag tests */
public function testEtagIsMicrotimeFormat()
{
$config = \Tripod\Mongo\Config::getInstance();
$updatedAt = \Tripod\Mongo\DateUtil::getMongoDate();
$_id = array('r' => 'http://talisaspire.com/resources/testEtag', 'c' => 'http://talisaspire.com/');
$doc = array('_id' => $_id, 'dct:title' => array('l' => 'etag'), '_version' => 0, '_cts' => $updatedAt, '_uts' => $updatedAt);
$config->getCollectionForCBD('tripod_php_testing', 'CBD_testing')->insertOne($doc, array("w" => 1));
$tripod = new \Tripod\Mongo\Driver('CBD_testing', 'tripod_php_testing', array('defaultContext' => 'http://talisaspire.com/'));
$this->assertRegExp('/^0.[0-9]{8} [0-9]{10}/', $tripod->getETag($_id['r']));
}
/**
* Returns a count according to the $query and $groupBy conditions
* @param array $query Mongo query object
* @param null $groupBy
* @param null $ttl acceptable time to live if you're willing to accept a cached version of this request
* @return array|int
*/
public function getCount($query, $groupBy = null, $ttl = null)
{
$t = new \Tripod\Timer();
$t->start();
$id = null;
$results = null;
if (!empty($ttl)) {
$id['query'] = $query;
$id['groupBy'] = $groupBy;
$this->debugLog("Looking in cache", array("id" => $id));
$candidate = $this->config->getCollectionForTTLCache($this->storeName)->findOne(array(_ID_KEY => $id));
if (!empty($candidate)) {
$this->debugLog("Found candidate", array("candidate" => $candidate));
$ttlTo = \Tripod\Mongo\DateUtil::getMongoDate(((int) $candidate['created']->__toString() / 1000 + $ttl) * 1000);
if ($ttlTo > \Tripod\Mongo\DateUtil::getMongoDate()) {
// cache hit!
$this->debugLog("Cache hit", array("id" => $id));
$results = $candidate['results'];
} else {
// cache miss
$this->debugLog("Cache miss", array("id" => $id));
}
}
}
if (empty($results)) {
if ($groupBy) {
$ops = [['$match' => $query], ['$group' => [_ID_KEY => '$' . $groupBy, 'total' => ['$sum' => 1]]]];
$cursor = $this->collection->aggregate($ops);
foreach ($cursor as $doc) {
if (!is_array($doc[_ID_KEY])) {
$results[$doc[_ID_KEY]] = $doc['total'];
} else {
$results[implode(';', $doc[_ID_KEY])] = $doc['total'];
}
}
} else {
$results = $this->collection->count($query);
}
if (!empty($ttl)) {
// add to cache
$cachedResults = array();
$cachedResults[_ID_KEY] = $id;
$cachedResults['results'] = $results;
$cachedResults['created'] = \Tripod\Mongo\DateUtil::getMongoDate();
$this->debugLog("Adding result to cache", $cachedResults);
$result = $this->config->getCollectionForTTLCache($this->storeName)->insertOne($cachedResults);
if (!$result->isAcknowledged()) {
$this->debugLog("Insert cache result not acknowledged");
}
}
}
$t->stop();
$op = $groupBy ? MONGO_GROUP : MONGO_COUNT;
$this->timingLog($op, array('duration' => $t->result(), 'query' => $query));
$this->getStat()->timer("{$op}.{$this->podName}", $t->result());
return $results;
}
/**
* @param string $subject
* @param string $transaction_id
*/
protected function lockDocument($subject, $transaction_id)
{
$collection = \Tripod\Mongo\Config::getInstance()->getCollectionForLocks('tripod_php_testing');
$labeller = new \Tripod\Mongo\Labeller();
$doc = array('_id' => array(_ID_RESOURCE => $labeller->uri_to_alias($subject), _ID_CONTEXT => \Tripod\Mongo\Config::getInstance()->getDefaultContextAlias()), _LOCKED_FOR_TRANS => $transaction_id, _LOCKED_FOR_TRANS_TS => \Tripod\Mongo\DateUtil::getMongoDate());
$collection->insertOne($doc, array("w" => 1));
}
/**
* @return \MongoDB\BSON\UTCDateTime
*/
protected function getMongoDate()
{
return \Tripod\Mongo\DateUtil::getMongoDate();
}
