public function run()
{
$match = array('sid' => $this->sid, 'nid' => $this->nid, 'mongodate' => array('$gte' => new MongoDate($this->start->getTimestamp()), '$lte' => new MongoDate($this->end->getTimestamp())));
$cursor = $this->conn->col('cv')->find($match, array('mongodate' => 1, 'vals' => 1));
$vals = array();
foreach ($cursor as $doc) {
$timestamp = $doc['mongodate']->sec;
foreach ($doc['vals'] as $seconds => $value) {
if ($value === null) {
continue;
}
$vals[$timestamp + $seconds] = $value;
}
}
return new Series($vals);
}
/**
* @param $cvs CounterValue[]
*
* @return CounterValue[]
*/
public function run(array $cvs)
{
if (count($cvs) == 0) {
return $cvs;
}
$col = $this->conn->col('cv');
$batchUpdate = new MongoUpdateBatch($col);
foreach ($cvs as $cv) {
$timestamp = $cv->datetime->getTimestamp();
$secondsIntoThisDay = $timestamp % 86400;
$mongodate = new MongoDate($timestamp - $secondsIntoThisDay);
$batchUpdate->add(['q' => ['sid' => $cv->sid, 'nid' => $cv->nid, 'mongodate' => $mongodate], 'u' => ['$set' => ['vals.' . $secondsIntoThisDay => $cv->value]], 'upsert' => true]);
}
$batchUpdate->execute(['w' => 1]);
return $cvs;
}
public function test_StoreAnomaly_RetrieveGroupedAndSortedByNofAnomalies()
{
/* This tries to test something that's a bit difficult to test.
* The anomalies should be retrieved in sorted order from highest number of anomalies to lowest,
* and contain all the anomalies for each group.
*
* We don't test limit and minCount here
*/
$start = new DateTime('-5 minutes');
$end = new DateTime('now');
// Group 1
$this->conn->col('anomalies')->insert(['nid' => '1', 'sid' => '1', 'predicted' => 30, 'actual' => 70, 'mongodate' => new MongoDate($start->getTimestamp())]);
$this->conn->col('anomalies')->insert(['nid' => '1', 'sid' => '1', 'predicted' => 40, 'actual' => 80, 'mongodate' => new MongoDate($end->getTimestamp())]);
$this->conn->col('anomalies')->insert(['nid' => '1', 'sid' => '1', 'predicted' => 45, 'actual' => 85, 'mongodate' => new MongoDate($end->getTimestamp())]);
// Group 2
$this->conn->col('anomalies')->insert(['nid' => '2', 'sid' => '1', 'predicted' => 50, 'actual' => 90, 'mongodate' => new MongoDate($start->getTimestamp())]);
// Group 3
$this->conn->col('anomalies')->insert(['nid' => '2', 'sid' => '2', 'predicted' => 60, 'actual' => 100, 'mongodate' => new MongoDate($start->getTimestamp())]);
$this->conn->col('anomalies')->insert(['nid' => '2', 'sid' => '2', 'predicted' => 60, 'actual' => 100, 'mongodate' => new MongoDate($end->getTimestamp())]);
$pipeline = new Pipeline();
$pipelineFactory = new Factory($this->conn);
$sequence = $pipelineFactory->createAnomalyAction($start, $end, ['1', '2'], ['1', '2'], 1, 3);
$res = $pipeline->run($sequence);
$this->assertEquals(3, count($res), 'Expected number of groups were wrong');
$this->assertEquals(3, count($res[0]['anomalies']), 'Expected number of anomalies in grp1 were wrong');
$this->assertEquals(2, count($res[1]['anomalies']), 'Expected number of anomalies in grp3 were wrong');
$this->assertEquals(1, count($res[2]['anomalies']), 'Expected number of anomalies in grp2 were wrong');
}
/**
* @param $cvs CounterValue[]
*
* @return CounterValue[]
*/
public function run(array $cvs)
{
$cvsDelta = [];
$col = $this->conn->col('cv_prev');
foreach ($cvs as $cv) {
if (!$cv->incremental) {
$cvsDelta[] = $cv;
continue;
}
$timestamp = $cv->datetime->getTimestamp();
$mongodate = new MongoDate($timestamp);
$doc = $col->findOne(['sid' => $cv->sid, 'nid' => $cv->nid], ['mongodate' => 1, 'value' => 1]);
if ($doc) {
$delta = $this->getDeltaValue($cv->value, $doc['value'], $timestamp, $doc['mongodate']->sec);
if ($delta !== false) {
$cv->value = $delta;
$cvsDelta[] = $cv;
}
}
$col->update(['sid' => $cv->sid, 'nid' => $cv->nid], ['$set' => ['mongodate' => $mongodate, 'value' => $cv->value]], ['upsert' => true]);
}
return $cvsDelta;
}
public function setUp()
{
$this->conn = new Connection('127.0.0.1', 'test', 'test');
$this->conn->db()->drop();
}
<?php
require __DIR__ . '/../../vendor/autoload.php';
use Mongotd\Connection;
use Mongotd\CounterValue;
use Mongotd\Pipeline\Factory;
use Mongotd\Pipeline\Pipeline;
use Mongotd\Storage;
use Mongotd\StorageMiddleware\FilterCounterValues;
use Mongotd\StorageMiddleware\InsertCounterValues;
$config = json_decode(file_get_contents('retrievalLoadTestConfig.json'), true);
date_default_timezone_set($config['timezone']);
$start = new \DateTime($config['starttime']);
$end = new \DateTime('now');
$conn = new Connection($config['dbhost'], $config['dbname'], $config['dbprefix']);
$sid = '1';
$nid = '1';
$pipelineFactory = new Factory($conn);
$pipeline = new Pipeline();
$storage = new Storage();
$storage->addMiddleware(new FilterCounterValues());
$storage->addMiddleware(new InsertCounterValues($conn));
if ($config['doInsertion']) {
$conn->db()->drop();
$conn->createIndexes();
$dateperiod = new \DatePeriod($start, DateInterval::createFromDateString($config['insertIntervalInSeconds'] . ' seconds'), $end);
$cvs = [];
foreach ($dateperiod as $datetime) {
$cvs[] = new CounterValue($sid, $nid, $datetime, rand());
}
$storage->store($cvs);
use Mongotd\CounterValue;
use Mongotd\Logger;
use Mongotd\Pipeline\Factory;
use Mongotd\Pipeline\Pipeline;
use Mongotd\Storage;
use Mongotd\StorageMiddleware\CalculateDeltas;
use Mongotd\StorageMiddleware\FilterCounterValues;
use Mongotd\StorageMiddleware\FindAnomaliesUsingHwTest;
use Mongotd\StorageMiddleware\FindAnomaliesUsingKsTest;
use Mongotd\StorageMiddleware\FindAnomaliesUsingSigmaTest;
use Mongotd\StorageMiddleware\InsertCounterValues;
use Mongotd\StorageMiddleware\StoreAnomalies;
$config = json_decode(file_get_contents('loadTestConfig.json'), true);
date_default_timezone_set($config['timezone']);
$start = new \DateTime($config['starttime']);
$conn = new Connection($config['dbhost'], $config['dbname'], $config['dbprefix']);
$conn->db()->drop();
$conn->createIndexes();
$pipelineFactory = new Factory($conn);
$pipeline = new Pipeline();
$storage = new Storage();
$storage->addMiddleware(new FilterCounterValues());
$storage->addMiddleware(new CalculateDeltas($conn, new Logger(null), $config['insertIntervalInSeconds']));
$storage->addMiddleware(new InsertCounterValues($conn));
if ($config['anomalyDetectionMethod'] == 'ks') {
$storage->addMiddleware(new FindAnomaliesUsingKsTest($conn));
$storage->addMiddleware(new StoreAnomalies($conn));
} else {
if ($config['anomalyDetectionMethod'] == 'hw') {
$storage->addMiddleware(new FindAnomaliesUsingHwTest($conn));
$storage->addMiddleware(new StoreAnomalies($conn));
