public function test_StoreTwoIncrementalValues_RetrievesDifference()
{
$resolution15min = Resolution::FIFTEEEN_MINUTES;
$someAggregation = Aggregation::SUM;
$storage = new Storage();
$storage->setDefaultMiddleware($this->conn, new Logger(null), $resolution15min);
$pipelineFactory = new Factory($this->conn);
$pipeline = new Pipeline();
$someSid = 1;
$someNid = 1;
$someValue1 = 600;
$someValue2 = 700;
$expectedDifference = $someValue2 - $someValue1;
$firstDatetime = new DateTime('2015-02-18 15:00:00');
$secondDateStr15MinAfterFirst = '2015-02-18 15:15:00';
$secondDatetime15MinAfterFirst = new DateTime($secondDateStr15MinAfterFirst);
$incrementalIsTrue = true;
$expectedValsByDate = array($secondDateStr15MinAfterFirst => $expectedDifference);
$storage->store([new CounterValue($someSid, $someNid, $firstDatetime, $someValue1, $incrementalIsTrue)]);
$storage->store([new CounterValue($someSid, $someNid, $secondDatetime15MinAfterFirst, $someValue2, $incrementalIsTrue)]);
$sequence = $pipelineFactory->createMultiAction($someSid, $someNid, $secondDatetime15MinAfterFirst, $secondDatetime15MinAfterFirst, $resolution15min, $someAggregation);
$sequence[] = new ConvertToDateStringKeys();
$valsByDate = $pipeline->run($sequence);
$this->assertEquals($expectedValsByDate, $valsByDate);
}
/**
* @dataProvider useCases
*
* @param $config
*/
public function test_UseCase($config)
{
date_default_timezone_set($config['timezone']);
$timezone = new DateTimeZone($config['timezone']);
$start = new DateTime($config['start'], $timezone);
$end = new DateTime($config['end'], $timezone);
$storage = new Storage();
$storage->setDefaultMiddleware($this->conn, new Logger(null));
$pipelineFactory = new Factory($this->conn);
$pipeline = new Pipeline();
$cvs = [];
$dateperiod = new DatePeriod($start, DateInterval::createFromDateString($config['insertResolution'] . ' seconds'), $end);
foreach ($dateperiod as $datetime) {
$cvs[] = new CounterValue($config['sid'], $config['nid'], $datetime, $config['val'], $config['incremental']);
}
$storage->store($cvs);
$sequence = $pipelineFactory->createMultiAction($config['sid'], $config['nid'], $start, $end, $config['retrieveResolution'], $config['aggregation']);
$sequence[] = new ConvertToDateStringKeys();
$valsByDate = $pipeline->run($sequence);
$msg = $config['description'];
$msg .= "\nExpected\n";
$msg .= json_encode($config['expected'], JSON_PRETTY_PRINT);
$msg .= "\nActual:\n";
$msg .= json_encode($valsByDate, JSON_PRETTY_PRINT);
$this->assertTrue($valsByDate === $config['expected'], $msg);
}
public function test_StoreAnomaly_RetrieveGroupedAndSortedByNofAnomalies()
{
/* This tries to test something that's a bit difficult to test.
* The anomalies should be retrieved in sorted order from highest number of anomalies to lowest,
* and contain all the anomalies for each group.
*
* We don't test limit and minCount here
*/
$start = new DateTime('-5 minutes');
$end = new DateTime('now');
// Group 1
$this->conn->col('anomalies')->insert(['nid' => '1', 'sid' => '1', 'predicted' => 30, 'actual' => 70, 'mongodate' => new MongoDate($start->getTimestamp())]);
$this->conn->col('anomalies')->insert(['nid' => '1', 'sid' => '1', 'predicted' => 40, 'actual' => 80, 'mongodate' => new MongoDate($end->getTimestamp())]);
$this->conn->col('anomalies')->insert(['nid' => '1', 'sid' => '1', 'predicted' => 45, 'actual' => 85, 'mongodate' => new MongoDate($end->getTimestamp())]);
// Group 2
$this->conn->col('anomalies')->insert(['nid' => '2', 'sid' => '1', 'predicted' => 50, 'actual' => 90, 'mongodate' => new MongoDate($start->getTimestamp())]);
// Group 3
$this->conn->col('anomalies')->insert(['nid' => '2', 'sid' => '2', 'predicted' => 60, 'actual' => 100, 'mongodate' => new MongoDate($start->getTimestamp())]);
$this->conn->col('anomalies')->insert(['nid' => '2', 'sid' => '2', 'predicted' => 60, 'actual' => 100, 'mongodate' => new MongoDate($end->getTimestamp())]);
$pipeline = new Pipeline();
$pipelineFactory = new Factory($this->conn);
$sequence = $pipelineFactory->createAnomalyAction($start, $end, ['1', '2'], ['1', '2'], 1, 3);
$res = $pipeline->run($sequence);
$this->assertEquals(3, count($res), 'Expected number of groups were wrong');
$this->assertEquals(3, count($res[0]['anomalies']), 'Expected number of anomalies in grp1 were wrong');
$this->assertEquals(2, count($res[1]['anomalies']), 'Expected number of anomalies in grp3 were wrong');
$this->assertEquals(1, count($res[2]['anomalies']), 'Expected number of anomalies in grp2 were wrong');
}
public function test_StoreValues_RetrieveAsFormula_FormulaIsCorrectlyCalculated()
{
$formula = '[sid=1,agg=1] + [sid=2,agg=1]';
$sid1 = 1;
$sid2 = 2;
$someNid = 1;
$valSid1 = 15;
$valSid2 = 30;
$expectedSum = $valSid1 + $valSid2;
$someDateString = '2015-02-18 15:00:00';
$datetime = new DateTime($someDateString);
$someFormulaResolution = Resolution::FIVE_MINUTES;
$someResultResolution = Resolution::FIVE_MINUTES;
$sumResultAggregation = Aggregation::SUM;
$someResolution = Resolution::FIVE_MINUTES;
$isIncremental = false;
$padding = false;
$storage = new Storage();
$storage->setDefaultMiddleware($this->conn, new Logger(null), $someResolution);
$pipeline = new Pipeline();
$pipelineFactory = new Factory($this->conn);
$expected = array($someDateString => $expectedSum);
$cvs = [];
$cvs[] = new CounterValue($sid1, $someNid, $datetime, $valSid1, $isIncremental);
$cvs[] = new CounterValue($sid2, $someNid, $datetime, $valSid2, $isIncremental);
$storage->store($cvs);
$sequence = $pipelineFactory->createMultiAction($formula, $someNid, $datetime, $datetime, $someResultResolution, $sumResultAggregation, $padding, null, null, null, true, $someFormulaResolution);
$sequence[] = new ConvertToDateStringKeys();
$output = $pipeline->run($sequence);
$this->assertEquals($expected, $output);
}
protected function getState(array $anomalies)
{
$statesByTimestamp = [];
foreach ($anomalies as $anomaly) {
$statesByTimestamp[$anomaly->cv->datetime->getTimestamp()] = 1;
}
$series = new Series($statesByTimestamp);
$pipeline = new Pipeline();
return $pipeline->run([new RollupTime($this->resolution, Aggregation::MAX), new Pad($this->resolution, $this->start, $this->end, 0)], $series)->vals;
}
/**
* @dataProvider rollupTimeAndPadProvider
*
* @param $config
*/
public function test_RollupTimeAndPad($config)
{
date_default_timezone_set($config['timezone']);
$vals = [];
foreach ($config['vals'] as $dateStr => $value) {
$vals[(new DateTime($dateStr))->getTimestamp()] = $value;
}
$series = new Series($vals);
$find = $this->getMockBuilder('Mongotd\\Pipeline\\Find')->disableOriginalConstructor()->getMock();
$find->method('run')->willReturn($series);
$pipeline = new Pipeline();
$output = $pipeline->run([$find, new RollupTime($config['resolution'], $config['aggregation'], $config['padding']), new Pad($config['resolution'], new DateTime($config['start']), new DateTime($config['end']), $config['padding']), new ConvertToDateStringKeys()]);
$this->assertEquals($config['expected'], $output);
}
public function createFormulaAction($formula, $nid, DateTime $start, DateTime $end, $resolution, $padding)
{
$parser = new Parser();
$ast = $parser->parse($formula);
$astEvaluator = new AstEvaluator();
$astEvaluator->setPaddingValue($padding);
$astEvaluator->setVariableEvaluatorCallback(function ($options) use($nid, $start, $end, $resolution, $padding) {
if (!isset($options['sid'])) {
throw new Exception('sid was not specified in variable. Need this to determine which sensor to get for the calculation of the formula.');
}
if (!isset($options['agg'])) {
throw new Exception('agg was not specified in variable. Need this in order to aggregate up to the correct resolution before calculating formula.');
}
$pipeline = new Pipeline();
return $pipeline->run([new Find($this->conn, $options['sid'], $nid, $start, $end), new RollupTime($resolution, $options['agg'], $padding), new Pad($resolution, $start, $end, $padding)])->vals;
});
return new Formula($ast, $astEvaluator);
}
public function getWindowedVals($sid, $nid, DateTime $end, $nDays)
{
// Set STARTTIME = ENDTIME - DAYS - WINDOWLENGTH + 1
// NB: Each window includes 0...N-1. We want to include the ending (N), so start at 1 instead of 0.
$start = clone $end;
$start->sub(DateInterval::createFromDateString($nDays . ' days'));
$start->sub(DateInterval::createFromDateString($this->windowLengthInSeconds - 1 . ' seconds'));
$pipeline = new Pipeline();
$series = $pipeline->run([new Find($this->conn, $sid, $nid, $start, $end), new FilterWindow($start, $this->windowLengthInSeconds, 86400)]);
return array_values($series->vals);
}
use Mongotd\Storage;
use Mongotd\StorageMiddleware\FilterCounterValues;
use Mongotd\StorageMiddleware\InsertCounterValues;
$config = json_decode(file_get_contents('retrievalLoadTestConfig.json'), true);
date_default_timezone_set($config['timezone']);
$start = new \DateTime($config['starttime']);
$end = new \DateTime('now');
$conn = new Connection($config['dbhost'], $config['dbname'], $config['dbprefix']);
$sid = '1';
$nid = '1';
$pipelineFactory = new Factory($conn);
$pipeline = new Pipeline();
$storage = new Storage();
$storage->addMiddleware(new FilterCounterValues());
$storage->addMiddleware(new InsertCounterValues($conn));
if ($config['doInsertion']) {
$conn->db()->drop();
$conn->createIndexes();
$dateperiod = new \DatePeriod($start, DateInterval::createFromDateString($config['insertIntervalInSeconds'] . ' seconds'), $end);
$cvs = [];
foreach ($dateperiod as $datetime) {
$cvs[] = new CounterValue($sid, $nid, $datetime, rand());
}
$storage->store($cvs);
echo "Insertion done\n";
}
$timerStart = microtime(true);
$sequence = $pipelineFactory->createMultiAction($sid, $nid, $start, $end, $config['retrieveResolution'], $config['retrieveAggregation']);
$pipeline->run($sequence);
$totalTime = microtime(true) - $timerStart;
echo 'Retrieved in: ' . $totalTime . " seconds\n";
$combinedNodeAggregation = aggregationStrToEnum($_GET['combinedNodeAggregation']);
}
if (isset($_GET['nodeResolution'])) {
$nodeResolution = resolutionStrToEnum($_GET['nodeResolution']);
}
if (isset($_GET['formulaResolution'])) {
$formulaResolution = resolutionStrToEnum($_GET['formulaResolution']);
}
$flotData = array();
$start = new DateTime($start);
$end = new DateTime($end);
DateTimeHelper::normalizeTimeRange($start, $end, $resolution);
foreach ($sids as $sid) {
$sequence = $pipelineFactory->createMultiAction($sid, $nids, $start, $end, $resolution, $aggregation, false, $nodeResolution, $singleNodeAggregation, $combinedNodeAggregation, $asFormula, $formulaResolution);
$sequence[] = new ConvertToFlotFormat();
$data = $pipeline->run($sequence);
$flotData[] = array('data' => $data, 'label' => 'Sensor ' . $sid, 'xaxis' => 1, 'yaxis' => 1);
}
$sequence = $pipelineFactory->createAnomalyAction($start, $end, $nids, $sids, 0, count($nids) * count($nids));
$sequence[] = new AddAnomalyState($start, $end, $resolution);
$res = $pipeline->run($sequence);
$anomalyStates = [];
$anomalies = array();
foreach ($res as $row) {
$anomalies = array_merge($anomalies, $row['anomalies']);
foreach ($row['state'] as $timestamp => $state) {
$anomalyStates[$timestamp] = $state;
}
}
$data = array_map(function ($timestamp, $value) {
return array($timestamp * 1000, $value);
$iteration++;
$expectedAnomaly = false;
if (rand() % 1000 == 0) {
$persist = $config['anomalyPersistIntervals'];
}
if ($persist > 0) {
$persist--;
$expectedAnomaly = true;
$totalAnomalies++;
$data['value'] *= 2;
}
$storage->store([new \Mongotd\CounterValue(1, 1, $data['datetime'], $data['value'])]);
$datetimeStartInterval = clone $data['datetime'];
$datetimeStartInterval->sub(DateInterval::createFromDateString($config['insertIntervalInSeconds'] - 1 . ' seconds'));
$sequence = $pipelineFactory->createAnomalyAction($datetimeStartInterval, $data['datetime']);
$anomalyList = $pipeline->run($sequence);
$isAnomaly = count($anomalyList) > 0 and count($anomalyList[0]['anomalies']) > 0;
if ($isAnomaly or $expectedAnomaly) {
echo $data['datetime']->format('Y-m-d H:i') . ' - ' . $data['value'];
if ($isAnomaly) {
echo ' Detected anomaly! (predicted avg ' . $anomalyList[0]['anomalies'][0]->predicted . ')';
}
if ($expectedAnomaly) {
echo ' Actual anomaly!';
}
if ($isAnomaly and $expectedAnomaly) {
$hits++;
} else {
if ($isAnomaly) {
$falsesPositives++;
} else {
