public function test_StoreTwoIncrementalValues_RetrievesDifference()
{
$resolution15min = Resolution::FIFTEEEN_MINUTES;
$someAggregation = Aggregation::SUM;
$storage = new Storage();
$storage->setDefaultMiddleware($this->conn, new Logger(null), $resolution15min);
$pipelineFactory = new Factory($this->conn);
$pipeline = new Pipeline();
$someSid = 1;
$someNid = 1;
$someValue1 = 600;
$someValue2 = 700;
$expectedDifference = $someValue2 - $someValue1;
$firstDatetime = new DateTime('2015-02-18 15:00:00');
$secondDateStr15MinAfterFirst = '2015-02-18 15:15:00';
$secondDatetime15MinAfterFirst = new DateTime($secondDateStr15MinAfterFirst);
$incrementalIsTrue = true;
$expectedValsByDate = array($secondDateStr15MinAfterFirst => $expectedDifference);
$storage->store([new CounterValue($someSid, $someNid, $firstDatetime, $someValue1, $incrementalIsTrue)]);
$storage->store([new CounterValue($someSid, $someNid, $secondDatetime15MinAfterFirst, $someValue2, $incrementalIsTrue)]);
$sequence = $pipelineFactory->createMultiAction($someSid, $someNid, $secondDatetime15MinAfterFirst, $secondDatetime15MinAfterFirst, $resolution15min, $someAggregation);
$sequence[] = new ConvertToDateStringKeys();
$valsByDate = $pipeline->run($sequence);
$this->assertEquals($expectedValsByDate, $valsByDate);
}
/**
* @dataProvider useCases
*
* @param $config
*/
public function test_UseCase($config)
{
date_default_timezone_set($config['timezone']);
$timezone = new DateTimeZone($config['timezone']);
$start = new DateTime($config['start'], $timezone);
$end = new DateTime($config['end'], $timezone);
$storage = new Storage();
$storage->setDefaultMiddleware($this->conn, new Logger(null));
$pipelineFactory = new Factory($this->conn);
$pipeline = new Pipeline();
$cvs = [];
$dateperiod = new DatePeriod($start, DateInterval::createFromDateString($config['insertResolution'] . ' seconds'), $end);
foreach ($dateperiod as $datetime) {
$cvs[] = new CounterValue($config['sid'], $config['nid'], $datetime, $config['val'], $config['incremental']);
}
$storage->store($cvs);
$sequence = $pipelineFactory->createMultiAction($config['sid'], $config['nid'], $start, $end, $config['retrieveResolution'], $config['aggregation']);
$sequence[] = new ConvertToDateStringKeys();
$valsByDate = $pipeline->run($sequence);
$msg = $config['description'];
$msg .= "\nExpected\n";
$msg .= json_encode($config['expected'], JSON_PRETTY_PRINT);
$msg .= "\nActual:\n";
$msg .= json_encode($valsByDate, JSON_PRETTY_PRINT);
$this->assertTrue($valsByDate === $config['expected'], $msg);
}
public function test_StoreAnomaly_RetrieveGroupedAndSortedByNofAnomalies()
{
/* This tries to test something that's a bit difficult to test.
* The anomalies should be retrieved in sorted order from highest number of anomalies to lowest,
* and contain all the anomalies for each group.
*
* We don't test limit and minCount here
*/
$start = new DateTime('-5 minutes');
$end = new DateTime('now');
// Group 1
$this->conn->col('anomalies')->insert(['nid' => '1', 'sid' => '1', 'predicted' => 30, 'actual' => 70, 'mongodate' => new MongoDate($start->getTimestamp())]);
$this->conn->col('anomalies')->insert(['nid' => '1', 'sid' => '1', 'predicted' => 40, 'actual' => 80, 'mongodate' => new MongoDate($end->getTimestamp())]);
$this->conn->col('anomalies')->insert(['nid' => '1', 'sid' => '1', 'predicted' => 45, 'actual' => 85, 'mongodate' => new MongoDate($end->getTimestamp())]);
// Group 2
$this->conn->col('anomalies')->insert(['nid' => '2', 'sid' => '1', 'predicted' => 50, 'actual' => 90, 'mongodate' => new MongoDate($start->getTimestamp())]);
// Group 3
$this->conn->col('anomalies')->insert(['nid' => '2', 'sid' => '2', 'predicted' => 60, 'actual' => 100, 'mongodate' => new MongoDate($start->getTimestamp())]);
$this->conn->col('anomalies')->insert(['nid' => '2', 'sid' => '2', 'predicted' => 60, 'actual' => 100, 'mongodate' => new MongoDate($end->getTimestamp())]);
$pipeline = new Pipeline();
$pipelineFactory = new Factory($this->conn);
$sequence = $pipelineFactory->createAnomalyAction($start, $end, ['1', '2'], ['1', '2'], 1, 3);
$res = $pipeline->run($sequence);
$this->assertEquals(3, count($res), 'Expected number of groups were wrong');
$this->assertEquals(3, count($res[0]['anomalies']), 'Expected number of anomalies in grp1 were wrong');
$this->assertEquals(2, count($res[1]['anomalies']), 'Expected number of anomalies in grp3 were wrong');
$this->assertEquals(1, count($res[2]['anomalies']), 'Expected number of anomalies in grp2 were wrong');
}
public function test_StoreValues_RetrieveAsFormula_FormulaIsCorrectlyCalculated()
{
$formula = '[sid=1,agg=1] + [sid=2,agg=1]';
$sid1 = 1;
$sid2 = 2;
$someNid = 1;
$valSid1 = 15;
$valSid2 = 30;
$expectedSum = $valSid1 + $valSid2;
$someDateString = '2015-02-18 15:00:00';
$datetime = new DateTime($someDateString);
$someFormulaResolution = Resolution::FIVE_MINUTES;
$someResultResolution = Resolution::FIVE_MINUTES;
$sumResultAggregation = Aggregation::SUM;
$someResolution = Resolution::FIVE_MINUTES;
$isIncremental = false;
$padding = false;
$storage = new Storage();
$storage->setDefaultMiddleware($this->conn, new Logger(null), $someResolution);
$pipeline = new Pipeline();
$pipelineFactory = new Factory($this->conn);
$expected = array($someDateString => $expectedSum);
$cvs = [];
$cvs[] = new CounterValue($sid1, $someNid, $datetime, $valSid1, $isIncremental);
$cvs[] = new CounterValue($sid2, $someNid, $datetime, $valSid2, $isIncremental);
$storage->store($cvs);
$sequence = $pipelineFactory->createMultiAction($formula, $someNid, $datetime, $datetime, $someResultResolution, $sumResultAggregation, $padding, null, null, null, true, $someFormulaResolution);
$sequence[] = new ConvertToDateStringKeys();
$output = $pipeline->run($sequence);
$this->assertEquals($expected, $output);
}
protected function getState(array $anomalies)
{
$statesByTimestamp = [];
foreach ($anomalies as $anomaly) {
$statesByTimestamp[$anomaly->cv->datetime->getTimestamp()] = 1;
}
$series = new Series($statesByTimestamp);
$pipeline = new Pipeline();
return $pipeline->run([new RollupTime($this->resolution, Aggregation::MAX), new Pad($this->resolution, $this->start, $this->end, 0)], $series)->vals;
}
/**
* @dataProvider rollupTimeAndPadProvider
*
* @param $config
*/
public function test_RollupTimeAndPad($config)
{
date_default_timezone_set($config['timezone']);
$vals = [];
foreach ($config['vals'] as $dateStr => $value) {
$vals[(new DateTime($dateStr))->getTimestamp()] = $value;
}
$series = new Series($vals);
$find = $this->getMockBuilder('Mongotd\\Pipeline\\Find')->disableOriginalConstructor()->getMock();
$find->method('run')->willReturn($series);
$pipeline = new Pipeline();
$output = $pipeline->run([$find, new RollupTime($config['resolution'], $config['aggregation'], $config['padding']), new Pad($config['resolution'], new DateTime($config['start']), new DateTime($config['end']), $config['padding']), new ConvertToDateStringKeys()]);
$this->assertEquals($config['expected'], $output);
}
public function createFormulaAction($formula, $nid, DateTime $start, DateTime $end, $resolution, $padding)
{
$parser = new Parser();
$ast = $parser->parse($formula);
$astEvaluator = new AstEvaluator();
$astEvaluator->setPaddingValue($padding);
$astEvaluator->setVariableEvaluatorCallback(function ($options) use($nid, $start, $end, $resolution, $padding) {
if (!isset($options['sid'])) {
throw new Exception('sid was not specified in variable. Need this to determine which sensor to get for the calculation of the formula.');
}
if (!isset($options['agg'])) {
throw new Exception('agg was not specified in variable. Need this in order to aggregate up to the correct resolution before calculating formula.');
}
$pipeline = new Pipeline();
return $pipeline->run([new Find($this->conn, $options['sid'], $nid, $start, $end), new RollupTime($resolution, $options['agg'], $padding), new Pad($resolution, $start, $end, $padding)])->vals;
});
return new Formula($ast, $astEvaluator);
}
public function getWindowedVals($sid, $nid, DateTime $end, $nDays)
{
// Set STARTTIME = ENDTIME - DAYS - WINDOWLENGTH + 1
// NB: Each window includes 0...N-1. We want to include the ending (N), so start at 1 instead of 0.
$start = clone $end;
$start->sub(DateInterval::createFromDateString($nDays . ' days'));
$start->sub(DateInterval::createFromDateString($this->windowLengthInSeconds - 1 . ' seconds'));
$pipeline = new Pipeline();
$series = $pipeline->run([new Find($this->conn, $sid, $nid, $start, $end), new FilterWindow($start, $this->windowLengthInSeconds, 86400)]);
return array_values($series->vals);
}
use Mongotd\Connection;
use Mongotd\CounterValue;
use Mongotd\Pipeline\Factory;
use Mongotd\Pipeline\Pipeline;
use Mongotd\Storage;
use Mongotd\StorageMiddleware\FilterCounterValues;
use Mongotd\StorageMiddleware\InsertCounterValues;
$config = json_decode(file_get_contents('retrievalLoadTestConfig.json'), true);
date_default_timezone_set($config['timezone']);
$start = new \DateTime($config['starttime']);
$end = new \DateTime('now');
$conn = new Connection($config['dbhost'], $config['dbname'], $config['dbprefix']);
$sid = '1';
$nid = '1';
$pipelineFactory = new Factory($conn);
$pipeline = new Pipeline();
$storage = new Storage();
$storage->addMiddleware(new FilterCounterValues());
$storage->addMiddleware(new InsertCounterValues($conn));
if ($config['doInsertion']) {
$conn->db()->drop();
$conn->createIndexes();
$dateperiod = new \DatePeriod($start, DateInterval::createFromDateString($config['insertIntervalInSeconds'] . ' seconds'), $end);
$cvs = [];
foreach ($dateperiod as $datetime) {
$cvs[] = new CounterValue($sid, $nid, $datetime, rand());
}
$storage->store($cvs);
echo "Insertion done\n";
}
$timerStart = microtime(true);
case 'DAY':
return Resolution::DAY;
case 'HOUR':
return Resolution::HOUR;
case 'FIFTEEN_MINUTES':
return Resolution::FIFTEEEN_MINUTES;
case 'FIVE_MINUTES':
return Resolution::FIVE_MINUTES;
case 'ONE_MINUTE':
return Resolution::MINUTE;
}
throw new Exception('Invalid resolution');
}
$conn = new Connection('127.0.0.1', 'test', 'test');
$pipelineFactory = new Factory($conn);
$pipeline = new Pipeline();
$asFormula = false;
$nids = array(1, 2);
$sids = array(1, 2);
$start = "-60 days";
$end = "now";
$aggregation = Aggregation::SUM;
$resolution = Resolution::HOUR;
$combinedNodeAggregation = Aggregation::SUM;
$singleNodeAggregation = Aggregation::SUM;
$nodeResolution = Resolution::FIVE_MINUTES;
$formulaResolution = Resolution::FIVE_MINUTES;
if (isset($_GET['asFormula'])) {
$asFormula = $_GET['asFormula'] === 'true';
}
if ($asFormula) {
use Mongotd\Connection;
use Mongotd\Pipeline\Factory;
use Mongotd\Pipeline\Pipeline;
use Mongotd\Storage;
use Mongotd\StorageMiddleware\FilterCounterValues;
use Mongotd\StorageMiddleware\InsertCounterValues;
use Mongotd\StorageMiddleware\FindAnomaliesUsingSigmaTest;
use Mongotd\StorageMiddleware\FindAnomaliesUsingHwTest;
use Mongotd\StorageMiddleware\FindAnomaliesUsingKsTest;
use Mongotd\StorageMiddleware\StoreAnomalies;
$config = json_decode(file_get_contents('anomalyTestConfig.json'), true);
$conn = new Connection($config['dbhost'], $config['dbname'], $config['dbprefix']);
$conn->db()->drop();
$conn->createIndexes();
$pipelineFactory = new Factory($conn);
$pipeline = new Pipeline();
$storage = new Storage();
$storage->addMiddleware(new FilterCounterValues());
$storage->addMiddleware(new InsertCounterValues($conn));
if ($config['anomalyDetectionMethod'] == 'ks') {
$storage->addMiddleware(new FindAnomaliesUsingKsTest($conn));
} else {
if ($config['anomalyDetectionMethod'] == 'hw') {
$storage->addMiddleware(new FindAnomaliesUsingHwTest($conn));
} else {
if ($config['anomalyDetectionMethod'] == 'sigma') {
$storage->addMiddleware(new FindAnomaliesUsingSigmaTest($conn));
} else {
throw new Exception('Unknown anomaly scan method');
}
}
