public function test_StoreTwoIncrementalValues_RetrievesDifference() { $resolution15min = Resolution::FIFTEEEN_MINUTES; $someAggregation = Aggregation::SUM; $storage = new Storage(); $storage->setDefaultMiddleware($this->conn, new Logger(null), $resolution15min); $pipelineFactory = new Factory($this->conn); $pipeline = new Pipeline(); $someSid = 1; $someNid = 1; $someValue1 = 600; $someValue2 = 700; $expectedDifference = $someValue2 - $someValue1; $firstDatetime = new DateTime('2015-02-18 15:00:00'); $secondDateStr15MinAfterFirst = '2015-02-18 15:15:00'; $secondDatetime15MinAfterFirst = new DateTime($secondDateStr15MinAfterFirst); $incrementalIsTrue = true; $expectedValsByDate = array($secondDateStr15MinAfterFirst => $expectedDifference); $storage->store([new CounterValue($someSid, $someNid, $firstDatetime, $someValue1, $incrementalIsTrue)]); $storage->store([new CounterValue($someSid, $someNid, $secondDatetime15MinAfterFirst, $someValue2, $incrementalIsTrue)]); $sequence = $pipelineFactory->createMultiAction($someSid, $someNid, $secondDatetime15MinAfterFirst, $secondDatetime15MinAfterFirst, $resolution15min, $someAggregation); $sequence[] = new ConvertToDateStringKeys(); $valsByDate = $pipeline->run($sequence); $this->assertEquals($expectedValsByDate, $valsByDate); }
/** * @dataProvider useCases * * @param $config */ public function test_UseCase($config) { date_default_timezone_set($config['timezone']); $timezone = new DateTimeZone($config['timezone']); $start = new DateTime($config['start'], $timezone); $end = new DateTime($config['end'], $timezone); $storage = new Storage(); $storage->setDefaultMiddleware($this->conn, new Logger(null)); $pipelineFactory = new Factory($this->conn); $pipeline = new Pipeline(); $cvs = []; $dateperiod = new DatePeriod($start, DateInterval::createFromDateString($config['insertResolution'] . ' seconds'), $end); foreach ($dateperiod as $datetime) { $cvs[] = new CounterValue($config['sid'], $config['nid'], $datetime, $config['val'], $config['incremental']); } $storage->store($cvs); $sequence = $pipelineFactory->createMultiAction($config['sid'], $config['nid'], $start, $end, $config['retrieveResolution'], $config['aggregation']); $sequence[] = new ConvertToDateStringKeys(); $valsByDate = $pipeline->run($sequence); $msg = $config['description']; $msg .= "\nExpected\n"; $msg .= json_encode($config['expected'], JSON_PRETTY_PRINT); $msg .= "\nActual:\n"; $msg .= json_encode($valsByDate, JSON_PRETTY_PRINT); $this->assertTrue($valsByDate === $config['expected'], $msg); }
public function test_StoreAnomaly_RetrieveGroupedAndSortedByNofAnomalies() { /* This tries to test something that's a bit difficult to test. * The anomalies should be retrieved in sorted order from highest number of anomalies to lowest, * and contain all the anomalies for each group. * * We don't test limit and minCount here */ $start = new DateTime('-5 minutes'); $end = new DateTime('now'); // Group 1 $this->conn->col('anomalies')->insert(['nid' => '1', 'sid' => '1', 'predicted' => 30, 'actual' => 70, 'mongodate' => new MongoDate($start->getTimestamp())]); $this->conn->col('anomalies')->insert(['nid' => '1', 'sid' => '1', 'predicted' => 40, 'actual' => 80, 'mongodate' => new MongoDate($end->getTimestamp())]); $this->conn->col('anomalies')->insert(['nid' => '1', 'sid' => '1', 'predicted' => 45, 'actual' => 85, 'mongodate' => new MongoDate($end->getTimestamp())]); // Group 2 $this->conn->col('anomalies')->insert(['nid' => '2', 'sid' => '1', 'predicted' => 50, 'actual' => 90, 'mongodate' => new MongoDate($start->getTimestamp())]); // Group 3 $this->conn->col('anomalies')->insert(['nid' => '2', 'sid' => '2', 'predicted' => 60, 'actual' => 100, 'mongodate' => new MongoDate($start->getTimestamp())]); $this->conn->col('anomalies')->insert(['nid' => '2', 'sid' => '2', 'predicted' => 60, 'actual' => 100, 'mongodate' => new MongoDate($end->getTimestamp())]); $pipeline = new Pipeline(); $pipelineFactory = new Factory($this->conn); $sequence = $pipelineFactory->createAnomalyAction($start, $end, ['1', '2'], ['1', '2'], 1, 3); $res = $pipeline->run($sequence); $this->assertEquals(3, count($res), 'Expected number of groups were wrong'); $this->assertEquals(3, count($res[0]['anomalies']), 'Expected number of anomalies in grp1 were wrong'); $this->assertEquals(2, count($res[1]['anomalies']), 'Expected number of anomalies in grp3 were wrong'); $this->assertEquals(1, count($res[2]['anomalies']), 'Expected number of anomalies in grp2 were wrong'); }
public function test_StoreValues_RetrieveAsFormula_FormulaIsCorrectlyCalculated() { $formula = '[sid=1,agg=1] + [sid=2,agg=1]'; $sid1 = 1; $sid2 = 2; $someNid = 1; $valSid1 = 15; $valSid2 = 30; $expectedSum = $valSid1 + $valSid2; $someDateString = '2015-02-18 15:00:00'; $datetime = new DateTime($someDateString); $someFormulaResolution = Resolution::FIVE_MINUTES; $someResultResolution = Resolution::FIVE_MINUTES; $sumResultAggregation = Aggregation::SUM; $someResolution = Resolution::FIVE_MINUTES; $isIncremental = false; $padding = false; $storage = new Storage(); $storage->setDefaultMiddleware($this->conn, new Logger(null), $someResolution); $pipeline = new Pipeline(); $pipelineFactory = new Factory($this->conn); $expected = array($someDateString => $expectedSum); $cvs = []; $cvs[] = new CounterValue($sid1, $someNid, $datetime, $valSid1, $isIncremental); $cvs[] = new CounterValue($sid2, $someNid, $datetime, $valSid2, $isIncremental); $storage->store($cvs); $sequence = $pipelineFactory->createMultiAction($formula, $someNid, $datetime, $datetime, $someResultResolution, $sumResultAggregation, $padding, null, null, null, true, $someFormulaResolution); $sequence[] = new ConvertToDateStringKeys(); $output = $pipeline->run($sequence); $this->assertEquals($expected, $output); }
protected function getState(array $anomalies) { $statesByTimestamp = []; foreach ($anomalies as $anomaly) { $statesByTimestamp[$anomaly->cv->datetime->getTimestamp()] = 1; } $series = new Series($statesByTimestamp); $pipeline = new Pipeline(); return $pipeline->run([new RollupTime($this->resolution, Aggregation::MAX), new Pad($this->resolution, $this->start, $this->end, 0)], $series)->vals; }
/** * @dataProvider rollupTimeAndPadProvider * * @param $config */ public function test_RollupTimeAndPad($config) { date_default_timezone_set($config['timezone']); $vals = []; foreach ($config['vals'] as $dateStr => $value) { $vals[(new DateTime($dateStr))->getTimestamp()] = $value; } $series = new Series($vals); $find = $this->getMockBuilder('Mongotd\\Pipeline\\Find')->disableOriginalConstructor()->getMock(); $find->method('run')->willReturn($series); $pipeline = new Pipeline(); $output = $pipeline->run([$find, new RollupTime($config['resolution'], $config['aggregation'], $config['padding']), new Pad($config['resolution'], new DateTime($config['start']), new DateTime($config['end']), $config['padding']), new ConvertToDateStringKeys()]); $this->assertEquals($config['expected'], $output); }
public function createFormulaAction($formula, $nid, DateTime $start, DateTime $end, $resolution, $padding) { $parser = new Parser(); $ast = $parser->parse($formula); $astEvaluator = new AstEvaluator(); $astEvaluator->setPaddingValue($padding); $astEvaluator->setVariableEvaluatorCallback(function ($options) use($nid, $start, $end, $resolution, $padding) { if (!isset($options['sid'])) { throw new Exception('sid was not specified in variable. Need this to determine which sensor to get for the calculation of the formula.'); } if (!isset($options['agg'])) { throw new Exception('agg was not specified in variable. Need this in order to aggregate up to the correct resolution before calculating formula.'); } $pipeline = new Pipeline(); return $pipeline->run([new Find($this->conn, $options['sid'], $nid, $start, $end), new RollupTime($resolution, $options['agg'], $padding), new Pad($resolution, $start, $end, $padding)])->vals; }); return new Formula($ast, $astEvaluator); }
public function getWindowedVals($sid, $nid, DateTime $end, $nDays) { // Set STARTTIME = ENDTIME - DAYS - WINDOWLENGTH + 1 // NB: Each window includes 0...N-1. We want to include the ending (N), so start at 1 instead of 0. $start = clone $end; $start->sub(DateInterval::createFromDateString($nDays . ' days')); $start->sub(DateInterval::createFromDateString($this->windowLengthInSeconds - 1 . ' seconds')); $pipeline = new Pipeline(); $series = $pipeline->run([new Find($this->conn, $sid, $nid, $start, $end), new FilterWindow($start, $this->windowLengthInSeconds, 86400)]); return array_values($series->vals); }
use Mongotd\Connection; use Mongotd\CounterValue; use Mongotd\Pipeline\Factory; use Mongotd\Pipeline\Pipeline; use Mongotd\Storage; use Mongotd\StorageMiddleware\FilterCounterValues; use Mongotd\StorageMiddleware\InsertCounterValues; $config = json_decode(file_get_contents('retrievalLoadTestConfig.json'), true); date_default_timezone_set($config['timezone']); $start = new \DateTime($config['starttime']); $end = new \DateTime('now'); $conn = new Connection($config['dbhost'], $config['dbname'], $config['dbprefix']); $sid = '1'; $nid = '1'; $pipelineFactory = new Factory($conn); $pipeline = new Pipeline(); $storage = new Storage(); $storage->addMiddleware(new FilterCounterValues()); $storage->addMiddleware(new InsertCounterValues($conn)); if ($config['doInsertion']) { $conn->db()->drop(); $conn->createIndexes(); $dateperiod = new \DatePeriod($start, DateInterval::createFromDateString($config['insertIntervalInSeconds'] . ' seconds'), $end); $cvs = []; foreach ($dateperiod as $datetime) { $cvs[] = new CounterValue($sid, $nid, $datetime, rand()); } $storage->store($cvs); echo "Insertion done\n"; } $timerStart = microtime(true);
case 'DAY': return Resolution::DAY; case 'HOUR': return Resolution::HOUR; case 'FIFTEEN_MINUTES': return Resolution::FIFTEEEN_MINUTES; case 'FIVE_MINUTES': return Resolution::FIVE_MINUTES; case 'ONE_MINUTE': return Resolution::MINUTE; } throw new Exception('Invalid resolution'); } $conn = new Connection('127.0.0.1', 'test', 'test'); $pipelineFactory = new Factory($conn); $pipeline = new Pipeline(); $asFormula = false; $nids = array(1, 2); $sids = array(1, 2); $start = "-60 days"; $end = "now"; $aggregation = Aggregation::SUM; $resolution = Resolution::HOUR; $combinedNodeAggregation = Aggregation::SUM; $singleNodeAggregation = Aggregation::SUM; $nodeResolution = Resolution::FIVE_MINUTES; $formulaResolution = Resolution::FIVE_MINUTES; if (isset($_GET['asFormula'])) { $asFormula = $_GET['asFormula'] === 'true'; } if ($asFormula) {
use Mongotd\Connection; use Mongotd\Pipeline\Factory; use Mongotd\Pipeline\Pipeline; use Mongotd\Storage; use Mongotd\StorageMiddleware\FilterCounterValues; use Mongotd\StorageMiddleware\InsertCounterValues; use Mongotd\StorageMiddleware\FindAnomaliesUsingSigmaTest; use Mongotd\StorageMiddleware\FindAnomaliesUsingHwTest; use Mongotd\StorageMiddleware\FindAnomaliesUsingKsTest; use Mongotd\StorageMiddleware\StoreAnomalies; $config = json_decode(file_get_contents('anomalyTestConfig.json'), true); $conn = new Connection($config['dbhost'], $config['dbname'], $config['dbprefix']); $conn->db()->drop(); $conn->createIndexes(); $pipelineFactory = new Factory($conn); $pipeline = new Pipeline(); $storage = new Storage(); $storage->addMiddleware(new FilterCounterValues()); $storage->addMiddleware(new InsertCounterValues($conn)); if ($config['anomalyDetectionMethod'] == 'ks') { $storage->addMiddleware(new FindAnomaliesUsingKsTest($conn)); } else { if ($config['anomalyDetectionMethod'] == 'hw') { $storage->addMiddleware(new FindAnomaliesUsingHwTest($conn)); } else { if ($config['anomalyDetectionMethod'] == 'sigma') { $storage->addMiddleware(new FindAnomaliesUsingSigmaTest($conn)); } else { throw new Exception('Unknown anomaly scan method'); } }