public function testSetAggregators() { $i = new IndexTaskQueryParameters(); $i->setAggregators(array(array('type' => 'count', 'name' => 'count'), array('type' => 'longSum', 'name' => 'total_referral_count', 'fieldName' => 'referral_count'))); $this->assertCount(2, $i->aggregators); $this->assertJson($i->aggregators[0]); $this->assertJson($i->aggregators[1]); $this->assertEquals('{"type":"count","name":"count"}', $i->aggregators[0]); $this->assertEquals('{"type":"longSum","name":"total_referral_count","fieldName":"referral_count"}', $i->aggregators[1]); }
public function getMockIndexTaskQueryParameters() { $params = new IndexTaskQueryParameters(); $params->intervals = new Interval('1981-01-01T4:20', '2012-03-01T3:00'); $params->granularityType = 'uniform'; $params->granularity = 'DAY'; $params->dataSource = $this->mockDataSourceName; $params->format = 'json'; $params->timeDimension = 'date_dim'; $params->dimensions = array('one_dim', 'two_dim'); $params->setIntervals('1981-01-01T4:20', '2012-03-01T3:00'); $params->setFilePath('/another/file/path/to/a/file.bebop'); $params->setAggregators(array(array('type' => 'count', 'name' => 'count'), array('type' => 'longSum', 'name' => 'total_referral_count', 'fieldName' => 'referral_count'))); return $params; }
protected function ingest($formattedStartTime, $formattedEndTime, InputInterface $input, OutputInterface $output) { /** @var $logger LoggerInterface */ $logger = new ConsoleLogger($output); $fetcher = new ReferralBatchFetcher(); $fetcher->setOutput($logger); $fetcher->setMySqlCredentials($this->host, $this->user, $this->pass, $this->db); $fetcher->setTimeWindow($formattedStartTime, $formattedEndTime); $filePathToUse = '/tmp/referral_temp_' . time() . '.json'; $preparer = new LocalPhpArrayToJsonFilePreparer(); $preparer->setFilePath($filePathToUse); $indexTaskQueryGenerator = new SimpleIndexQueryGenerator(); $indexTaskQueryParameters = new IndexTaskQueryParameters(); $indexTaskQueryParameters->dataSource = 'referral-report-referrals-cli-test'; $indexTaskQueryParameters->setIntervals($formattedStartTime, $formattedEndTime); $indexTaskQueryParameters->dimensions = array('referral_id', 'facility_id', 'patient_id'); $indexTaskQueryParameters->timeDimension = 'date'; // TODO Fill in with dimensions, etc for referrals. $indexTaskQueryParameters->validate(); $basicDruidJobWatcher = new CallbackBasedIndexingTaskDruidJobWatcher(); /** * @param CallbackBasedIndexingTaskDruidJobWatcher $jobWatcher */ $myOnPendingCallback = function ($jobWatcher) use($output) { $output->writeln('Druid says the job is still running. Trying again in ' . $jobWatcher->watchAttemptDelay . ' seconds...'); }; $basicDruidJobWatcher->setOnJobPending($myOnPendingCallback); $basicDruidJobWatcher->setOutput($logger); $basicDruidJobWatcher->setDruidIp($this->druidIp); $basicDruidJobWatcher->setDruidPort($this->druidPort); $druidQueryExecutor = new DruidNodeDruidQueryExecutor($this->druidIp, $this->druidPort, '/druid/indexer/v1/task'); try { $output->writeln("Fetching referral data from source..."); $fetchedData = $fetcher->fetch(); //////////////////////////////// $output->writeln("Fetched " . count($fetchedData) . " referrals."); if (count($fetchedData) === 0) { $output->writeln("Fetched no records, so stopping here."); return; } if ($output->getVerbosity() >= OutputInterface::VERBOSITY_VERY_VERBOSE) { $exampleData = print_r($fetchedData[0], true); $output->writeln("The first record looks like:\n\n" . $exampleData . "\n\n"); } $pathOfPreparedData = $preparer->prepare($fetchedData); $indexTaskQueryParameters->setFilePath($pathOfPreparedData); $output->writeln('File is prepared for druid at "' . $pathOfPreparedData . '"'); $output->writeln('Requesting Druid index the source data into dataSource "' . $indexTaskQueryParameters->dataSource . '"'); $ingestionTaskId = $druidQueryExecutor->executeQuery($indexTaskQueryGenerator, $indexTaskQueryParameters, new IndexingTaskResponseHandler()); $output->writeln('Druid has received the job. Job id is "' . $ingestionTaskId . '"'); $output->writeln('Checking Druid for the job status:'); $success = $basicDruidJobWatcher->watchJob($ingestionTaskId); if ($success) { $output->writeln('Done checking druid job status.'); $output->writeln('Cleaning up prepared file "' . $pathOfPreparedData . '"...'); $cleanedUpSuccess = $preparer->cleanup($pathOfPreparedData); if ($cleanedUpSuccess) { $output->writeln('Succesfully cleaned up file.'); } } else { $output->writeln('Job failed or did not finish in time.'); $output->writeln('In the future I will ask you if you want to delete the temporary file but for now I will leave it in place so that ingestion is not interrupted.'); } $output->writeln('Done.'); //////////////////////////////// } catch (\Exception $e) { throw $e; } }