public function testSetAggregators()
{
$i = new IndexTaskQueryParameters();
$i->setAggregators(array(array('type' => 'count', 'name' => 'count'), array('type' => 'longSum', 'name' => 'total_referral_count', 'fieldName' => 'referral_count')));
$this->assertCount(2, $i->aggregators);
$this->assertJson($i->aggregators[0]);
$this->assertJson($i->aggregators[1]);
$this->assertEquals('{"type":"count","name":"count"}', $i->aggregators[0]);
$this->assertEquals('{"type":"longSum","name":"total_referral_count","fieldName":"referral_count"}', $i->aggregators[1]);
}
public function getMockIndexTaskQueryParameters()
{
$params = new IndexTaskQueryParameters();
$params->intervals = new Interval('1981-01-01T4:20', '2012-03-01T3:00');
$params->granularityType = 'uniform';
$params->granularity = 'DAY';
$params->dataSource = $this->mockDataSourceName;
$params->format = 'json';
$params->timeDimension = 'date_dim';
$params->dimensions = array('one_dim', 'two_dim');
$params->setIntervals('1981-01-01T4:20', '2012-03-01T3:00');
$params->setFilePath('/another/file/path/to/a/file.bebop');
$params->setAggregators(array(array('type' => 'count', 'name' => 'count'), array('type' => 'longSum', 'name' => 'total_referral_count', 'fieldName' => 'referral_count')));
return $params;
}
protected function ingest($formattedStartTime, $formattedEndTime, InputInterface $input, OutputInterface $output)
{
/** @var $logger LoggerInterface */
$logger = new ConsoleLogger($output);
$fetcher = new ReferralBatchFetcher();
$fetcher->setOutput($logger);
$fetcher->setMySqlCredentials($this->host, $this->user, $this->pass, $this->db);
$fetcher->setTimeWindow($formattedStartTime, $formattedEndTime);
$filePathToUse = '/tmp/referral_temp_' . time() . '.json';
$preparer = new LocalPhpArrayToJsonFilePreparer();
$preparer->setFilePath($filePathToUse);
$indexTaskQueryGenerator = new SimpleIndexQueryGenerator();
$indexTaskQueryParameters = new IndexTaskQueryParameters();
$indexTaskQueryParameters->dataSource = 'referral-report-referrals-cli-test';
$indexTaskQueryParameters->setIntervals($formattedStartTime, $formattedEndTime);
$indexTaskQueryParameters->dimensions = array('referral_id', 'facility_id', 'patient_id');
$indexTaskQueryParameters->timeDimension = 'date';
// TODO Fill in with dimensions, etc for referrals.
$indexTaskQueryParameters->validate();
$basicDruidJobWatcher = new CallbackBasedIndexingTaskDruidJobWatcher();
/**
* @param CallbackBasedIndexingTaskDruidJobWatcher $jobWatcher
*/
$myOnPendingCallback = function ($jobWatcher) use($output) {
$output->writeln('Druid says the job is still running. Trying again in ' . $jobWatcher->watchAttemptDelay . ' seconds...');
};
$basicDruidJobWatcher->setOnJobPending($myOnPendingCallback);
$basicDruidJobWatcher->setOutput($logger);
$basicDruidJobWatcher->setDruidIp($this->druidIp);
$basicDruidJobWatcher->setDruidPort($this->druidPort);
$druidQueryExecutor = new DruidNodeDruidQueryExecutor($this->druidIp, $this->druidPort, '/druid/indexer/v1/task');
try {
$output->writeln("Fetching referral data from source...");
$fetchedData = $fetcher->fetch();
////////////////////////////////
$output->writeln("Fetched " . count($fetchedData) . " referrals.");
if (count($fetchedData) === 0) {
$output->writeln("Fetched no records, so stopping here.");
return;
}
if ($output->getVerbosity() >= OutputInterface::VERBOSITY_VERY_VERBOSE) {
$exampleData = print_r($fetchedData[0], true);
$output->writeln("The first record looks like:\n\n" . $exampleData . "\n\n");
}
$pathOfPreparedData = $preparer->prepare($fetchedData);
$indexTaskQueryParameters->setFilePath($pathOfPreparedData);
$output->writeln('File is prepared for druid at "' . $pathOfPreparedData . '"');
$output->writeln('Requesting Druid index the source data into dataSource "' . $indexTaskQueryParameters->dataSource . '"');
$ingestionTaskId = $druidQueryExecutor->executeQuery($indexTaskQueryGenerator, $indexTaskQueryParameters, new IndexingTaskResponseHandler());
$output->writeln('Druid has received the job. Job id is "' . $ingestionTaskId . '"');
$output->writeln('Checking Druid for the job status:');
$success = $basicDruidJobWatcher->watchJob($ingestionTaskId);
if ($success) {
$output->writeln('Done checking druid job status.');
$output->writeln('Cleaning up prepared file "' . $pathOfPreparedData . '"...');
$cleanedUpSuccess = $preparer->cleanup($pathOfPreparedData);
if ($cleanedUpSuccess) {
$output->writeln('Succesfully cleaned up file.');
}
} else {
$output->writeln('Job failed or did not finish in time.');
$output->writeln('In the future I will ask you if you want to delete the temporary file but for now I will leave it in place so that ingestion is not interrupted.');
}
$output->writeln('Done.');
////////////////////////////////
} catch (\Exception $e) {
throw $e;
}
}
