/**
* @param $url
* @param array $arguments
* @param string $uniqueId
*
* @throws \Exception
*
* @return string
*/
public function rasterizeUrl($url, $arguments = array(), $uniqueId = "")
{
if ($uniqueId === "") {
$uniqueId = uniqid("rasterize-");
}
if ($this->stopwatch instanceof Stopwatch) {
if ($this->stopwatch->isStarted($uniqueId)) {
$this->stopwatch->lap($uniqueId);
} else {
$this->stopwatch->start($uniqueId);
}
}
$process = $this->configHelper->buildProcess($url, $uniqueId, $arguments);
$exitCode = $process->run();
if ($exitCode != 0) {
throw new \Exception(sprintf("Rasterize script failed.\nCommandLine: %s\nExitCode: %d\nErrorOutput: %s", $process->getCommandLine(), $process->getExitCode(), $process->getErrorOutput()));
}
if ($this->stopwatch instanceof Stopwatch) {
$this->stopwatch->stop($uniqueId);
}
$output = $this->configHelper->getOutputFilePath($uniqueId);
$content = file_get_contents($output);
unlink($output);
return $content;
}
public function testLap()
{
$stopwatch = new Stopwatch();
$stopwatch->start('foo', 'cat');
usleep(100000);
$event = $stopwatch->lap('foo');
usleep(100000);
$stopwatch->stop('foo');
$this->assertInstanceof('Symfony\\Component\\Stopwatch\\StopwatchEvent', $event);
$this->assertEquals(200, $event->getDuration(), null, self::DELTA);
}
/**
* Collects the time and duration of processing a file, logs it and returns the new amount of memory in use.
*
* @param integer $memory
*
* @return integer
*/
protected function logAfterParsingAFile($memory)
{
if (!$this->stopwatch) {
return $memory;
}
$lap = $this->stopwatch->lap('parser.parse');
$oldMemory = $memory;
$periods = $lap->getPeriods();
$memory = end($periods)->getMemory();
$this->log('>> Memory after processing of file: ' . number_format($memory / 1024 / 1024, 2) . ' megabytes (' . ($memory - $oldMemory >= 0 ? '+' : '-') . number_format(($memory - $oldMemory) / 1024) . ' kilobytes)', LogLevel::DEBUG);
return $memory;
}
/**
* @return Result
*/
public function run()
{
$stopwatch = new Stopwatch();
$result = '';
$stopwatch->start('parsing');
$this->driver->initialize();
$stopwatch->lap('parsing');
for ($i = 0; $i < $this->loopCount; $i++) {
$result = $this->driver->run($this->content);
}
$event = $stopwatch->stop('parsing');
return new Result($this->driver->getName(), $this->driver->getDialect(), $event, $result, memory_get_peak_usage(true));
}
public function run($collectionSize)
{
$testData = $this->prepareTestData($collectionSize);
/** @var Result[] $results */
$results = array();
foreach ($this->tasks as $task) {
if (!$task->isValid()) {
continue;
}
$stopwatch = new Stopwatch();
$stopwatch->start('mapping');
$task->prepare();
$stopwatch->lap('mapping');
$task->run($testData);
$event = $stopwatch->stop('mapping');
$results[] = new Result($task->getName(), $event, memory_get_peak_usage(true));
}
return $results;
}
/**
* Stopwatch's example
*
* An example show you how to use Symfony Stopwatch
*
* @author nguyenvanduocit
*/
require_once 'vendor/autoload.php';
require_once 'functions.php';
use Symfony\Component\Stopwatch\Stopwatch;
$stopwatch = new Stopwatch();
$stopwatch->start('example_lap');
$totalLap = 10;
for ($count = 0; $count < $totalLap; $count++) {
doSomeFunction();
$stopwatch->lap('example_lap');
}
$event = $stopwatch->stop('example_lap');
$periods = $event->getPeriods();
?>
<ul>
<?php
foreach ($periods as $index => $period) {
echo '<li>#' . $index . ':' . $period->getDuration() . 'ms</li>';
}
?>
<li>Total time: <?php
echo $event->getDuration();
?>
ms</li>
<li>The fastest runner's time: <?php
protected function execute(InputInterface $input, OutputInterface $output)
{
$em = $this->getContainer()->get('doctrine')->getManager();
$logger = $this->getContainer()->get('logger');
$year = date("Y") - 1;
$interval = 20;
$stopwatch = new Stopwatch();
$stopwatch->start('syncing');
$id = $input->getArgument('id');
$capturedYear = $input->getArgument('year');
$orgunitid = $input->getArgument('orgunitid');
$entity = $em->getRepository('HrisIntergrationBundle:DHISDataConnection')->find($id);
if (!empty($orgunitid)) {
$orgunit = $em->getRepository('HrisOrganisationunitBundle:Organisationunit')->find($orgunitid);
} else {
$orgunit = $entity->getParentOrganisationunit();
}
if (!empty($capturedYear)) {
$year = $capturedYear;
}
if ($id) {
$this->dhisConnectionId = $id;
} else {
throw new NotFoundHttpException("Data connection not found!");
}
$this->xmlContents = "<?xml version='1.0' encoding='UTF-8'?>\n<dataValueSet xmlns=\"http://dhis2.org/schema/dxf/2.0\">";
$xmlFile = "/tmp/hrhis_data_" . str_replace(' ', '_', $orgunit) . ".xml";
file_put_contents($xmlFile, $this->xmlContents);
// Aggregate data for each orgunit in the current level.
$em = $this->getContainer()->get('doctrine')->getManager();
$entity = $em->getRepository('HrisIntergrationBundle:DHISDataConnection')->find($this->dhisConnectionId);
/*
* Initializing query for dhis dataset calculation
*/
// Get Standard Resource table name
$resourceTableName = str_replace(' ', '_', trim(strtolower(ResourceTable::getStandardResourceTableName())));
$resourceTableAlias = "ResourceTable";
/*
* Aggregate organisationunit for all the children
*/
$queryBuilder = $em->createQueryBuilder();
$allOrganisationunitsChildren = $queryBuilder->select('DISTINCT organisationunit')->from('HrisOrganisationunitBundle:Organisationunit', 'organisationunit')->join('organisationunit.organisationunitStructure', 'organisationunitStructure')->join('organisationunitStructure.level', 'level')->andWhere('
(
level.level >= :organisationunitLevel
AND organisationunitStructure.level' . $orgunit->getOrganisationunitStructure()->getLevel()->getLevel() . 'Organisationunit=:levelOrganisationunit
) AND organisationunit.dhisUid is not null
AND organisationunit.id IN ( SELECT DISTINCT(recordOrganisationunit.id) FROM HrisRecordsBundle:Record record INNER JOIN record.organisationunit recordOrganisationunit )')->setParameters(array('levelOrganisationunit' => $orgunit, 'organisationunitLevel' => $orgunit->getOrganisationunitStructure()->getLevel()->getLevel()))->getQuery()->getResult();
// Query for Options to exclude from reports
$fieldOptionsToSkip = $this->getContainer()->get('doctrine')->getManager()->getRepository('HrisFormBundle:FieldOption')->findBy(array('skipInReport' => True));
//filter the records with exclude report tag
foreach ($fieldOptionsToSkip as $key => $fieldOptionToSkip) {
if (empty($fieldOptionsToSkipQuery)) {
$fieldOptionsToSkipQuery = "{$resourceTableAlias}." . $fieldOptionToSkip->getField()->getName() . " !='" . $fieldOptionToSkip->getValue() . "'";
} else {
$fieldOptionsToSkipQuery .= " AND {$resourceTableAlias}." . $fieldOptionToSkip->getField()->getName() . " !='" . $fieldOptionToSkip->getValue() . "'";
}
}
$dataValueColumnName = NULL;
$selectQuery = NULL;
$organisationunitNames = NULL;
$incr = 0;
$totalIncr = 0;
$lastLap = $stopwatch->lap('syncing');
$lastLapDuration = round($lastLap->getDuration() / 1000, 2);
$previousTotalLapTime = round($lastLap->getDuration() / 1000, 2);
foreach ($allOrganisationunitsChildren as $organisationunitKey => $organisationunit) {
$organisationunitNames .= $organisationunit->getLongname() . ', ';
$incr++;
// Aggregate data for each orgunit in the current level.
$em = $this->getContainer()->get('doctrine')->getManager();
$entity = $em->getRepository('HrisIntergrationBundle:DHISDataConnection')->find($this->dhisConnectionId);
$fromClause = " FROM {$resourceTableName} {$resourceTableAlias} ";
$organisationunitLevelsWhereClause = " {$resourceTableAlias}.organisationunit_id=" . $organisationunit->getId() . " ";
// Dataelement field option relation
$dataelementFieldOptionRelation = $entity->getDataelementFieldOptionRelation();
foreach ($dataelementFieldOptionRelation as $dataelementFieldOptionKey => $dataelementFieldOptionValue) {
// Formulate Query for calculating field option
$columnFieldOptionGroup = $dataelementFieldOptionValue->getColumnFieldOptionGroup();
$rowFieldOptionGroup = $dataelementFieldOptionValue->getRowFieldOptionGroup();
$seriesFieldName = $rowFieldOptionGroup->getName();
//Column Query construction
$queryColumnNames[] = str_replace('-', '_', str_replace(' ', '', $columnFieldOptionGroup->getName()));
$categoryColumnFieldNames[] = $columnFieldOptionGroup->getField()->getName();
$categoryRowFieldName = $columnFieldOptionGroup->getField()->getName();
$columnWhereClause = NULL;
foreach ($columnFieldOptionGroup->getFieldOption() as $columnFieldOptionKey => $columnFieldOption) {
$operator = $columnFieldOptionGroup->getOperator();
if (empty($operator)) {
$operator = "OR";
}
$categoryColumnFieldOptionValue = str_replace('-', '_', $columnFieldOption->getValue());
$categoryColumnFieldName = $columnFieldOption->getField()->getName();
$categoryColumnResourceTableName = $resourceTableAlias;
if (!empty($columnWhereClause)) {
$columnWhereClause = $columnWhereClause . " " . strtoupper($operator) . " {$categoryColumnResourceTableName}.{$categoryColumnFieldName}='" . $categoryColumnFieldOptionValue . "'";
} else {
$columnWhereClause = "{$categoryColumnResourceTableName}.{$categoryColumnFieldName}='" . $categoryColumnFieldOptionValue . "'";
}
}
// Row Query construction
$rowWhereClause = NULL;
foreach ($rowFieldOptionGroup->getFieldOption() as $rowFieldOptionKey => $rowFieldOption) {
$operator = $rowFieldOptionGroup->getOperator();
if (empty($operator)) {
$operator = "OR";
}
$categoryRowFieldOptionValue = str_replace('-', '_', $rowFieldOption->getValue());
$categoryRowFieldName = $rowFieldOption->getField()->getName();
$categoryRowResourceTableName = $resourceTableAlias;
if (!empty($rowWhereClause)) {
$rowWhereClause = $rowWhereClause . " " . strtoupper($operator) . " {$categoryRowResourceTableName}.{$categoryRowFieldName}='" . $categoryRowFieldOptionValue . "'";
} else {
$rowWhereClause = "{$categoryRowResourceTableName}.{$categoryRowFieldName}='" . $categoryRowFieldOptionValue . "'";
}
}
$verboseDataValaueColumnNames = $dataelementFieldOptionValue->getRowFieldOptionGroup() . "--" . $dataelementFieldOptionValue->getColumnFieldOptionGroup();
$dataValueColumnName = $organisationunit->getDhisUid() . '--' . $dataelementFieldOptionValue->getDataelementUid() . '--' . $dataelementFieldOptionValue->getCategoryComboUid();
if (!empty($selectQuery)) {
$selectQuery .= " UNION ALL SELECT CAST('" . $dataValueColumnName . "' AS text) AS datavaluelabel, COUNT(DISTINCT(instance)) AS value " . " {$fromClause} WHERE ({$rowWhereClause}) AND ({$columnWhereClause}) AND " . $resourceTableAlias . ".first_appointment_year<=" . $year . " AND {$organisationunitLevelsWhereClause}" . (!empty($fieldOptionsToSkipQuery) ? " AND ( {$fieldOptionsToSkipQuery} )" : "") . " HAVING COUNT(DISTINCT(instance))>0";
} else {
$selectQuery = "SELECT CAST('" . $dataValueColumnName . "' AS text) AS datavaluelabel, COUNT(DISTINCT(instance)) AS value " . " {$fromClause} WHERE ({$rowWhereClause}) AND ({$columnWhereClause}) AND " . $resourceTableAlias . ".first_appointment_year<=" . $year . " AND {$organisationunitLevelsWhereClause}" . (!empty($fieldOptionsToSkipQuery) ? " AND ( {$fieldOptionsToSkipQuery} )" : "") . " HAVING COUNT(DISTINCT(instance))>0";
}
unset($dhisUid);
// Intercept after certain number of orgunits for fetching results
// So as not to exceed database max_stack_depth
if ($incr >= $interval) {
// Reset counter
$totalIncr = $incr + $totalIncr;
$incr = 0;
// Process SQL Batch
$sqlResult = $this->getContainer()->get('doctrine')->getManager()->getConnection()->fetchAll($selectQuery);
foreach ($sqlResult as $resultKey => $resultRow) {
$dataValueKeys = explode('--', $resultRow['datavaluelabel']);
//dhisUid--dataelementUid--categoryComboUid$recordRow ='<!--'.$resultRow['datavaluelabelverbose'].'-->';
$recordRow = '<dataValue orgUnit="' . $dataValueKeys[0] . '" period="' . $year . '" dataElement="' . $dataValueKeys[1] . '" categoryOptionCombo="' . $dataValueKeys[2] . '" value="' . $resultRow['value'] . '" storedBy="hrhis" lastUpdated="' . date("c") . '" followUp="false" />';
file_put_contents($xmlFile, $recordRow, FILE_APPEND);
}
$currentLap = $stopwatch->lap('syncing');
$currentLapDuration = NULL;
$currentLapDuration = round($currentLap->getDuration() / 1000, 2) - $previousTotalLapTime;
if ($currentLapDuration < 60) {
$durationMessage = round($currentLapDuration, 2) . ' seconds';
} elseif ($currentLapDuration >= 60 && $currentLapDuration < 3600) {
$durationMessage = round($currentLapDuration / 60, 2) . ' minutes';
} elseif ($currentLapDuration >= 3600 && $currentLapDuration < 216000) {
$durationMessage = round($currentLapDuration / 3600, 2) . ' hours';
} else {
$durationMessage = round($currentLapDuration / 86400, 2) . ' hours';
}
$lastLapDuration = NULL;
$lastLapDuration = $currentLapDuration;
$previousTotalLapTime = round($currentLap->getDuration() / 1000, 2);
$output->writeln('Fetched records for ' . $totalIncr . ' out of ' . count($allOrganisationunitsChildren) . ' organisationunits in ' . $durationMessage . " " . count($sqlResult) . " results found");
//$output->writeln('Organisationunits parsed:'."\n".$organisationunitNames."\n");
$selectQuery = NULL;
$organisationunitNames = NULL;
}
}
}
// Process last remaining SQL Batch
$sqlResult = $this->getContainer()->get('doctrine')->getManager()->getConnection()->fetchAll($selectQuery);
foreach ($sqlResult as $resultKey => $resultRow) {
$dataValueKeys = explode('--', $resultRow['datavaluelabel']);
//dhisUid--dataelementUid--categoryComboUid
$recordRow = '<dataValue orgUnit="' . $dataValueKeys[0] . '" period="' . $year . '" dataElement="' . $dataValueKeys[1] . '" categoryOptionCombo="' . $dataValueKeys[2] . '" value="' . $resultRow['value'] . '" storedBy="hrhis" lastUpdated="' . date("c") . '" followUp="false" />';
file_put_contents($xmlFile, $recordRow, FILE_APPEND);
}
$currentLap = $stopwatch->lap('syncing');
$currentLapDuration = NULL;
$currentLapDuration = round($currentLap->getDuration() / 1000, 2) - $previousTotalLapTime;
if ($currentLapDuration < 60) {
$durationMessage = round($currentLapDuration, 2) . ' seconds';
} elseif ($currentLapDuration >= 60 && $currentLapDuration < 3600) {
$durationMessage = round($currentLapDuration / 60, 2) . ' minutes';
} elseif ($currentLapDuration >= 3600 && $currentLapDuration < 216000) {
$durationMessage = round($currentLapDuration / 3600, 2) . ' hours';
} else {
$durationMessage = round($currentLapDuration / 86400, 2) . ' hours';
}
$lastLapDuration = NULL;
$lastLapDuration = $currentLapDuration;
$previousTotalLapTime = round($currentLap->getDuration() / 1000, 2);
$output->writeln('Fetched records for ' . $interval . ' out of ' . count($allOrganisationunitsChildren) . ' organisationunits in ' . $durationMessage . " " . count($sqlResult) . " results found ");
//$output->writeln('Organisationunits parsed:'."\n".$organisationunitNames."\n");
$selectQuery = NULL;
$organisationunitNames = NULL;
$this->xmlContents = $this->xmlContents . '</dataValueSet>';
file_put_contents($xmlFile, '</dataValueSet>', FILE_APPEND);
/*
* Check Clock for time spent
*/
$totalTime = $stopwatch->stop('syncing');
$duration = $totalTime->getDuration() / 1000;
unset($stopwatch);
if ($duration < 60) {
$durationMessage = round($duration, 2) . ' seconds';
} elseif ($duration >= 60 && $duration < 3600) {
$durationMessage = round($duration / 60, 2) . ' minutes';
} elseif ($duration >= 3600 && $duration < 216000) {
$durationMessage = round($duration / 3600, 2) . ' hours';
} else {
$durationMessage = round($duration / 86400, 2) . ' hours';
}
$output->writeln("Data Syncing completed in " . $durationMessage . ".\n\n");
$this->messageLog = "Sync aggregation for " . $orgunit->getLongname() . " operation is complete";
$output->writeln($this->messageLog);
}
/**
* @ApiDoc(
* description="Check Public Api Status",
* statusCodes={200="Success"},
* section="Public Api"
* )
* @Route("/public-api/performance")
* @Template{}
* @Method({"GET"})
* @return mixed
*/
public function performanceAction()
{
$stopwatch = new Stopwatch();
$stopwatch->start('main');
$address = new Address();
$address->setCity('London');
$address->setCountry('GB');
$address->setLat('51.5286416');
$address->setLng('-0.1015987');
$geoname = $this->geonameRepository->getOneByAddress($address);
$stopwatch->lap('main');
$cuisines = $this->cuisineRepository->findAll();
$event = $stopwatch->stop('main');
$duration = $event->getDuration();
$endTime = $event->getEndTime();
$memory = $event->getMemory();
$periodsData = array();
foreach ($event->getPeriods() as $period) {
$periodsData[] = array('duration' => $period->getDuration(), 'memory' => $period->getMemory());
}
return $this->view(array('duration' => $duration, 'endTime' => $endTime, 'memory' => $memory, 'periods' => $periodsData, 'geoname' => $geoname, 'cuisines' => $cuisines));
}
/**
* @param string $name
*/
public function lap($name)
{
if ($this->stopwatch) {
$this->stopwatch->lap($name);
}
}
/**
* @param $batchElementName
* @param $eventName
* @return int
*/
public function getDuration($batchElementName, $eventName)
{
$event = $this->timer->lap($batchElementName . ":" . $eventName);
return $event->getDuration();
}
<?php
require_once 'vendor/autoload.php';
require_once 'functions.php';
use Symfony\Component\Stopwatch\Stopwatch;
$stopwatch = new Stopwatch();
$stopwatch->openSection();
$stopwatch->start('do_phase_1');
doSomeFunction();
$stopwatch->stopSection('step1');
$stopwatch->openSection();
$stopwatch->start('do_phase_1');
$totalLap = 10;
for ($count = 0; $count < $totalLap; $count++) {
doSomeFunction();
$stopwatch->lap('do_phase_1');
}
$stopwatch->stopSection('step2');
echo '<p>Step 1 :</p>';
$events_1 = $stopwatch->getSectionEvents('step1');
echo '<ul>';
foreach ($events_1 as $id => $event) {
echo '<li> phase ' . $id . ':' . $event->getDuration() . '</li>';
}
echo '</ul>';
echo '<p>Step 2 :</p>';
$events_2 = $stopwatch->getSectionEvents('step2');
echo '<ul>';
foreach ($events_2 as $id => $event) {
echo '<li> phase ' . $id . ':' . $event->getDuration() . '</li>';
}
public function load(ObjectManager $manager)
{
$logger = $this->container->get('logger');
$stopwatch = new Stopwatch();
$stopwatch->start('dummyResourceTableGeneration');
// Populate dummy forms
$this->addDummyResourceTables();
// Seek dummy fields
$loadFieldData = new LoadFieldData();
$loadFieldData->addDummyFields();
$dummyFields = $loadFieldData->getFields();
foreach ($this->resourceTables as $resourceTableKey => $humanResourceResourceTable) {
$resourceTable = new ResourceTable();
$resourceTable->setName($humanResourceResourceTable['name']);
$resourceTable->setDescription($humanResourceResourceTable['description']);
$resourceTableRefernce = strtolower(str_replace(' ', '', $humanResourceResourceTable['name'])) . '-resourcetable';
$this->addReference($resourceTableRefernce, $resourceTable);
$manager->persist($resourceTable);
// Add Field Members for the resource table created
$sort = 1;
foreach ($dummyFields as $key => $dummyField) {
//Filter addition of fields not compliant to filter
if ($humanResourceResourceTable['filter'] == false || $humanResourceResourceTable['inputType'] == $dummyField['inputType'] || $humanResourceResourceTable['compulsory'] == $dummyField['compulsory']) {
$resourceTableMember = new ResourceTableFieldMember();
$resourceTableMember->setField($manager->merge($this->getReference(strtolower(str_replace(' ', '', $dummyField['name'])) . '-field')));
$resourceTableMember->setResourceTable($manager->merge($this->getReference($resourceTableRefernce)));
$resourceTableMember->setSort($sort++);
$referenceName = strtolower(str_replace(' ', '', $humanResourceResourceTable['name']) . str_replace(' ', '', $dummyField['name'])) . '-resourcetable-field-member';
$this->addReference($referenceName, $resourceTableMember);
$manager->persist($resourceTableMember);
$resourceTable->addResourceTableFieldMember($resourceTableMember);
unset($resourceTableMember);
}
}
$manager->persist($resourceTable);
unset($resourceTable);
}
$manager->flush();
$dummyResourceTableGenerationLap = $stopwatch->lap('dummyResourceTableGeneration');
$dummyResourceTableGenerationDuration = round($dummyResourceTableGenerationLap->getDuration() / 1000, 2);
if ($dummyResourceTableGenerationDuration < 60) {
$dummyResourceTableGenerationDurationMessage = round($dummyResourceTableGenerationDuration, 2) . ' sec.';
} elseif ($dummyResourceTableGenerationDuration >= 60 && $dummyResourceTableGenerationDuration < 3600) {
$dummyResourceTableGenerationDurationMessage = round($dummyResourceTableGenerationDuration / 60, 2) . ' min.';
} elseif ($dummyResourceTableGenerationDuration >= 3600 && $dummyResourceTableGenerationDuration < 216000) {
$dummyResourceTableGenerationDurationMessage = round($dummyResourceTableGenerationDuration / 3600, 2) . ' hrs';
} else {
$dummyResourceTableGenerationDurationMessage = round($dummyResourceTableGenerationDuration / 86400, 2) . ' days';
}
echo "\tDummy data schema generation complete in " . $dummyResourceTableGenerationDurationMessage . "\n";
// Generate resource tables
$resourceTables = $manager->getRepository('HrisFormBundle:ResourceTable')->findAll();
foreach ($resourceTables as $resourceTableKey => $resourceTable) {
// Ugly hack to generate resource table for "All Fields" only
if ($resourceTable->getName() == "All Fields") {
$success = $resourceTable->generateResourceTable($manager, $logger);
$messageLog = $resourceTable->getMessageLog();
if ($success) {
echo $messageLog;
} else {
echo "Failed with:" . $messageLog;
}
}
}
}
/**
* Generate resource table
*
* @param $entityManager
* @return string
*/
public function generateResourceTable($entityManager, $logger = NULL)
{
$totalInsertedRecords = NULL;
$totalResourceTableFields = NULL;
$stopwatch = new Stopwatch();
$stopwatch->start('resourceTableGeneration');
$returnMessage = NULL;
$schemaManager = $entityManager->getConnection()->getSchemaManager();
$resourceTableName = '_resource_' . str_replace(' ', '_', trim(strtolower($this->getName())));
//Prepare database name
if ($this->getIsgenerating() == False && ($this->isResourceTableOutdated($entityManager) == True || $this->isResourceTableCompletelyGenerated($entityManager) == False)) {
$logger->info('Resource table is out dated, was not completely generated');
/*
* Resource table is out dated, was not completely generated
*/
//Switch state to generating
$this->setIsgenerating(True);
$entityManager->persist($this);
try {
$entityManager->flush();
} catch (\Doctrine\Orm\NoResultException $e) {
//@todo Cross-check, if database is left in generating state, it'll never be udpated ever.
$error = "Error in Changing State to generating";
echo $error;
return False;
}
if ($this->isResourceTableOutdated($entityManager)) {
$this->messagelog = "Regeneration Trigger: Outdated Resource Table.\n";
} elseif ($this->isResourceTableCompletelyGenerated($entityManager) == False) {
$this->messagelog = "Regeneration Trigger:Incomplete Resource Tabe.\n";
} else {
$this->messagelog = '';
}
/**
* @var $resourceTable String
*/
// Cleanup any residue temporary resourcetable left
if ($schemaManager->tablesExist($resourceTableName . '_temporary')) {
$schemaManager->dropTable($resourceTableName . '_temporary');
}
$resourceTable = new Table($resourceTableName . '_temporary');
//Create database table
// Create primary key
$resourceTable->addColumn('id', "integer", array('nullable' => true, 'precision' => 0, 'scale' => 0));
$resourceTable->addColumn('instance', "string", array('length' => 64, 'notnull' => false));
$resourceTable->setPrimaryKey(array('id'), 'IDX_' . uniqid(''));
$resourceTable->addIndex(array('id'), 'IDX_' . uniqid(''));
// Create other columns(fields, organisationunits,etc) in the resource table
foreach ($this->getResourceTableFieldMember() as $resourceTableKey => $resourceTableFieldMember) {
$field = $resourceTableFieldMember->getField();
if ($field->getDataType()->getName() == "String") {
$resourceTable->addColumn($field->getName(), "string", array('length' => 64, 'notnull' => false));
} elseif ($field->getDataType()->getName() == "Integer") {
$resourceTable->addColumn($field->getName(), "integer", array('notnull' => false, 'precision' => 0, 'scale' => 0));
} elseif ($field->getDataType()->getName() == "Double") {
$resourceTable->addColumn($field->getName(), "float", array('notnull' => false, 'precision' => 0, 'scale' => 0));
} elseif ($field->getDataType()->getName() == "Date") {
$resourceTable->addColumn($field->getName(), "date", array('notnull' => false, 'precision' => 0, 'scale' => 0));
// Additional analysis columns
//$resourceTable->addColumn($field->getName().'_day', "string",array('length'=>64, 'notnull'=>false));
//$resourceTable->addColumn($field->getName().'_month_number', "integer",array('notnull'=>false,'precision'=>0, 'scale'=>0));
$resourceTable->addColumn($field->getName() . '_month_text', "string", array('length' => 64, 'notnull' => false));
$resourceTable->addColumn($field->getName() . '_year', "integer", array('notnull' => false, 'precision' => 0, 'scale' => 0));
//$resourceTable->addColumn($field->getName().'_month_and_year', "string",array('length'=>64, 'notnull'=>false));
}
// @todo implement after creation of history date class
// Add History date field for fields with history
if ($field->getHashistory()) {
$resourceTable->addColumn($field->getName() . '_last_updated', "date", array('notnull' => false, 'precision' => 0, 'scale' => 0));
// Additional analysis columns
//$resourceTable->addColumn($field->getName().'_last_updated_day', "string",array('length'=>64, 'notnull'=>false));
//$resourceTable->addColumn($field->getName().'_last_updated_month_number', "integer",array('notnull'=>false,'precision'=>0, 'scale'=>0));
$resourceTable->addColumn($field->getName() . '_last_updated_month_text', "string", array('length' => 64, 'notnull' => false));
$resourceTable->addColumn($field->getName() . '_last_updated_year', "integer", array('notnull' => false, 'precision' => 0, 'scale' => 0));
//$resourceTable->addColumn($field->getName().'_last_updated_month_and_year', "string",array('length'=>64, 'notnull'=>false));
}
$totalResourceTableFields++;
unset($field);
}
// Make OrganisationunitLevels of orgunit
$organisationunitLevels = $entityManager->createQuery('SELECT DISTINCT organisationunitLevel FROM HrisOrganisationunitBundle:OrganisationunitLevel organisationunitLevel ORDER BY organisationunitLevel.level ')->getResult();
foreach ($organisationunitLevels as $organisationunitLevelKey => $organisationunitLevel) {
$organisationunitLevelName = "level" . $organisationunitLevel->getLevel() . "_" . str_replace(',', '_', str_replace('.', '_', str_replace('/', '_', str_replace(' ', '_', $organisationunitLevel->getName()))));
$resourceTable->addColumn($organisationunitLevelName, "string", array('length' => 64, 'notnull' => false));
}
// Make OrganisationunitGroupsets Column
$organisationunitGroupsets = $entityManager->getRepository('HrisOrganisationunitBundle:OrganisationunitGroupset')->findAll();
foreach ($organisationunitGroupsets as $organisationunitGroupsetKey => $organisationunitGroupset) {
$resourceTable->addColumn($organisationunitGroupset->getName(), "string", array('length' => 64, 'notnull' => false));
}
// Form and Organisationunit name
$resourceTable->addColumn("Organisationunit_name", "string", array('length' => 64, 'notnull' => false));
$resourceTable->addColumn("Form_name", "string", array('length' => 64, 'notnull' => false));
$resourceTable->addColumn('Organisationunit_id', "integer", array('notnull' => false, 'precision' => 0, 'scale' => 0));
$resourceTable->addColumn('Form_id', "integer", array('notnull' => false, 'precision' => 0, 'scale' => 0));
$resourceTable->addColumn("Lastupdated", "datetime", array('notnull' => false, 'precision' => 0, 'scale' => 0));
// Creating table
$schemaManager->createTable($resourceTable);
unset($resourceTable);
$schemaGenerationLap = $stopwatch->lap('resourceTableGeneration');
$schemaGenerationDuration = round($schemaGenerationLap->getDuration() / 1000, 2);
$this->messagelog .= 'Operation: Table named ' . $resourceTableName . ' with ' . $totalResourceTableFields . " Fields Generated in " . $schemaGenerationDuration . " seconds.\n";
// Populating data into created table
$queryBuilder = $entityManager->createQueryBuilder()->select('record')->from('HrisRecordsBundle:Record', 'record')->join('record.organisationunit', 'organisationunit')->join('record.form', 'form')->join('organisationunit.organisationunitStructure', 'organisationunitStructure')->getQuery();
try {
$records = $queryBuilder->getResult();
} catch (\Doctrine\Orm\NoResultException $e) {
echo 'Error in returning Data Values';
}
if (!empty($records)) {
/**
* Make sure organisationunitstructure is uptodate for good measure
*/
// Check and Notify if organisationunit structure doesn't exist
$queryBuilder = $entityManager->createQueryBuilder();
$organisationunitStructureCount = $queryBuilder->select('count( organisationunitStructure.id )')->from('HrisOrganisationunitBundle:OrganisationunitStructure', 'organisationunitStructure')->getQuery()->getSingleScalarResult();
$queryBuilder = $entityManager->createQueryBuilder();
$organisationunitCount = $queryBuilder->select('count( organisationunit.id )')->from('HrisOrganisationunitBundle:Organisationunit', 'organisationunit')->getQuery()->getSingleScalarResult();
// Regenerate Orgunit Stucture of Orgunit and OrgunitStructure Differs
if ($organisationunitCount != $organisationunitStructureCount) {
$logger->info('Regenerating organisationunit structure');
$this->returnMessage = '';
// Regenerate Orgunit Structure
$organisationunitStructure = new OrganisationunitStructureController();
$this->returnMessage = $organisationunitStructure->regenerateOrganisationunitStructure($entityManager);
} else {
$this->returnMessage = 'Organisationunit structure is complete!';
}
// Regenerate Levels if OrgunitLevel and DISTINCT OrgunitStructure.level differs
$organisationunitStructureLevels = $entityManager->createQuery('SELECT DISTINCT organisationunitLevel.level FROM HrisOrganisationunitBundle:OrganisationunitStructure organisationunitStructure INNER JOIN organisationunitStructure.level organisationunitLevel ORDER BY organisationunitLevel.level ')->getResult();
$organisationunitLevelInfos = $entityManager->createQuery('SELECT organisationunitLevel.level,organisationunitLevel.name,organisationunitLevel.description FROM HrisOrganisationunitBundle:OrganisationunitLevel organisationunitLevel ORDER BY organisationunitLevel.level ')->getResult();
$organisationunitStructureLevels = $this->array_value_recursive('level', $organisationunitStructureLevels);
$organisationunitLevelsLevel = $this->array_value_recursive('level', $organisationunitLevelInfos);
if ($organisationunitLevelsLevel != $organisationunitStructureLevels && !empty($organisationunitStructureLevels)) {
$logger->info('Regenerating organisationunit levels');
if (!empty($organisationunitLevelInfos)) {
// Cache in-memory saved Level names and descriptions
$organisationunitLevelsName = $this->array_value_recursive('name', $organisationunitLevelInfos);
$organisationunitLevelsDescription = $this->array_value_recursive('description', $organisationunitLevelInfos);
$organisationunitLevelsName = array_combine($organisationunitLevelsLevel, $organisationunitLevelsName);
$organisationunitLevelsDescription = array_combine($organisationunitLevelsLevel, $organisationunitLevelsDescription);
$qb = $entityManager->createQueryBuilder('organisationunitLevel')->delete('HrisOrganisationunitBundle:OrganisationunitLevel', 'organisationunitLevel')->getQuery()->getResult();
}
foreach ($organisationunitStructureLevels as $key => $organisationunitStructureLevel) {
// Update Levels
$organisationunitLevel = new OrganisationunitLevel();
if (in_array($organisationunitStructureLevel, $organisationunitLevelsLevel)) {
$organisationunitLevel->setName($organisationunitLevelsName[$organisationunitStructureLevel]);
$organisationunitLevel->setDescription($organisationunitLevelsDescription[$organisationunitStructureLevel]);
$organisationunitLevel->setLevel($organisationunitStructureLevel);
$entityManager->persist($organisationunitLevel);
} else {
$organisationunitLevel->setName('Level' . $organisationunitStructureLevel);
$organisationunitLevel->setDescription('Level' . $organisationunitStructureLevel);
$organisationunitLevel->setLevel($organisationunitStructureLevel);
$entityManager->persist($organisationunitLevel);
}
}
$entityManager->flush();
}
$dataArray = NULL;
$id = 0;
//Prepare field Option map, converting from stored FieldOption key in record value array to actual text value
$fieldOptions = $entityManager->getRepository('HrisFormBundle:FieldOption')->findAll();
foreach ($fieldOptions as $fieldOptionKey => $fieldOption) {
$recordFieldOptionKey = ucfirst(Record::getFieldOptionKey());
$fieldOptionMap[call_user_func_array(array($fieldOption, "get{$recordFieldOptionKey}"), array())] = $fieldOption->getValue();
}
unset($fieldOptions);
foreach ($records as $recordKey => $record) {
$currentInstance = $record->getInstance();
$dataValue = $record->getValue();
$id++;
$dataArray['id'] = $id;
$age = NULL;
$retirementDate = NULL;
$employmentDuration = NULL;
$dataArray['instance'] = $record->getInstance();
foreach ($this->getResourceTableFieldMember() as $resourceTableKey => $resourceTableFieldMember) {
unset($field);
$field = $resourceTableFieldMember->getField();
// Field Options
/**
* Made dynamic, on which field column is used as key, i.e. uid, name or id.
*/
// Translates to $field->getUid()
// or $field->getUid() depending on value of $recordKeyName
$recordFieldKey = ucfirst(Record::getFieldKey());
$valueKey = call_user_func_array(array($field, "get{$recordFieldKey}"), array());
if ($field->getIsCalculated()) {
if (preg_match_all('/\\#{([^\\}]+)\\}/', $field->getCalculatedExpression(), $match)) {
$fields = $entityManager->getRepository('HrisFormBundle:Field')->findOneBy(array('name' => $match[1][0]));
// @todo mechanism to notified on flawed formula(resulting in no match in db)
$valueKey = @@call_user_func_array(array($fields, "get{$recordFieldKey}"), array());
}
}
if (isset($dataValue[$valueKey])) {
$dataArray[$field->getName()] = $dataValue[$valueKey];
if ($field->getInputType()->getName() == 'Select') {
if (isset($fieldOptionMap[$dataValue[$valueKey]])) {
// Resolve actual value from stored key
$dataArray[$field->getName()] = trim($fieldOptionMap[$dataValue[$valueKey]]);
} else {
$dataArray[$field->getName()] = NULL;
}
} else {
if ($field->getInputType()->getName() == 'Date') {
if ($field->getIsCalculated() == true) {
if (!empty($dataValue[$valueKey])) {
$displayValue = new \DateTime($dataValue[$valueKey]['date'], new \DateTimeZone($dataValue[$valueKey]['timezone']));
$datavalue = str_replace($match[0][0], $displayValue->format('Y-m-d'), $field->getCalculatedExpression());
$dataArray[$field->getName()] = eval("return {$datavalue};");
//$dataArray[$field->getName()] = trim($displayValue->format('Y-m-d H:i:s.u')); //working on date format fix
if ($field->getDataType()->getName() == 'Date') {
$dataArray[$field->getName() . '_month_text'] = trim($displayValue->format('F'));
$dataArray[$field->getName() . '_year'] = trim($displayValue->format('Y'));
}
} else {
$dataArray[$field->getName()] = NULL;
}
} else {
if (!empty($dataValue[$valueKey])) {
$displayValue = new \DateTime($dataValue[$valueKey]['date'], new \DateTimeZone($dataValue[$valueKey]['timezone']));
$dataArray[$field->getName()] = trim($displayValue->format('Y-m-d'));
//working on date format fix
//$dataArray[$field->getName().'_day'] = trim($displayValue->format('l'));
//$dataArray[$field->getName().'_month_number'] = trim($displayValue->format('m'));
$dataArray[$field->getName() . '_month_text'] = trim($displayValue->format('F'));
$dataArray[$field->getName() . '_year'] = trim($displayValue->format('Y'));
//$dataArray[$field->getName().'_month_and_year'] = trim($displayValue->format('F Y'));
} else {
$dataArray[$field->getName()] = NULL;
}
}
} else {
if ($field->getDataType()->getName() == 'Integer') {
if (!empty($dataValue[$valueKey])) {
$intValue = (int) $dataValue[$valueKey];
$dataArray[$field->getName()] = trim($intValue);
//working on Integers format fix
} else {
$dataArray[$field->getName()] = NULL;
}
} else {
if ($field->getDataType()->getName() == 'Double') {
if (!empty($dataValue[$valueKey])) {
$floatValue = (double) $dataValue[$valueKey];
$dataArray[$field->getName()] = trim($floatValue);
//working on float format fix
} else {
$dataArray[$field->getName()] = NULL;
}
} else {
$dataArray[$field->getName()] = substr(trim($dataValue[$valueKey]), 0, 63);
}
}
}
}
} else {
$dataArray[$field->getName()] = NULL;
}
// @todo implement after creation of history date class
if ($field->getHashistory() && $field->getInputType()->getName() == "Select" && isset($dataValue[$valueKey])) {
// Fetch history date with instance same as our current data
$historyDates = $entityManager->getRepository('HrisRecordsBundle:HistoryDate')->findOneBy(array('instance' => $record->getInstance(), 'history' => $dataValue[$valueKey], 'field' => $field));
if (!empty($historyDates)) {
if (!empty($historyDates)) {
$dataArray[$field->getName() . '_last_updated'] = trim($historyDates->getPreviousdate()->format('Y-m-d H:i:s.u'));
//$dataArray[$field->getName().'_last_updated_day'] = trim($historyDates->getPreviousdate()->format('l'));
//$dataArray[$field->getName().'_last_updated_month_number'] = trim($historyDates->getPreviousdate()->format('m'));
$dataArray[$field->getName() . '_last_updated_month_text'] = trim($historyDates->getPreviousdate()->format('F'));
$dataArray[$field->getName() . '_last_updated_year'] = trim($historyDates->getPreviousdate()->format('Y'));
//$dataArray[$field->getName().'_last_updated_month_and_year'] = trim($historyDates->getPreviousdate()->format('F Y'));
}
}
}
}
// Fill in Levels
foreach ($organisationunitLevels as $organisationunitLevelKey => $organisationunitLevel) {
$organisationunitLevelName = str_replace(' ', '_', "level" . $organisationunitLevel->getLevel() . "_" . str_replace(',', '_', str_replace('.', '_', str_replace('/', '_', $organisationunitLevel->getName()))));
$organisationunitStructure = $record->getOrganisationunit()->getOrganisationunitStructure();
$nLevelParent = $organisationunitStructure->getParentByNLevelsBack($record->getOrganisationunit(), $organisationunitStructure->getLevel()->getLevel() - $organisationunitLevel->getLevel());
if (!empty($nLevelParent)) {
$dataArray[$organisationunitLevelName] = $nLevelParent->getLongname();
}
$thisrganisationunitLevel = $entityManager->getRepository('HrisOrganisationunitBundle:OrganisationunitLevel')->findOneBy(array('level' => $organisationunitStructure->getLevel()->getLevel()));
$organisationunitLevelName = str_replace(' ', '_', "level" . $thisrganisationunitLevel->getLevel() . "_" . str_replace(',', '_', str_replace('.', '_', str_replace('/', '_', $thisrganisationunitLevel->getName()))));
$dataArray[$organisationunitLevelName] = $record->getOrganisationunit()->getLongname();
unset($nLevelParent);
unset($organisationunitLevelName);
unset($organisationunitStructure);
}
// Fill in Groupset Columns
foreach ($organisationunitGroupsets as $organisationunitGroupsetKey => $organisationunitGroupset) {
$organisationunitGroupsetNames = NULL;
foreach ($organisationunitGroupset->getOrganisationunitGroup() as $organisationunitGroupKey => $organisationunitGroup) {
if ($organisationunitGroup->getOrganisationunit()->contains($record->getOrganisationunit())) {
if (empty($organisationunitGroupNames)) {
$organisationunitGroupNames = $organisationunitGroup->getName();
} else {
if (!preg_match("/" . $organisationunitGroup->getName() . "/", $organisationunitGroupNames)) {
$organisationunitGroupNames .= ',' . ($organisationunitGroupNames = $organisationunitGroup->getName());
}
}
}
}
if (!isset($organisationunitGroupNames)) {
$organisationunitGroupNames = NULL;
}
$dataArray[$organisationunitGroupset->getName()] = $organisationunitGroupNames;
unset($organisationunitGroupNames);
}
// Form and Orgunit
$dataArray['Organisationunit_name'] = $record->getOrganisationunit()->getLongname();
$dataArray['Form_name'] = $record->getForm()->getName();
$dataArray['Organisationunit_id'] = $record->getOrganisationunit()->getId();
$dataArray['Form_id'] = $record->getForm()->getId();
$dataArray['Lastupdated'] = trim($record->getLastupdated()->format('Y-m-d H:i:s.u'));
$entityManager->getConnection()->insert($resourceTableName . '_temporary', $dataArray);
$logger->info('Inserted record instance ' . $dataArray['instance'] . ' for ' . $dataArray['Organisationunit_name'] . ' on form: ' . $record->getForm()->getName());
$totalInsertedRecords++;
unset($dataArray);
unset($dataValue);
unset($currentInstance);
}
}
unset($records);
$dataInsertionLap = $stopwatch->lap('resourceTableGeneration');
$dataInsertionDuration = round($dataInsertionLap->getDuration() / 1000, 2) - $schemaGenerationDuration;
$singleDataInsertionDuration = round($dataInsertionDuration / $totalInsertedRecords, 2);
if ($dataInsertionDuration < 60) {
$dataInsertionDurationMessage = round($dataInsertionDuration, 2) . ' sec.';
} elseif ($dataInsertionDuration >= 60 && $dataInsertionDuration < 3600) {
$dataInsertionDurationMessage = round($dataInsertionDuration / 60, 2) . ' min.';
} elseif ($dataInsertionDuration >= 3600 && $dataInsertionDuration < 216000) {
$dataInsertionDurationMessage = round($dataInsertionDuration / 3600, 2) . ' hrs';
} else {
$dataInsertionDurationMessage = round($dataInsertionDuration / 86400, 2) . ' days';
}
if ($singleDataInsertionDuration < 60) {
$singleDataInsertionDurationMessage = "(" . round($singleDataInsertionDuration, 2) . ' sec./record)';
} elseif ($singleDataInsertionDuration >= 60 && $singleDataInsertionDuration < 3600) {
$singleDataInsertionDurationMessage = "(" . round($singleDataInsertionDuration / 60, 2) . ' min./record)';
} elseif ($singleDataInsertionDuration >= 3600 && $singleDataInsertionDuration < 216000) {
$singleDataInsertionDurationMessage = "(" . round($singleDataInsertionDuration / 3600, 2) . ' hrs/record)';
} else {
$singleDataInsertionDurationMessage = "(" . round($singleDataInsertionDuration / 86400, 2) . ' days/record)';
}
$this->messagelog .= "Operation: " . $totalInsertedRecords . " Records Inserted into " . $resourceTableName . " in " . $dataInsertionDurationMessage . $singleDataInsertionDurationMessage . ".\n";
$logger->info($this->messagelog);
/*
* Replace existing resource table with completely regenerated temporary resource table
*/
// Drop table if it exists
if ($schemaManager->tablesExist($resourceTableName)) {
$schemaManager->dropTable($resourceTableName);
}
$schemaManager->renameTable($resourceTableName . '_temporary', $resourceTableName);
unset($schemaManager);
$stopwatch->lap('resourceTableGeneration');
$offlineDuration = round($dataInsertionLap->getDuration() / 1000, 2) - ($schemaGenerationDuration + $dataInsertionDuration);
if ($offlineDuration < 60) {
$offlineDurationMessage = round($offlineDuration, 2) . ' sec.';
} elseif ($offlineDuration >= 60 && $offlineDuration < 3600) {
$offlineDurationMessage = round($offlineDuration / 60, 2) . ' min.';
} elseif ($offlineDuration >= 3600 && $offlineDuration < 216000) {
$offlineDurationMessage = round($offlineDuration / 3600, 2) . ' hrs';
} else {
$offlineDurationMessage = round($offlineDuration / 86400, 2) . ' days';
}
$this->messagelog .= "Reports Offline Time: Resourcetable was offline for " . $offlineDurationMessage . "\n";
// Update last generated after running the script
$this->setLastgenerated(new \DateTime('now'));
$this->setIsgenerating(False);
$entityManager->persist($this);
try {
$entityManager->flush();
} catch (\Doctrine\Orm\NoResultException $e) {
$error = "Error Last generated";
}
/*
* Check Clock for time spent
*/
$resourceTableGenerationTime = $stopwatch->stop('resourceTableGeneration');
$duration = $resourceTableGenerationTime->getDuration() / 1000;
unset($stopwatch);
if ($duration < 60) {
$durationMessage = round($duration, 2) . ' seconds';
} elseif ($duration >= 60 && $duration < 3600) {
$durationMessage = round($duration / 60, 2) . ' minutes';
} elseif ($duration >= 3600 && $duration < 216000) {
$durationMessage = round($duration / 3600, 2) . ' hours';
} else {
$durationMessage = round($duration / 86400, 2) . ' hours';
}
$this->messagelog .= "Operation: Resource Table generation completeted in " . $durationMessage . ".\n\n";
return True;
} else {
$this->messagelog .= "Status: Resource Table " . $resourceTableName . " is upto date.\n";
return False;
}
}
protected function _mp3Data()
{
if (empty($this->urls)) {
$this->getUrls();
}
$combined_data = [];
$stopwatch = new Stopwatch();
$stopwatch->start('download');
foreach ($this->urls as $url) {
set_time_limit(90);
list($filename, $data) = $this->_download($url);
if (!$filename) {
continue;
}
if (!empty($data)) {
sleep($this->sleep);
} else {
$data = file_get_contents($filename);
}
$combined_data[] = $data;
$stopwatch->lap('download');
}
$this->spent_time = $stopwatch->stop('download');
return $combined_data;
}
