/**
* {@inheritDoc}
*/
public function load(ObjectManager $manager)
{
$SC1 = new SyntaxonCore();
$SC1->setId(1);
// ...
$manager->persist($SC1);
$manager->flush();
}
/**
* {@inheritDoc}
*/
public function load(ObjectManager $manager)
{
$syntaxon0 = new SyntaxonCore();
$syntaxon0->setfixedCode(1);
$syntaxon0->setCatminatCode('01/');
$syntaxon0->setLevel('HAB');
$syntaxon0->setSyntaxonName('Prairies (habitat)');
$syntaxon0->setSyntaxonAuthor('');
$syntaxon0->setCommonName('Prairies (habitat nom commun)');
$syntaxon1 = new SyntaxonCore();
$syntaxon1->setfixedCode(2);
$syntaxon1->setCatminatCode('01/1.');
$syntaxon1->setLevel('CLA');
$syntaxon1->setSyntaxonName('Agrostio - Arrhenatheretea');
$syntaxon1->setSyntaxonAuthor('Br. Bl. 1900');
$syntaxon1->setCommonName('Prairies');
$syntaxon2 = new SyntaxonCore();
$syntaxon2->setfixedCode(3);
$syntaxon2->setCatminatCode('01/1.1');
$syntaxon2->setLevel('SUBCLA');
$syntaxon2->setSyntaxonName('Arrhenatherenea');
$syntaxon2->setSyntaxonAuthor('Br. Bl. 1901');
$syntaxon2->setCommonName('Prairies etalia');
$syntaxon3 = new SyntaxonCore();
$syntaxon3->setfixedCode(4);
$syntaxon3->setCatminatCode('01/1.1.1');
$syntaxon3->setLevel('ORD');
$syntaxon3->setSyntaxonName('Arrhenatheretalia');
$syntaxon3->setSyntaxonAuthor('Br. Bl. 1902');
$syntaxon3->setCommonName('Prairies');
$syntaxon4 = new SyntaxonCore();
$syntaxon4->setfixedCode(3);
$syntaxon4->setCatminatCode('01/1.1');
$syntaxon4->setLevel('SUBCLA');
$syntaxon4->setSyntaxonName('Agrostienea');
$syntaxon4->setSyntaxonAuthor('Br. Bl. 1903');
$syntaxon4->setCommonName('Prairies etalia');
$manager->persist($syntaxon0);
$manager->persist($syntaxon1);
$manager->persist($syntaxon2);
$manager->persist($syntaxon3);
$manager->persist($syntaxon4);
$manager->flush();
}
/**
*
* Import the baseveg core data (csv file from baseveg)
*
* @Security("has_role('ROLE_SUPER_ADMIN')")
*/
public function importCoreAction(Request $request)
{
// retrieves ioHelpers service
$ioHelpers = $this->get('eveg_app.io_helpers');
$request = $this->getRequest();
// Creates the form...
$form = $this->createFormBuilder()->add('importFile', 'file')->add('onlyLog', 'checkbox', array('data' => true, 'label' => "Ne pas importer les données, simuler l'importation et afficher le log", 'required' => false))->getForm();
// ... and then hydrates it
$form->handleRequest($request);
// Only log ?
$onlyLog = $form->get('onlyLog')->getData();
// Logs
$logIsUpToDate = (string) "";
// log for data already up to date
$logToUpdate = (string) "";
// data to update
$logToCreate = (string) "";
// data to create in db
$logToRemove = (string) "";
// data to remove in db
$countIsUpToDate = 0;
// counter for data already up to date
$countToUpdate = 0;
// -- data to update
$countToCreate = 0;
// -- data to create in db
$countToRemove = 0;
// -- data to remove in db
// Job routine
if ($form->isValid()) {
$batchSize = 100;
// $entity will be flushed and detached each $batchSize time (prevent memory overload)
$i = 0;
// just an incremental
$cycle = 0;
// counts the number of batch cycles
// Recovers the uploaded file
$file = $form->get('importFile')->getData();
$mimeType = $file->getMimeType();
$clientOriginaFilelName = $file->getClientOriginalName();
$clientOriginaFilelNameWhitoutExtension = basename($file->getClientOriginalName(), '.csv');
$clientOriginalFileExtension = $file->getClientOriginalExtension();
// File type verification
if ($mimeType !== 'text/csv' && FALSE == preg_match('/\\.csv/i', $clientOriginaFilelName)) {
throw new HttpException(400, "The file must be a csv (the file MIME type should be 'text/csv' or the file name should contains '.csv').");
}
// Variables
$dateTime = new \DateTime('now');
$day = $dateTime->format('Y-m-d');
$hour = $dateTime->format('H-i-s');
// Moves the uploaded file
$importFileName = $clientOriginaFilelNameWhitoutExtension . '_' . $day . '_' . $hour . '.' . $clientOriginalFileExtension;
$dir = __DIR__ . '/../../../../web/uploads/import/core';
$file->move($dir, $importFileName);
// Variables
$import = $ioHelpers->csv_to_array($dir . '/' . $importFileName, ';');
// push the csv data into an array
$importKeys = array_column($import, 'fixedCode');
// get the id values
$importKeys = array_map('intval', $importKeys);
// make sure that $importKeys contains integer data
$batchSize = 100;
// $entity will be flushed and detached each $batchSize time (prevent memory overload)
$i = 0;
// just an incremental
$cycle = 0;
// counts the number of batch cycles
// Retrieves all syntaxonCore entities
$em = $this->getDoctrine()->getManager();
$entities = $em->getRepository('evegAppBundle:SyntaxonCore')->findAll();
// Detach useless entities :
// By default, Doctrine persist linked entities even if they're empty.
// We only keep SyntaxonCore entities in the Doctrine's unit of work
foreach ($em->getUnitOfWork()->getIdentityMap() as $key_unit => $persistedUnit) {
foreach ($persistedUnit as $key_element => $persistedElement) {
$elementClass = get_class($persistedElement);
if ($elementClass !== "eveg\\AppBundle\\Entity\\SyntaxonCore") {
$em->detach($persistedElement);
}
}
$em->flush();
}
// Push SCore fixed codes (==id) into array
$dbKeys = array();
foreach ($entities as $key => $entity) {
array_push($dbKeys, $entity->getFixedCode());
}
// Parse import data and detach SyntaxonCore entities already up to date
foreach ($import as $key => $importedEntity) {
$idToRetrieve = array_search($importedEntity['fixedCode'], $dbKeys);
// Syntaxon already exists in db
if ($idToRetrieve !== FALSE) {
// Find differences
$diffFixedCode = false;
$diffCatminatCode = false;
$diffLevel = false;
$diffSyntaxonName = false;
$diffSyntaxonAuthor = false;
$diffCommonName = false;
$diffCommonNameEn = false;
if ($entities[$idToRetrieve]->getFixedCode() != $importedEntity['fixedCode']) {
$diffFixedCode = true;
}
if ($entities[$idToRetrieve]->getCatminatCode() != $importedEntity['catminatCode']) {
$diffCatminatCode = true;
}
if ($entities[$idToRetrieve]->getLevel() != $importedEntity['level']) {
$diffLevel = true;
}
if ($entities[$idToRetrieve]->getSyntaxonName() != $importedEntity['syntaxonName']) {
$diffSyntaxonName = true;
}
if ($entities[$idToRetrieve]->getSyntaxonAuthor() != $importedEntity['syntaxonAuthor']) {
$diffSyntaxonAuthor = true;
}
if ($entities[$idToRetrieve]->getCommonName() != $importedEntity['commonNameFr']) {
$diffCommonName = true;
}
if ($entities[$idToRetrieve]->getCommonNameEn() != $importedEntity['commonNameEn']) {
$diffCommonNameEn = true;
}
// If there is no difference between csv file and db
// then we detach the entity
if (!$diffFixedCode && !$diffCatminatCode && !$diffLevel && !$diffSyntaxonName && !$diffSyntaxonAuthor && !$diffCommonName && !$diffCommonNameEn) {
$countIsUpToDate++;
// Log : no difference for this entity, nothing to deal with doctrine
/*$logIsUpToDate .= "<table class='table element'>
<tr class='header'>
<td class='title'>Syntaxon</td>
<td>
<table class='table'>
<tr><td><b>db :</b> (".$entities[$idToRetrieve]->getFixedCode().")(".$entities[$idToRetrieve]->getCatminatCode().") ".$entities[$idToRetrieve]->getSyntaxonName()." ".$entities[$idToRetrieve]->getSyntaxonAuthor()."</td></tr>
<tr><td><b>csv :</b> (".$importedEntity['fixedCode'].") (".$importedEntity['catminatCode'].") ".$importedEntity['syntaxonName']." ".$importedEntity['syntaxonAuthor']."</td></tr>
</table>
</td>
</tr>
</table>";*/
// Delete item from csv file and from entity manager
unset($import[$key]);
$em->detach($entities[$idToRetrieve]);
}
}
}
$em->flush();
// Push entities into array
$dbKeys = array();
foreach ($entities as $key => $entity) {
array_push($dbKeys, $entity->getFixedCode());
}
// Check data before import
//$this->checkBeofreImportCoreAction($import);
foreach ($import as $key => $importedEntity) {
// Does the entity already exists (compare both id) ?
$idToRetrieve = array_search($importedEntity['fixedCode'], $dbKeys);
// Yes, it already exists
if ($idToRetrieve !== FALSE) {
// Find differences
$diffFixedCode = false;
$diffCatminatCode = false;
$diffLevel = false;
$diffSyntaxonName = false;
$diffSyntaxonAuthor = false;
$diffCommonName = false;
$diffCommonNameEn = false;
if ($entities[$idToRetrieve]->getFixedCode() != $import[$key]['fixedCode']) {
$diffFixedCode = true;
}
if ($entities[$idToRetrieve]->getCatminatCode() != $import[$key]['catminatCode']) {
$diffCatminatCode = true;
}
if ($entities[$idToRetrieve]->getLevel() != $import[$key]['level']) {
$diffLevel = true;
}
if ($entities[$idToRetrieve]->getSyntaxonName() != $import[$key]['syntaxonName']) {
$diffSyntaxonName = true;
}
if ($entities[$idToRetrieve]->getSyntaxonAuthor() != $import[$key]['syntaxonAuthor']) {
$diffSyntaxonAuthor = true;
}
if ($entities[$idToRetrieve]->getCommonName() != $import[$key]['commonNameFr']) {
$diffCommonName = true;
}
if ($entities[$idToRetrieve]->getCommonNameEn() != $import[$key]['commonNameEn']) {
$diffCommonNameEn = true;
}
// There is no difference (redundant)
if (!$diffFixedCode && !$diffCatminatCode && !$diffLevel && !$diffSyntaxonName && !$diffSyntaxonAuthor && !$diffCommonName && !$diffCommonNameEn) {
// Log : no difference for this entity, nothing to deal with doctrine
// There is, at less, one difference
} else {
$countToUpdate++;
$logToUpdate .= "<table class='table element'>\n\t\t\t\t\t\t\t\t\t\t\t<tr class='header'>\n\t\t\t\t\t\t\t\t\t\t\t\t<td class='title'>Syntaxon</td>\n\t\t\t\t\t\t\t\t\t\t\t\t<td>\n\t\t\t\t\t\t\t\t\t\t\t\t\t<table class='table'>\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t<tr><td><b>db :</b> (" . $entities[$idToRetrieve]->getFixedCode() . ")(" . $entities[$idToRetrieve]->getCatminatCode() . ") " . $entities[$idToRetrieve]->getSyntaxonName() . " " . $entities[$idToRetrieve]->getSyntaxonAuthor() . "</td></tr>\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t<tr><td><b>csv :</b> (" . $importedEntity['fixedCode'] . ") (" . $importedEntity['catminatCode'] . ") " . $importedEntity['syntaxonName'] . " " . $importedEntity['syntaxonAuthor'] . "</td></tr>\n\t\t\t\t\t\t\t\t\t\t\t\t\t</table>\n\t\t\t\t\t\t\t\t\t\t\t\t</td>\n\t\t\t\t\t\t\t\t\t\t\t</tr>";
// update entity
if ($diffCatminatCode) {
$logToUpdate .= $ioHelpers->getTableFragment('catminat code', $entities[$idToRetrieve]->getCatminatCode(), $importedEntity['catminatCode']);
$entities[$idToRetrieve]->setCatminatCode($importedEntity['catminatCode']);
}
if ($diffLevel) {
$logToUpdate .= $ioHelpers->getTableFragment('level', $entities[$idToRetrieve]->getLevel(), $importedEntity['level']);
$entities[$idToRetrieve]->setLevel($importedEntity['level']);
}
if ($diffSyntaxonName) {
$logToUpdate .= $ioHelpers->getTableFragment('syntaxon name', $entities[$idToRetrieve]->getSyntaxonName(), $importedEntity['syntaxonName']);
$entities[$idToRetrieve]->setSyntaxonName($importedEntity['syntaxonName']);
}
if ($diffSyntaxonAuthor) {
$logToUpdate .= $ioHelpers->getTableFragment('syntaxon author', $entities[$idToRetrieve]->getSyntaxonAuthor(), $importedEntity['syntaxonAuthor']);
$entities[$idToRetrieve]->setSyntaxonAuthor($importedEntity['syntaxonAuthor']);
}
if ($diffCommonName) {
$logToUpdate .= $ioHelpers->getTableFragment('common name (fr)', $entities[$idToRetrieve]->getCommonName(), $importedEntity['commonNameFr']);
$entities[$idToRetrieve]->setCommonName($import[$key]['commonNameFr']);
}
if ($diffCommonNameEn) {
$logToUpdate .= $ioHelpers->getTableFragment('common name (en)', $entities[$idToRetrieve]->getCommonNameEn(), $importedEntity['commonNameEn']);
$entities[$idToRetrieve]->setCommonNameEn($importedEntity['commonNameEn']);
}
$logToUpdate .= '</table>';
}
// No (have to create a new entity)
} else {
// create new entity
$countToCreate++;
$newSyntaxonCore = new SyntaxonCore();
$newSyntaxonCore->setFixedCode($importedEntity['fixedCode'])->setCatminatCode($importedEntity['catminatCode'])->setLevel($importedEntity['level'])->setSyntaxonName($importedEntity['syntaxonName'])->setSyntaxonAuthor($importedEntity['syntaxonAuthor'])->setCommonName($importedEntity['commonNameFr'])->setCommonNameEn($importedEntity['commonNameEn'])->setId($importedEntity['fixedCode']);
$em->persist($newSyntaxonCore);
// Log entity to create
$logToCreate .= "<table class='table element'>\n\t\t\t\t\t\t\t\t\t\t<tr class='header'>\n\t\t\t\t\t\t\t\t\t\t\t<td class='title'>Syntaxon</td>\n\t\t\t\t\t\t\t\t\t\t\t<td>\n\t\t\t\t\t\t\t\t\t\t\t\t<table class='table'>\n\t\t\t\t\t\t\t\t\t\t\t\t\t<tr><td><b>db :</b> null</td></tr>\n\t\t\t\t\t\t\t\t\t\t\t\t\t<tr><td><b>csv :</b> (" . $importedEntity['fixedCode'] . ") (" . $importedEntity['catminatCode'] . ") " . $importedEntity['syntaxonName'] . " " . $importedEntity['syntaxonAuthor'] . "</td></tr>\n\t\t\t\t\t\t\t\t\t\t\t\t</table>\n\t\t\t\t\t\t\t\t\t\t\t</td>\n\t\t\t\t\t\t\t\t\t\t</tr>";
$logToCreate .= $ioHelpers->getTableFragment('id', 'null', $importedEntity['id']);
$logToCreate .= $ioHelpers->getTableFragment('catminat code', 'null', $importedEntity['catminatCode']);
$logToCreate .= $ioHelpers->getTableFragment('level', 'null', $importedEntity['level']);
$logToCreate .= $ioHelpers->getTableFragment('syntaxon name', 'null', $importedEntity['syntaxonName']);
$logToCreate .= $ioHelpers->getTableFragment('syntaxon author', 'null', $importedEntity['syntaxonAuthor']);
$logToCreate .= $ioHelpers->getTableFragment('common name (fr)', 'null', $importedEntity['commonNameFr']);
$logToCreate .= $ioHelpers->getTableFragment('common name (en)', 'null', $importedEntity['commonNameEn']);
$logToCreate .= "</table>";
}
// Flush and detach entities each $batchSize time
if ($i % $batchSize === 0) {
// Force setId : http://www.ens.ro/2012/07/03/symfony2-doctrine-force-entity-id-on-persist/
if (isset($newSyntaxonCore)) {
$metadata = $em->getClassMetaData(get_class($newSyntaxonCore));
$metadata->setIdGeneratorType(\Doctrine\ORM\Mapping\ClassMetadata::GENERATOR_TYPE_NONE);
}
// Flush & detach entities already flushed
if (!$onlyLog) {
$em->flush();
}
// Before detaching $batchsize entities, check if we've got enought
if (count($entities) - $i > $batchSize) {
for ($j = 0; $j <= $batchSize; $j++) {
$em->detach($entities[$cycle * $batchSize + $j]);
}
}
$cycle++;
}
$i++;
}
// for each imported entity loop
// Entity to be removed
foreach ($dbKeys as $dbKey => $dbId) {
$idToRetrieve = array_search($dbId, $importKeys);
if ($idToRetrieve != FALSE) {
// There is an occurence
continue;
} else {
// No occurence : syntaxon id to remove
$countToRemove++;
$logToRemove .= "<table class='table element'>\n\t\t\t\t\t\t\t<tr class='header'>\n\t\t\t\t\t\t\t\t<td class='title'>Syntaxon</td>\n\t\t\t\t\t\t\t\t<td>\n\t\t\t\t\t\t\t\t\t<table class='table'>\n\t\t\t\t\t\t\t\t\t\t<tr><td><b>db :</b> (" . $entities[$dbKey]->getFixedCode() . ") (" . $entities[$dbKey]->getCatminatCode() . ") " . $entities[$dbKey]->getSyntaxonName() . " " . $entities[$dbKey]->getSyntaxonAuthor() . "</td></tr>\n\t\t\t\t\t\t\t\t\t\t<tr><td><b>csv :</b> null</td></tr>\n\t\t\t\t\t\t\t\t\t</table>\n\t\t\t\t\t\t\t\t</td>\n\t\t\t\t\t\t\t</tr>";
$logToRemove .= $ioHelpers->getTableFragment('id', $entities[$dbKey]->getFixedCode(), 'null');
$logToRemove .= $ioHelpers->getTableFragment('catminat code', $entities[$dbKey]->getCatminatCode(), 'null');
$logToRemove .= $ioHelpers->getTableFragment('level', $entities[$dbKey]->getLevel(), 'null');
$logToRemove .= $ioHelpers->getTableFragment('syntaxon name', $entities[$dbKey]->getSyntaxonName(), 'null');
$logToRemove .= $ioHelpers->getTableFragment('syntaxon author', $entities[$dbKey]->getSyntaxonAuthor(), 'null');
$logToRemove .= $ioHelpers->getTableFragment('common name (fr)', $entities[$dbKey]->getCommonName(), 'null');
$logToRemove .= $ioHelpers->getTableFragment('common name (en)', $entities[$dbKey]->getCommonNameEn(), 'null');
}
}
$logToRemove .= "</table>";
// Flush the last entities (the last batch cycle may not be full)
// Force setId : http://www.ens.ro/2012/07/03/symfony2-doctrine-force-entity-id-on-persist/
if (isset($newSyntaxonCore)) {
$metadata = $em->getClassMetaData(get_class($newSyntaxonCore));
$metadata->setIdGeneratorType(\Doctrine\ORM\Mapping\ClassMetadata::GENERATOR_TYPE_NONE);
}
// Update last update param
if (!$onlyLog) {
$lastUp = $em->getRepository('evegAppBundle:Infos')->findAll()[0];
$lastUp->setLastUpdate(new \Datetime('now'));
}
// Flush & clear EM
if (!$onlyLog) {
$em->flush();
}
$em->clear();
// Flash messages
if (!$onlyLog) {
$this->get('session')->getFlashBag()->add('success', $countToUpdate . ' données ont été mises à jour et ' . $countToCreate . ' ont été créées.');
} else {
$this->get('session')->getFlashBag()->add('success', "Simulation d'import pour " . count($importKeys) . " données.");
}
// Save logs files
$path = __DIR__ . '/../../../../web/uploads/import/logs/';
$countToUpdate > 0 ? $fileNameLogToUpdate = 'bvCore_toUpdate_' . $day . '_' . $hour . '.html' : ($fileNameLogToUpdate = null);
$countIsUpToDate > 0 ? $fileNameLogIsUpToDate = 'bvCore_isUpToDate_' . $day . '_' . $hour . '.html' : ($fileNameLogIsUpToDate = null);
$countToCreate > 0 ? $fileNameLogToCreate = 'bvCore_toCreate_' . $day . '_' . $hour . '.html' : ($fileNameLogToCreate = null);
$countToRemove > 0 ? $fileNameLogToRemove = 'bvCore_toRemove_' . $day . '_' . $hour . '.html' : ($fileNameLogToRemove = null);
if (!file_exists($path)) {
mkdir($path, 0777, true);
}
if ($countToUpdate > 0) {
$outputLogToUpdate = $ioHelpers->getHtmlBase('Import Core (to update) ' . $day . ' ' . $hour, $logToUpdate, $onlyLog);
file_put_contents($path . $fileNameLogToUpdate, $outputLogToUpdate);
}
if ($countIsUpToDate > 0) {
$outputLogIsUpToDate = $ioHelpers->getHtmlBase('Import Core (is up to date) ' . $day . ' ' . $hour, $logIsUpToDate, $onlyLog);
file_put_contents($path . $fileNameLogIsUpToDate, $outputLogIsUpToDate);
}
if ($countToCreate > 0) {
$outputLogToCreate = $ioHelpers->getHtmlBase('Import Core (to create) ' . $day . ' ' . $hour, $logToCreate, $onlyLog);
file_put_contents($path . $fileNameLogToCreate, $outputLogToCreate);
}
if ($countToRemove > 0) {
$outputLogToRemove = $ioHelpers->getHtmlBase('Import Core (to remove) ' . $day . ' ' . $hour, $logToRemove, $onlyLog);
file_put_contents($path . $fileNameLogToRemove, $outputLogToRemove);
}
// Logs into database
$currentUser = $this->getUser();
$log_isUpToDate = new ImportLog();
$log_toUpdate = new ImportLog();
$log_toCreate = new ImportLog();
$log_toRemove = new ImportLog();
$em->persist($log_isUpToDate);
$em->persist($log_toUpdate);
$em->persist($log_toCreate);
$em->persist($log_toRemove);
$log_isUpToDate->setReference('core')->setState('up to date')->setDate($dateTime)->setUser($currentUser->getUserName())->setCount($countIsUpToDate)->setSimulation($onlyLog)->setLogFilename($fileNameLogIsUpToDate)->setImportFilename($importFileName)->setImportClientOriginalFilename($clientOriginaFilelName);
$log_toUpdate->setReference('core')->setState('to update')->setDate($dateTime)->setUser($currentUser->getUserName())->setCount($countToUpdate)->setSimulation($onlyLog)->setLogFilename($fileNameLogToUpdate)->setImportFilename($importFileName)->setImportClientOriginalFilename($clientOriginaFilelName);
$log_toCreate->setReference('core')->setState('to create')->setDate($dateTime)->setUser($currentUser->getUserName())->setCount($countToCreate)->setSimulation($onlyLog)->setLogFilename($fileNameLogToCreate)->setImportFilename($importFileName)->setImportClientOriginalFilename($clientOriginaFilelName);
$log_toRemove->setReference('core')->setState('to remove')->setDate($dateTime)->setUser($currentUser->getUserName())->setCount($countToRemove)->setSimulation($onlyLog)->setLogFilename($fileNameLogToRemove)->setImportFilename($importFileName)->setImportClientOriginalFilename($clientOriginaFilelName);
$em->flush();
$em->clear();
}
// end form is valid
return $this->render('evegAppBundle:Admin:importCore.html.twig', array('form' => $form->createView(), 'countIsUpToDate' => $countIsUpToDate, 'countToUpdate' => $countToUpdate, 'countToCreate' => $countToCreate, 'countToRemove' => $countToRemove, 'onlyLog' => $onlyLog, 'fileLogToUpdate' => isset($fileNameLogToUpdate) ? $fileNameLogToUpdate : null, 'fileLogIsUpToDate' => isset($fileNameLogIsUpToDate) ? $fileNameLogIsUpToDate : null, 'fileLogToCreate' => isset($fileNameLogToCreate) ? $fileNameLogToCreate : null, 'fileLogToRemove' => isset($fileNameLogToRemove) ? $fileNameLogToRemove : null));
}
