protected function execute(InputInterface $input, OutputInterface $output)
{
$id = $input->getArgument('collection');
$idmodule = $input->getArgument('module');
$dbname = $input->getArgument('dbname');
$usermail = $input->getArgument('usermail');
if ($id && $idmodule && $dbname && $usermail) {
$orphans = array();
$error = '';
$dm = $this->getContainer()->get('doctrine.odm.mongodb.document_manager');
$dm->getConfiguration()->setDefaultDB($dbname);
$configuration = $dm->getConnection()->getConfiguration();
$configuration->setLoggerCallable(null);
\MongoCursor::$timeout = -1;
$module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule);
if (!$module) {
$error = 'Unable to find Module entity.';
}
if ($module->getType() == 'text') {
$error = 'Module entity: Wrong type.';
}
if (empty($error)) {
/*
* Open the uploaded csv
*/
$csvfile = __DIR__ . '/../Resources/uploads/' . $module->getCollection()->getAlias() . '/' . $module->getAlias() . '.csv';
$handle = fopen($csvfile, "r");
/*
* Get the module properties
*/
$columns = fgetcsv($handle, 0, ";");
$fields = array();
$attributes = $module->getProperties();
foreach ($attributes as $field) {
$fields[] = $field;
}
$s = microtime(true);
$batchSize = 100;
$size = 0;
$rowCount = 0;
$errorCount = 0;
if ($module->getType() == 'image') {
while (($data = fgetcsv($handle, 0, ';')) !== false) {
$num = count($data);
$image = new Image();
$attributes = array();
for ($c = 0; $c < $num; $c++) {
$value = trim($this->data_encode($data[$c]));
//check for int or float value
if (is_numeric($value)) {
$tmp_value = intval($value);
if ($value == $tmp_value) {
$value = $tmp_value;
} else {
$tmp_value = floatval($value);
if ($value == $tmp_value) {
$value = $tmp_value;
}
}
}
//
$attributes[$fields[$c]->getId()] = $value;
switch ($fields[$c]->getType()) {
case 'file':
$image->setPath($value . '');
break;
case 'copyright':
$image->setCopyright($value . '');
break;
case 'idparent':
$image->setIdparent($value . '');
break;
case 'idmodule':
$image->setIdentifier($value . '');
break;
}
}
$image->setProperty($attributes);
$image->setModule($module);
$parent = null;
if ($module->getParent()) {
$parent_q = $dm->createQueryBuilder('PlantnetDataBundle:Plantunit')->field('module.id')->equals($module->getParent()->getId())->field('identifier')->equals($image->getIdparent())->getQuery()->execute();
foreach ($parent_q as $p) {
$parent = $p;
}
}
if ($parent) {
$image->setPlantunit($parent);
$image->setTitle1($parent->getTitle1());
$image->setTitle2($parent->getTitle2());
$image->setTitle3($parent->getTitle3());
$dm->persist($image);
$rowCount++;
$size++;
if (!$parent->getHasimages()) {
$parent->setHasimages(true);
$dm->persist($parent);
$size++;
}
//update Taxons
$taxons = $parent->getTaxonsrefs();
if (count($taxons)) {
foreach ($taxons as $taxon) {
if (!$taxon->getHasimages()) {
$taxon->setHasimages(true);
$dm->persist($taxon);
$size++;
}
if ($taxon->getIssynonym()) {
$taxon_valid = $taxon->getChosen();
if (!$taxon_valid->getHasimages()) {
$taxon_valid->setHasimages(true);
$dm->persist($taxon_valid);
$size++;
}
$parent_taxon_valid = $taxon_valid->getParent();
while ($parent_taxon_valid) {
if (!$parent_taxon_valid->getHasimages()) {
$parent_taxon_valid->setHasimages(true);
$dm->persist($parent_taxon_valid);
$size++;
}
$parent_taxon_valid = $parent_taxon_valid->getParent();
}
}
}
}
if ($size >= $batchSize) {
$dm->flush();
$dm->clear();
gc_collect_cycles();
$module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule);
$size = 0;
}
} else {
$orphans[$image->getIdparent()] = $image->getIdparent();
$errorCount++;
$dm->detach($image);
}
}
$dm->flush();
$dm->clear();
gc_collect_cycles();
$module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule);
} elseif ($module->getType() == 'locality') {
while (($data = fgetcsv($handle, 0, ';')) !== false) {
$geo_error = false;
$num = count($data);
$location = new Location();
$coordinates = new Coordinates();
$attributes = array();
for ($c = 0; $c < $num; $c++) {
$value = trim($this->data_encode($data[$c]));
//check for int or float value
if (is_numeric($value)) {
$tmp_value = intval($value);
if ($value == $tmp_value) {
$value = $tmp_value;
} else {
$tmp_value = floatval($value);
if ($value == $tmp_value) {
$value = $tmp_value;
}
}
}
//
$attributes[$fields[$c]->getId()] = $value;
switch ($fields[$c]->getType()) {
case 'lon':
if (strlen($value) == 0) {
$geo_error = true;
}
$value = str_replace(',', '.', $value);
$value = floatval($value);
$location->setLongitude($value);
$coordinates->setX($value);
break;
case 'lat':
if (strlen($value) == 0) {
$geo_error = true;
}
$value = str_replace(',', '.', $value);
$value = floatval($value);
$location->setLatitude($value);
$coordinates->setY($value);
break;
case 'idparent':
$location->setIdparent($value . '');
break;
case 'idmodule':
$location->setIdentifier($value . '');
break;
}
}
if (!$geo_error) {
$location->setCoordinates($coordinates);
$location->setProperty($attributes);
$location->setModule($module);
$parent = null;
if ($module->getParent()) {
$parent_q = $dm->createQueryBuilder('PlantnetDataBundle:Plantunit')->field('module.id')->equals($module->getParent()->getId())->field('identifier')->equals($location->getIdparent())->getQuery()->execute();
foreach ($parent_q as $p) {
$parent = $p;
}
}
if ($parent) {
$location->setPlantunit($parent);
$location->setTitle1($parent->getTitle1());
$location->setTitle2($parent->getTitle2());
$location->setTitle3($parent->getTitle3());
$dm->persist($location);
$rowCount++;
$size++;
if (!$parent->getHaslocations()) {
$parent->setHaslocations(true);
$dm->persist($parent);
$size++;
}
//update Taxons
$taxons = $parent->getTaxonsrefs();
if (count($taxons)) {
foreach ($taxons as $taxon) {
if (!$taxon->getHaslocations()) {
$taxon->setHaslocations(true);
$dm->persist($taxon);
$size++;
}
if ($taxon->getIssynonym()) {
$taxon_valid = $taxon->getChosen();
if (!$taxon_valid->getHaslocations()) {
$taxon_valid->setHaslocations(true);
$dm->persist($taxon_valid);
$size++;
}
$parent_taxon_valid = $taxon_valid->getParent();
while ($parent_taxon_valid) {
if (!$parent_taxon_valid->getHaslocations()) {
$parent_taxon_valid->setHaslocations(true);
$dm->persist($parent_taxon_valid);
}
$parent_taxon_valid = $parent_taxon_valid->getParent();
}
}
}
}
if ($size >= $batchSize) {
$dm->flush();
$dm->clear();
gc_collect_cycles();
$module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule);
$size = 0;
}
} else {
$orphans[$location->getIdparent()] = $location->getIdparent();
$errorCount++;
$dm->detach($location);
}
} else {
$orphans[$location->getIdparent()] = $location->getIdparent();
$errorCount++;
$dm->detach($location);
}
}
$dm->flush();
$dm->clear();
gc_collect_cycles();
$module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule);
} elseif ($module->getType() == 'other') {
while (($data = fgetcsv($handle, 0, ';')) !== false) {
$num = count($data);
$other = new Other();
$attributes = array();
for ($c = 0; $c < $num; $c++) {
$value = trim($this->data_encode($data[$c]));
//check for int or float value
if (is_numeric($value)) {
$tmp_value = intval($value);
if ($value == $tmp_value) {
$value = $tmp_value;
} else {
$tmp_value = floatval($value);
if ($value == $tmp_value) {
$value = $tmp_value;
}
}
}
//
$attributes[$fields[$c]->getId()] = $value;
switch ($fields[$c]->getType()) {
case 'idparent':
$other->setIdparent($value . '');
break;
case 'idmodule':
$other->setIdentifier($value . '');
break;
}
}
$other->setProperty($attributes);
$other->setModule($module);
$parent = null;
if ($module->getParent()) {
$parent_q = $dm->createQueryBuilder('PlantnetDataBundle:Plantunit')->field('module.id')->equals($module->getParent()->getId())->field('identifier')->equals($other->getIdparent())->getQuery()->execute();
foreach ($parent_q as $p) {
$parent = $p;
}
}
if ($parent) {
$other->setPlantunit($parent);
$dm->persist($other);
$rowCount++;
$size++;
if ($size >= $batchSize) {
$dm->flush();
$dm->clear();
gc_collect_cycles();
$module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule);
$size = 0;
}
} else {
$orphans[$other->getIdparent()] = $other->getIdparent();
$errorCount++;
$dm->detach($other);
}
}
$dm->flush();
$dm->clear();
gc_collect_cycles();
$module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule);
}
fclose($handle);
$module->setNbrows($rowCount);
$module->setUpdating(false);
$dm->persist($module);
$dm->flush();
$dm->clear();
$e = microtime(true);
echo ' Inserted ' . $rowCount . ' objects in ' . ($e - $s) . ' seconds' . PHP_EOL;
if (file_exists($csvfile)) {
unlink($csvfile);
}
$message = 'Importation Success: ' . $rowCount . ' objects imported in ' . ($e - $s) . ' seconds';
$message .= "\n";
$message .= 'Importation Error: ' . $errorCount;
if (count($orphans)) {
$message .= "\n" . 'These "id parent" do not exist:' . "\n";
$message .= implode(', ', $orphans);
}
} else {
$message = $error;
}
$message_mail = \Swift_Message::newInstance()->setSubject('Publish : task ended')->setFrom($this->getContainer()->getParameter('from_email_adress'))->setTo($usermail)->setBody($message . $this->getContainer()->get('templating')->render('PlantnetDataBundle:Backend\\Mail:task.txt.twig'));
$this->getContainer()->get('mailer')->send($message_mail);
$spool = $this->getContainer()->get('mailer')->getTransport()->getSpool();
$transport = $this->getContainer()->get('swiftmailer.transport.real');
$spool->flushQueue($transport);
//
$connection = new \MongoClient();
$db = $connection->{$dbname};
$db->Module->update(array('_id' => new \MongoId($idmodule)), array('$set' => array('updating' => false)));
}
}
private function update_locality($dm, $module)
{
\MongoCursor::$timeout = -1;
$idmodule = $module->getId();
$parent_to_update = null;
if ($module->getType() == 'image' || $module->getType() == 'locality') {
if ($module->getParent() && $module->getParent()->getDeleting() === false) {
$parent_to_update = $module->getParent()->getId();
$this->reset_taxo($dm, $module->getParent());
}
}
if ($parent_to_update == null) {
return $module;
}
//remove old data
$dm->createQueryBuilder('PlantnetDataBundle:Location')->remove()->field('module')->references($module)->getQuery()->execute();
$module_parent = $dm->getRepository('PlantnetDataBundle:Module')->find($parent_to_update);
if ($module_parent) {
$has_loc_child = false;
$loc_child_ids = array();
$tmp_children = $module_parent->getChildren();
if (count($tmp_children)) {
foreach ($tmp_children as $tmp_child) {
if ($tmp_child->getType() == 'locality' && $tmp_child->getId() != $idmodule) {
$has_loc_child = true;
$loc_child_ids[] = $tmp_child->getId();
}
}
}
//locations
\MongoCursor::$timeout = -1;
if ($has_loc_child) {
//Punit with locations
$ids_with_loc = array();
$punits = $dm->createQueryBuilder('PlantnetDataBundle:Plantunit')->hydrate(false)->select('_id')->field('module')->references($module_parent)->field('haslocations')->equals(true)->getQuery()->execute();
foreach ($punits as $id) {
$ids_with_loc[] = $id['_id']->{'$id'};
}
$punits = null;
unset($punits);
//Punit ids in locations
$ids_in_loc = array();
$punits = $dm->createQueryBuilder('PlantnetDataBundle:Location')->hydrate(false)->select('plantunit')->field('module.id')->in($loc_child_ids)->getQuery()->execute();
foreach ($punits as $id) {
$ids_in_loc[] = $id['plantunit']['$id']->{'$id'};
}
$punits = null;
unset($punits);
//Punits without locations
$punit_ids_without_locations = array_diff($ids_with_loc, $ids_in_loc);
if (count($punit_ids_without_locations)) {
$tabs_to_update = array_chunk($punit_ids_without_locations, 100);
foreach ($tabs_to_update as $sub_tab) {
$dm->createQueryBuilder('PlantnetDataBundle:Plantunit')->update()->multiple(true)->field('module')->references($module_parent)->field('id')->in($sub_tab)->field('haslocations')->set(false)->getQuery()->execute();
}
}
$dm->clear();
$module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule);
$module_parent = $dm->getRepository('PlantnetDataBundle:Module')->find($parent_to_update);
} else {
$dm->createQueryBuilder('PlantnetDataBundle:Plantunit')->update()->multiple(true)->field('module')->references($module_parent)->field('haslocations')->equals(true)->field('haslocations')->set(false)->getQuery()->execute();
}
}
//add new data
$csvfile = __DIR__ . '/../Resources/uploads/' . $module->getCollection()->getAlias() . '/' . $module->getAlias() . '.csv';
$handle = fopen($csvfile, "r");
$columns = fgetcsv($handle, 0, ";");
$fields = array();
$attributes = $module->getProperties();
foreach ($attributes as $field) {
$fields[] = $field;
}
$batchSize = 100;
$size = 0;
$rowCount = 0;
while (($data = fgetcsv($handle, 0, ';')) !== false) {
$geo_error = false;
$num = count($data);
$location = new Location();
$coordinates = new Coordinates();
$attributes = array();
for ($c = 0; $c < $num; $c++) {
$value = trim($this->data_encode($data[$c]));
//check for int or float value
if (is_numeric($value)) {
$tmp_value = intval($value);
if ($value == $tmp_value) {
$value = $tmp_value;
} else {
$tmp_value = floatval($value);
if ($value == $tmp_value) {
$value = $tmp_value;
}
}
}
//
$attributes[$fields[$c]->getId()] = $value;
switch ($fields[$c]->getType()) {
case 'lon':
if (strlen($value) == 0) {
$geo_error = true;
}
$value = str_replace(',', '.', $value);
$value = floatval($value);
$location->setLongitude($value);
$coordinates->setX($value);
break;
case 'lat':
if (strlen($value) == 0) {
$geo_error = true;
}
$value = str_replace(',', '.', $value);
$value = floatval($value);
$location->setLatitude($value);
$coordinates->setY($value);
break;
case 'idparent':
$location->setIdparent($value . '');
break;
case 'idmodule':
$location->setIdentifier($value . '');
break;
}
}
if (!$geo_error) {
$location->setCoordinates($coordinates);
$location->setProperty($attributes);
$location->setModule($module);
$parent = null;
if ($module->getParent()) {
$parent_q = $dm->createQueryBuilder('PlantnetDataBundle:Plantunit')->field('module.id')->equals($module->getParent()->getId())->field('identifier')->equals($location->getIdparent())->getQuery()->execute();
foreach ($parent_q as $p) {
$parent = $p;
}
}
if ($parent) {
$location->setPlantunit($parent);
$location->setTitle1($parent->getTitle1());
$location->setTitle2($parent->getTitle2());
$location->setTitle3($parent->getTitle3());
$dm->persist($location);
$rowCount++;
$size++;
if (!$parent->getHaslocations()) {
$parent->setHaslocations(true);
$dm->persist($parent);
$size++;
}
if ($size >= $batchSize) {
$dm->flush();
$dm->clear();
gc_collect_cycles();
$module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule);
$size = 0;
}
} else {
$dm->detach($location);
}
} else {
$dm->detach($location);
}
}
fclose($handle);
$module->setNbrows($rowCount);
$dm->persist($module);
$dm->flush();
$dm->clear();
gc_collect_cycles();
$module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule);
return $module;
}