protected function execute(InputInterface $input, OutputInterface $output)
 {
     $id = $input->getArgument('collection');
     $idmodule = $input->getArgument('module');
     $dbname = $input->getArgument('dbname');
     $usermail = $input->getArgument('usermail');
     if ($id && $idmodule && $dbname && $usermail) {
         $orphans = array();
         $error = '';
         $dm = $this->getContainer()->get('doctrine.odm.mongodb.document_manager');
         $dm->getConfiguration()->setDefaultDB($dbname);
         $configuration = $dm->getConnection()->getConfiguration();
         $configuration->setLoggerCallable(null);
         \MongoCursor::$timeout = -1;
         $module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule);
         if (!$module) {
             $error = 'Unable to find Module entity.';
         }
         if ($module->getType() == 'text') {
             $error = 'Module entity: Wrong type.';
         }
         if (empty($error)) {
             /*
              * Open the uploaded csv
              */
             $csvfile = __DIR__ . '/../Resources/uploads/' . $module->getCollection()->getAlias() . '/' . $module->getAlias() . '.csv';
             $handle = fopen($csvfile, "r");
             /*
              * Get the module properties
              */
             $columns = fgetcsv($handle, 0, ";");
             $fields = array();
             $attributes = $module->getProperties();
             foreach ($attributes as $field) {
                 $fields[] = $field;
             }
             $s = microtime(true);
             $batchSize = 100;
             $size = 0;
             $rowCount = 0;
             $errorCount = 0;
             if ($module->getType() == 'image') {
                 while (($data = fgetcsv($handle, 0, ';')) !== false) {
                     $num = count($data);
                     $image = new Image();
                     $attributes = array();
                     for ($c = 0; $c < $num; $c++) {
                         $value = trim($this->data_encode($data[$c]));
                         //check for int or float value
                         if (is_numeric($value)) {
                             $tmp_value = intval($value);
                             if ($value == $tmp_value) {
                                 $value = $tmp_value;
                             } else {
                                 $tmp_value = floatval($value);
                                 if ($value == $tmp_value) {
                                     $value = $tmp_value;
                                 }
                             }
                         }
                         //
                         $attributes[$fields[$c]->getId()] = $value;
                         switch ($fields[$c]->getType()) {
                             case 'file':
                                 $image->setPath($value . '');
                                 break;
                             case 'copyright':
                                 $image->setCopyright($value . '');
                                 break;
                             case 'idparent':
                                 $image->setIdparent($value . '');
                                 break;
                             case 'idmodule':
                                 $image->setIdentifier($value . '');
                                 break;
                         }
                     }
                     $image->setProperty($attributes);
                     $image->setModule($module);
                     $parent = null;
                     if ($module->getParent()) {
                         $parent_q = $dm->createQueryBuilder('PlantnetDataBundle:Plantunit')->field('module.id')->equals($module->getParent()->getId())->field('identifier')->equals($image->getIdparent())->getQuery()->execute();
                         foreach ($parent_q as $p) {
                             $parent = $p;
                         }
                     }
                     if ($parent) {
                         $image->setPlantunit($parent);
                         $image->setTitle1($parent->getTitle1());
                         $image->setTitle2($parent->getTitle2());
                         $image->setTitle3($parent->getTitle3());
                         $dm->persist($image);
                         $rowCount++;
                         $size++;
                         if (!$parent->getHasimages()) {
                             $parent->setHasimages(true);
                             $dm->persist($parent);
                             $size++;
                         }
                         //update Taxons
                         $taxons = $parent->getTaxonsrefs();
                         if (count($taxons)) {
                             foreach ($taxons as $taxon) {
                                 if (!$taxon->getHasimages()) {
                                     $taxon->setHasimages(true);
                                     $dm->persist($taxon);
                                     $size++;
                                 }
                                 if ($taxon->getIssynonym()) {
                                     $taxon_valid = $taxon->getChosen();
                                     if (!$taxon_valid->getHasimages()) {
                                         $taxon_valid->setHasimages(true);
                                         $dm->persist($taxon_valid);
                                         $size++;
                                     }
                                     $parent_taxon_valid = $taxon_valid->getParent();
                                     while ($parent_taxon_valid) {
                                         if (!$parent_taxon_valid->getHasimages()) {
                                             $parent_taxon_valid->setHasimages(true);
                                             $dm->persist($parent_taxon_valid);
                                             $size++;
                                         }
                                         $parent_taxon_valid = $parent_taxon_valid->getParent();
                                     }
                                 }
                             }
                         }
                         if ($size >= $batchSize) {
                             $dm->flush();
                             $dm->clear();
                             gc_collect_cycles();
                             $module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule);
                             $size = 0;
                         }
                     } else {
                         $orphans[$image->getIdparent()] = $image->getIdparent();
                         $errorCount++;
                         $dm->detach($image);
                     }
                 }
                 $dm->flush();
                 $dm->clear();
                 gc_collect_cycles();
                 $module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule);
             } elseif ($module->getType() == 'locality') {
                 while (($data = fgetcsv($handle, 0, ';')) !== false) {
                     $geo_error = false;
                     $num = count($data);
                     $location = new Location();
                     $coordinates = new Coordinates();
                     $attributes = array();
                     for ($c = 0; $c < $num; $c++) {
                         $value = trim($this->data_encode($data[$c]));
                         //check for int or float value
                         if (is_numeric($value)) {
                             $tmp_value = intval($value);
                             if ($value == $tmp_value) {
                                 $value = $tmp_value;
                             } else {
                                 $tmp_value = floatval($value);
                                 if ($value == $tmp_value) {
                                     $value = $tmp_value;
                                 }
                             }
                         }
                         //
                         $attributes[$fields[$c]->getId()] = $value;
                         switch ($fields[$c]->getType()) {
                             case 'lon':
                                 if (strlen($value) == 0) {
                                     $geo_error = true;
                                 }
                                 $value = str_replace(',', '.', $value);
                                 $value = floatval($value);
                                 $location->setLongitude($value);
                                 $coordinates->setX($value);
                                 break;
                             case 'lat':
                                 if (strlen($value) == 0) {
                                     $geo_error = true;
                                 }
                                 $value = str_replace(',', '.', $value);
                                 $value = floatval($value);
                                 $location->setLatitude($value);
                                 $coordinates->setY($value);
                                 break;
                             case 'idparent':
                                 $location->setIdparent($value . '');
                                 break;
                             case 'idmodule':
                                 $location->setIdentifier($value . '');
                                 break;
                         }
                     }
                     if (!$geo_error) {
                         $location->setCoordinates($coordinates);
                         $location->setProperty($attributes);
                         $location->setModule($module);
                         $parent = null;
                         if ($module->getParent()) {
                             $parent_q = $dm->createQueryBuilder('PlantnetDataBundle:Plantunit')->field('module.id')->equals($module->getParent()->getId())->field('identifier')->equals($location->getIdparent())->getQuery()->execute();
                             foreach ($parent_q as $p) {
                                 $parent = $p;
                             }
                         }
                         if ($parent) {
                             $location->setPlantunit($parent);
                             $location->setTitle1($parent->getTitle1());
                             $location->setTitle2($parent->getTitle2());
                             $location->setTitle3($parent->getTitle3());
                             $dm->persist($location);
                             $rowCount++;
                             $size++;
                             if (!$parent->getHaslocations()) {
                                 $parent->setHaslocations(true);
                                 $dm->persist($parent);
                                 $size++;
                             }
                             //update Taxons
                             $taxons = $parent->getTaxonsrefs();
                             if (count($taxons)) {
                                 foreach ($taxons as $taxon) {
                                     if (!$taxon->getHaslocations()) {
                                         $taxon->setHaslocations(true);
                                         $dm->persist($taxon);
                                         $size++;
                                     }
                                     if ($taxon->getIssynonym()) {
                                         $taxon_valid = $taxon->getChosen();
                                         if (!$taxon_valid->getHaslocations()) {
                                             $taxon_valid->setHaslocations(true);
                                             $dm->persist($taxon_valid);
                                             $size++;
                                         }
                                         $parent_taxon_valid = $taxon_valid->getParent();
                                         while ($parent_taxon_valid) {
                                             if (!$parent_taxon_valid->getHaslocations()) {
                                                 $parent_taxon_valid->setHaslocations(true);
                                                 $dm->persist($parent_taxon_valid);
                                             }
                                             $parent_taxon_valid = $parent_taxon_valid->getParent();
                                         }
                                     }
                                 }
                             }
                             if ($size >= $batchSize) {
                                 $dm->flush();
                                 $dm->clear();
                                 gc_collect_cycles();
                                 $module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule);
                                 $size = 0;
                             }
                         } else {
                             $orphans[$location->getIdparent()] = $location->getIdparent();
                             $errorCount++;
                             $dm->detach($location);
                         }
                     } else {
                         $orphans[$location->getIdparent()] = $location->getIdparent();
                         $errorCount++;
                         $dm->detach($location);
                     }
                 }
                 $dm->flush();
                 $dm->clear();
                 gc_collect_cycles();
                 $module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule);
             } elseif ($module->getType() == 'other') {
                 while (($data = fgetcsv($handle, 0, ';')) !== false) {
                     $num = count($data);
                     $other = new Other();
                     $attributes = array();
                     for ($c = 0; $c < $num; $c++) {
                         $value = trim($this->data_encode($data[$c]));
                         //check for int or float value
                         if (is_numeric($value)) {
                             $tmp_value = intval($value);
                             if ($value == $tmp_value) {
                                 $value = $tmp_value;
                             } else {
                                 $tmp_value = floatval($value);
                                 if ($value == $tmp_value) {
                                     $value = $tmp_value;
                                 }
                             }
                         }
                         //
                         $attributes[$fields[$c]->getId()] = $value;
                         switch ($fields[$c]->getType()) {
                             case 'idparent':
                                 $other->setIdparent($value . '');
                                 break;
                             case 'idmodule':
                                 $other->setIdentifier($value . '');
                                 break;
                         }
                     }
                     $other->setProperty($attributes);
                     $other->setModule($module);
                     $parent = null;
                     if ($module->getParent()) {
                         $parent_q = $dm->createQueryBuilder('PlantnetDataBundle:Plantunit')->field('module.id')->equals($module->getParent()->getId())->field('identifier')->equals($other->getIdparent())->getQuery()->execute();
                         foreach ($parent_q as $p) {
                             $parent = $p;
                         }
                     }
                     if ($parent) {
                         $other->setPlantunit($parent);
                         $dm->persist($other);
                         $rowCount++;
                         $size++;
                         if ($size >= $batchSize) {
                             $dm->flush();
                             $dm->clear();
                             gc_collect_cycles();
                             $module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule);
                             $size = 0;
                         }
                     } else {
                         $orphans[$other->getIdparent()] = $other->getIdparent();
                         $errorCount++;
                         $dm->detach($other);
                     }
                 }
                 $dm->flush();
                 $dm->clear();
                 gc_collect_cycles();
                 $module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule);
             }
             fclose($handle);
             $module->setNbrows($rowCount);
             $module->setUpdating(false);
             $dm->persist($module);
             $dm->flush();
             $dm->clear();
             $e = microtime(true);
             echo ' Inserted ' . $rowCount . ' objects in ' . ($e - $s) . ' seconds' . PHP_EOL;
             if (file_exists($csvfile)) {
                 unlink($csvfile);
             }
             $message = 'Importation Success: ' . $rowCount . ' objects imported in ' . ($e - $s) . ' seconds';
             $message .= "\n";
             $message .= 'Importation Error: ' . $errorCount;
             if (count($orphans)) {
                 $message .= "\n" . 'These "id parent" do not exist:' . "\n";
                 $message .= implode(', ', $orphans);
             }
         } else {
             $message = $error;
         }
         $message_mail = \Swift_Message::newInstance()->setSubject('Publish : task ended')->setFrom($this->getContainer()->getParameter('from_email_adress'))->setTo($usermail)->setBody($message . $this->getContainer()->get('templating')->render('PlantnetDataBundle:Backend\\Mail:task.txt.twig'));
         $this->getContainer()->get('mailer')->send($message_mail);
         $spool = $this->getContainer()->get('mailer')->getTransport()->getSpool();
         $transport = $this->getContainer()->get('swiftmailer.transport.real');
         $spool->flushQueue($transport);
         //
         $connection = new \MongoClient();
         $db = $connection->{$dbname};
         $db->Module->update(array('_id' => new \MongoId($idmodule)), array('$set' => array('updating' => false)));
     }
 }
Пример #2
0
 private function update_locality($dm, $module)
 {
     \MongoCursor::$timeout = -1;
     $idmodule = $module->getId();
     $parent_to_update = null;
     if ($module->getType() == 'image' || $module->getType() == 'locality') {
         if ($module->getParent() && $module->getParent()->getDeleting() === false) {
             $parent_to_update = $module->getParent()->getId();
             $this->reset_taxo($dm, $module->getParent());
         }
     }
     if ($parent_to_update == null) {
         return $module;
     }
     //remove old data
     $dm->createQueryBuilder('PlantnetDataBundle:Location')->remove()->field('module')->references($module)->getQuery()->execute();
     $module_parent = $dm->getRepository('PlantnetDataBundle:Module')->find($parent_to_update);
     if ($module_parent) {
         $has_loc_child = false;
         $loc_child_ids = array();
         $tmp_children = $module_parent->getChildren();
         if (count($tmp_children)) {
             foreach ($tmp_children as $tmp_child) {
                 if ($tmp_child->getType() == 'locality' && $tmp_child->getId() != $idmodule) {
                     $has_loc_child = true;
                     $loc_child_ids[] = $tmp_child->getId();
                 }
             }
         }
         //locations
         \MongoCursor::$timeout = -1;
         if ($has_loc_child) {
             //Punit with locations
             $ids_with_loc = array();
             $punits = $dm->createQueryBuilder('PlantnetDataBundle:Plantunit')->hydrate(false)->select('_id')->field('module')->references($module_parent)->field('haslocations')->equals(true)->getQuery()->execute();
             foreach ($punits as $id) {
                 $ids_with_loc[] = $id['_id']->{'$id'};
             }
             $punits = null;
             unset($punits);
             //Punit ids in locations
             $ids_in_loc = array();
             $punits = $dm->createQueryBuilder('PlantnetDataBundle:Location')->hydrate(false)->select('plantunit')->field('module.id')->in($loc_child_ids)->getQuery()->execute();
             foreach ($punits as $id) {
                 $ids_in_loc[] = $id['plantunit']['$id']->{'$id'};
             }
             $punits = null;
             unset($punits);
             //Punits without locations
             $punit_ids_without_locations = array_diff($ids_with_loc, $ids_in_loc);
             if (count($punit_ids_without_locations)) {
                 $tabs_to_update = array_chunk($punit_ids_without_locations, 100);
                 foreach ($tabs_to_update as $sub_tab) {
                     $dm->createQueryBuilder('PlantnetDataBundle:Plantunit')->update()->multiple(true)->field('module')->references($module_parent)->field('id')->in($sub_tab)->field('haslocations')->set(false)->getQuery()->execute();
                 }
             }
             $dm->clear();
             $module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule);
             $module_parent = $dm->getRepository('PlantnetDataBundle:Module')->find($parent_to_update);
         } else {
             $dm->createQueryBuilder('PlantnetDataBundle:Plantunit')->update()->multiple(true)->field('module')->references($module_parent)->field('haslocations')->equals(true)->field('haslocations')->set(false)->getQuery()->execute();
         }
     }
     //add new data
     $csvfile = __DIR__ . '/../Resources/uploads/' . $module->getCollection()->getAlias() . '/' . $module->getAlias() . '.csv';
     $handle = fopen($csvfile, "r");
     $columns = fgetcsv($handle, 0, ";");
     $fields = array();
     $attributes = $module->getProperties();
     foreach ($attributes as $field) {
         $fields[] = $field;
     }
     $batchSize = 100;
     $size = 0;
     $rowCount = 0;
     while (($data = fgetcsv($handle, 0, ';')) !== false) {
         $geo_error = false;
         $num = count($data);
         $location = new Location();
         $coordinates = new Coordinates();
         $attributes = array();
         for ($c = 0; $c < $num; $c++) {
             $value = trim($this->data_encode($data[$c]));
             //check for int or float value
             if (is_numeric($value)) {
                 $tmp_value = intval($value);
                 if ($value == $tmp_value) {
                     $value = $tmp_value;
                 } else {
                     $tmp_value = floatval($value);
                     if ($value == $tmp_value) {
                         $value = $tmp_value;
                     }
                 }
             }
             //
             $attributes[$fields[$c]->getId()] = $value;
             switch ($fields[$c]->getType()) {
                 case 'lon':
                     if (strlen($value) == 0) {
                         $geo_error = true;
                     }
                     $value = str_replace(',', '.', $value);
                     $value = floatval($value);
                     $location->setLongitude($value);
                     $coordinates->setX($value);
                     break;
                 case 'lat':
                     if (strlen($value) == 0) {
                         $geo_error = true;
                     }
                     $value = str_replace(',', '.', $value);
                     $value = floatval($value);
                     $location->setLatitude($value);
                     $coordinates->setY($value);
                     break;
                 case 'idparent':
                     $location->setIdparent($value . '');
                     break;
                 case 'idmodule':
                     $location->setIdentifier($value . '');
                     break;
             }
         }
         if (!$geo_error) {
             $location->setCoordinates($coordinates);
             $location->setProperty($attributes);
             $location->setModule($module);
             $parent = null;
             if ($module->getParent()) {
                 $parent_q = $dm->createQueryBuilder('PlantnetDataBundle:Plantunit')->field('module.id')->equals($module->getParent()->getId())->field('identifier')->equals($location->getIdparent())->getQuery()->execute();
                 foreach ($parent_q as $p) {
                     $parent = $p;
                 }
             }
             if ($parent) {
                 $location->setPlantunit($parent);
                 $location->setTitle1($parent->getTitle1());
                 $location->setTitle2($parent->getTitle2());
                 $location->setTitle3($parent->getTitle3());
                 $dm->persist($location);
                 $rowCount++;
                 $size++;
                 if (!$parent->getHaslocations()) {
                     $parent->setHaslocations(true);
                     $dm->persist($parent);
                     $size++;
                 }
                 if ($size >= $batchSize) {
                     $dm->flush();
                     $dm->clear();
                     gc_collect_cycles();
                     $module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule);
                     $size = 0;
                 }
             } else {
                 $dm->detach($location);
             }
         } else {
             $dm->detach($location);
         }
     }
     fclose($handle);
     $module->setNbrows($rowCount);
     $dm->persist($module);
     $dm->flush();
     $dm->clear();
     gc_collect_cycles();
     $module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule);
     return $module;
 }