private function glossarize($action, $dbname, $id_glossary, $usermail) { $error = ''; $dm = $this->getContainer()->get('doctrine.odm.mongodb.document_manager'); $dm->getConfiguration()->setDefaultDB($dbname); $configuration = $dm->getConnection()->getConfiguration(); $configuration->setLoggerCallable(null); $glossary = $dm->getRepository('PlantnetDataBundle:Glossary')->findOneBy(array('id' => $id_glossary)); if (!$glossary) { $error = 'Unable to find Glossary entity.'; } $collection = $glossary->getCollection(); if (!$collection) { $error = 'Unable to find Glossary entity.'; } // if ($action == 'syn') { $csv = __DIR__ . '/../Resources/uploads/' . $collection->getAlias() . '/glossary_syn.csv'; if (file_exists($csv)) { $handle = fopen($csv, "r"); $field = fgetcsv($handle, 0, ";"); $csv_error = false; if (!isset($field[0]) || empty($field[0])) { $csv_error = true; } if (!isset($field[1]) || empty($field[1])) { $csv_error = true; } if (!$csv_error) { $i = 0; $batch_size = 100; \MongoCursor::$timeout = -1; while (($data = fgetcsv($handle, 0, ';')) !== false) { $def_word = isset($data[0]) ? trim($data[0]) : ''; $cur_encoding = mb_detect_encoding($def_word); if ($cur_encoding == "UTF-8" && mb_check_encoding($def_word, "UTF-8")) { $def_word = $def_word; } else { $def_word = utf8_encode($def_word); } $syn_word = isset($data[1]) ? trim($data[1]) : ''; $cur_encoding = mb_detect_encoding($syn_word); if ($cur_encoding == "UTF-8" && mb_check_encoding($syn_word, "UTF-8")) { $syn_word = $syn_word; } else { $syn_word = utf8_encode($syn_word); } if ($def_word && $syn_word) { $definition = $dm->getRepository('PlantnetDataBundle:Definition')->findOneBy(array('glossary.id' => $glossary->getId(), 'name' => $def_word)); if ($definition) { $i++; if (!$definition->getHaschildren()) { $definition->setHaschildren(true); $dm->persist($definition); } $synonym = new Definition(); $synonym->setGlossary($glossary); $synonym->setParent($definition); $synonym->setName($syn_word); $synonym->setDisplayedname($syn_word); $synonym->setDefinition($definition->getDefinition()); $synonym->setPath($definition->getPath()); $dm->persist($synonym); if ($i >= $batch_size) { $dm->flush(); $i = 0; $dm->clear(); gc_collect_cycles(); $glossary = $dm->getRepository('PlantnetDataBundle:Glossary')->findOneBy(array('id' => $id_glossary)); } } } } $dm->flush(); $dm->clear(); gc_collect_cycles(); $glossary = $dm->getRepository('PlantnetDataBundle:Glossary')->findOneBy(array('id' => $id_glossary)); } else { $error = 'Error in Glossary Syn file.'; } } else { $error = 'Unable to find Glossary Syn file.'; } $message = $error; if (empty($message)) { $message = 'Synonyms were created successfully.'; } $message_mail = \Swift_Message::newInstance()->setSubject('Publish : task ended')->setFrom($this->getContainer()->getParameter('from_email_adress'))->setTo($usermail)->setBody($message . $this->getContainer()->get('templating')->render('PlantnetDataBundle:Backend\\Mail:task.txt.twig')); $this->getContainer()->get('mailer')->send($message_mail); $spool = $this->getContainer()->get('mailer')->getTransport()->getSpool(); $transport = $this->getContainer()->get('swiftmailer.transport.real'); $spool->flushQueue($transport); } elseif ($action == 'unsyn') { $dm->createQueryBuilder('PlantnetDataBundle:Definition')->remove()->field('glossary')->references($glossary)->field('parent')->notEqual(null)->getQuery()->execute(); } }
/** * @Route("/collection/{id}/glossary_importation", name="glossary_importation") * @Method("post") * @Template("PlantnetDataBundle:Backend\Glossary:glossary_import_glossarydata.html.twig") */ public function glossary_importationAction($id) { $user = $this->container->get('security.context')->getToken()->getUser(); $dm = $this->get('doctrine.odm.mongodb.document_manager'); $dm->getConfiguration()->setDefaultDB($this->getDataBase($user, $dm)); $configuration = $dm->getConnection()->getConfiguration(); $configuration->setLoggerCallable(null); $request = $this->container->get('request'); set_time_limit(0); if ($request->isXmlHttpRequest()) { $collection = $dm->getRepository('PlantnetDataBundle:Collection')->findOneBy(array('id' => $id)); if (!$collection) { throw $this->createNotFoundException('Unable to find Collection entity.'); } $glossary = $collection->getGlossary(); if (!$glossary) { throw $this->createNotFoundException('Unable to find Glossary entity.'); } /* * Open the uploaded csv */ $csvfile = __DIR__ . '/../../Resources/uploads/' . $collection->getAlias() . '/glossary.csv'; $handle = fopen($csvfile, "r"); /* * Get the glossary properties */ $columns = fgetcsv($handle, 0, ";"); $fields = array(); $attributes = $glossary->getProperties(); foreach ($attributes as $field) { $fields[] = $field; } /* * Initialise the metrics */ //echo "Memory usage before: " . (memory_get_usage() / 1024) . " KB" . PHP_EOL; $s = microtime(true); $batchSize = 500; $rowCount = ''; while (($data = fgetcsv($handle, 0, ';')) !== FALSE) { $num = count($data); $rowCount++; $definition = new Definition(); $definition->setGlossary($glossary); $attributes = array(); $def_error = false; for ($c = 0; $c < $num; $c++) { $value = trim($this->data_encode($data[$c])); $attributes[$fields[$c]->getId()] = $value; switch ($fields[$c]->getType()) { case 'keyword': if (empty($value)) { $def_error = true; } $definition->setName($value); $definition->setDisplayedname($value); break; case 'definition': $definition->setDefinition($value); break; case 'file': $definition->setPath($value); break; } } if (!$def_error) { $definition->setHaschildren(false); $dm->persist($definition); } if ($rowCount % $batchSize == 0) { $dm->flush(); $dm->clear(); $collection = $dm->getRepository('PlantnetDataBundle:Collection')->findOneBy(array('id' => $id)); $glossary = $collection->getGlossary(); } } $dm->persist($glossary); $dm->flush(); $dm->clear(); //echo "Memory usage after: " . (memory_get_usage() / 1024) . " KB" . PHP_EOL; $e = microtime(true); echo ' Inserted ' . $rowCount . ' objects in ' . ($e - $s) . ' seconds' . PHP_EOL; fclose($handle); if (file_exists($csvfile)) { unlink($csvfile); } return $this->container->get('templating')->renderResponse('PlantnetDataBundle:Backend\\Glossary:glossary_import_glossarydata.html.twig', array('importCount' => 'Importation Success: ' . $rowCount . ' objects imported')); } else { return $this->glossary_import_dataAction($id); } }