private function glossarize($action, $dbname, $id_glossary, $usermail)
{
$error = '';
$dm = $this->getContainer()->get('doctrine.odm.mongodb.document_manager');
$dm->getConfiguration()->setDefaultDB($dbname);
$configuration = $dm->getConnection()->getConfiguration();
$configuration->setLoggerCallable(null);
$glossary = $dm->getRepository('PlantnetDataBundle:Glossary')->findOneBy(array('id' => $id_glossary));
if (!$glossary) {
$error = 'Unable to find Glossary entity.';
}
$collection = $glossary->getCollection();
if (!$collection) {
$error = 'Unable to find Glossary entity.';
}
//
if ($action == 'syn') {
$csv = __DIR__ . '/../Resources/uploads/' . $collection->getAlias() . '/glossary_syn.csv';
if (file_exists($csv)) {
$handle = fopen($csv, "r");
$field = fgetcsv($handle, 0, ";");
$csv_error = false;
if (!isset($field[0]) || empty($field[0])) {
$csv_error = true;
}
if (!isset($field[1]) || empty($field[1])) {
$csv_error = true;
}
if (!$csv_error) {
$i = 0;
$batch_size = 100;
\MongoCursor::$timeout = -1;
while (($data = fgetcsv($handle, 0, ';')) !== false) {
$def_word = isset($data[0]) ? trim($data[0]) : '';
$cur_encoding = mb_detect_encoding($def_word);
if ($cur_encoding == "UTF-8" && mb_check_encoding($def_word, "UTF-8")) {
$def_word = $def_word;
} else {
$def_word = utf8_encode($def_word);
}
$syn_word = isset($data[1]) ? trim($data[1]) : '';
$cur_encoding = mb_detect_encoding($syn_word);
if ($cur_encoding == "UTF-8" && mb_check_encoding($syn_word, "UTF-8")) {
$syn_word = $syn_word;
} else {
$syn_word = utf8_encode($syn_word);
}
if ($def_word && $syn_word) {
$definition = $dm->getRepository('PlantnetDataBundle:Definition')->findOneBy(array('glossary.id' => $glossary->getId(), 'name' => $def_word));
if ($definition) {
$i++;
if (!$definition->getHaschildren()) {
$definition->setHaschildren(true);
$dm->persist($definition);
}
$synonym = new Definition();
$synonym->setGlossary($glossary);
$synonym->setParent($definition);
$synonym->setName($syn_word);
$synonym->setDisplayedname($syn_word);
$synonym->setDefinition($definition->getDefinition());
$synonym->setPath($definition->getPath());
$dm->persist($synonym);
if ($i >= $batch_size) {
$dm->flush();
$i = 0;
$dm->clear();
gc_collect_cycles();
$glossary = $dm->getRepository('PlantnetDataBundle:Glossary')->findOneBy(array('id' => $id_glossary));
}
}
}
}
$dm->flush();
$dm->clear();
gc_collect_cycles();
$glossary = $dm->getRepository('PlantnetDataBundle:Glossary')->findOneBy(array('id' => $id_glossary));
} else {
$error = 'Error in Glossary Syn file.';
}
} else {
$error = 'Unable to find Glossary Syn file.';
}
$message = $error;
if (empty($message)) {
$message = 'Synonyms were created successfully.';
}
$message_mail = \Swift_Message::newInstance()->setSubject('Publish : task ended')->setFrom($this->getContainer()->getParameter('from_email_adress'))->setTo($usermail)->setBody($message . $this->getContainer()->get('templating')->render('PlantnetDataBundle:Backend\\Mail:task.txt.twig'));
$this->getContainer()->get('mailer')->send($message_mail);
$spool = $this->getContainer()->get('mailer')->getTransport()->getSpool();
$transport = $this->getContainer()->get('swiftmailer.transport.real');
$spool->flushQueue($transport);
} elseif ($action == 'unsyn') {
$dm->createQueryBuilder('PlantnetDataBundle:Definition')->remove()->field('glossary')->references($glossary)->field('parent')->notEqual(null)->getQuery()->execute();
}
}
/**
* @Route("/collection/{id}/glossary_importation", name="glossary_importation")
* @Method("post")
* @Template("PlantnetDataBundle:Backend\Glossary:glossary_import_glossarydata.html.twig")
*/
public function glossary_importationAction($id)
{
$user = $this->container->get('security.context')->getToken()->getUser();
$dm = $this->get('doctrine.odm.mongodb.document_manager');
$dm->getConfiguration()->setDefaultDB($this->getDataBase($user, $dm));
$configuration = $dm->getConnection()->getConfiguration();
$configuration->setLoggerCallable(null);
$request = $this->container->get('request');
set_time_limit(0);
if ($request->isXmlHttpRequest()) {
$collection = $dm->getRepository('PlantnetDataBundle:Collection')->findOneBy(array('id' => $id));
if (!$collection) {
throw $this->createNotFoundException('Unable to find Collection entity.');
}
$glossary = $collection->getGlossary();
if (!$glossary) {
throw $this->createNotFoundException('Unable to find Glossary entity.');
}
/*
* Open the uploaded csv
*/
$csvfile = __DIR__ . '/../../Resources/uploads/' . $collection->getAlias() . '/glossary.csv';
$handle = fopen($csvfile, "r");
/*
* Get the glossary properties
*/
$columns = fgetcsv($handle, 0, ";");
$fields = array();
$attributes = $glossary->getProperties();
foreach ($attributes as $field) {
$fields[] = $field;
}
/*
* Initialise the metrics
*/
//echo "Memory usage before: " . (memory_get_usage() / 1024) . " KB" . PHP_EOL;
$s = microtime(true);
$batchSize = 500;
$rowCount = '';
while (($data = fgetcsv($handle, 0, ';')) !== FALSE) {
$num = count($data);
$rowCount++;
$definition = new Definition();
$definition->setGlossary($glossary);
$attributes = array();
$def_error = false;
for ($c = 0; $c < $num; $c++) {
$value = trim($this->data_encode($data[$c]));
$attributes[$fields[$c]->getId()] = $value;
switch ($fields[$c]->getType()) {
case 'keyword':
if (empty($value)) {
$def_error = true;
}
$definition->setName($value);
$definition->setDisplayedname($value);
break;
case 'definition':
$definition->setDefinition($value);
break;
case 'file':
$definition->setPath($value);
break;
}
}
if (!$def_error) {
$definition->setHaschildren(false);
$dm->persist($definition);
}
if ($rowCount % $batchSize == 0) {
$dm->flush();
$dm->clear();
$collection = $dm->getRepository('PlantnetDataBundle:Collection')->findOneBy(array('id' => $id));
$glossary = $collection->getGlossary();
}
}
$dm->persist($glossary);
$dm->flush();
$dm->clear();
//echo "Memory usage after: " . (memory_get_usage() / 1024) . " KB" . PHP_EOL;
$e = microtime(true);
echo ' Inserted ' . $rowCount . ' objects in ' . ($e - $s) . ' seconds' . PHP_EOL;
fclose($handle);
if (file_exists($csvfile)) {
unlink($csvfile);
}
return $this->container->get('templating')->renderResponse('PlantnetDataBundle:Backend\\Glossary:glossary_import_glossarydata.html.twig', array('importCount' => 'Importation Success: ' . $rowCount . ' objects imported'));
} else {
return $this->glossary_import_dataAction($id);
}
}
