public function execute(InputInterface $input, OutputInterface $output)
{
$mongoDbName = $input->getArgument('mongo-db');
$mongoCollectionName = $input->getArgument('mongo-collection');
$mongoHost = $input->getArgument('mongo-host');
$mongoPort = $input->getArgument('mongo-port');
$mongoUserName = $input->getOption('mongo-user');
$mongoPassword = $input->getOption('mongo-password');
$options = array();
if (!empty($mongoUserName)) {
$options['username'] = $mongoUserName;
}
if (!empty($mongoPassword)) {
$options['password'] = $mongoPassword;
}
$m = new MongoClient("mongodb://{$mongoHost}:{$mongoPort}", $options);
$db = $m->{$mongoDbName};
\MongoCursor::$timeout = -1;
$output->writeln("Connected to the database <info>{$mongoDbName}</info>");
$collection = $db->{$mongoCollectionName};
$cursor = $this->getCursor($collection);
$i = 0;
$output->writeln("Removing elastic-search index...");
$this->doDeleteIndex($input, $output);
$output->writeln("Creating elastic-search index...");
$this->doCreateIndex($input, $output);
$output->writeln("Setting-up elastic-search mapping...");
$this->doSetupMapping($input, $output);
$output->writeln("\nIndexing...");
$guzzle = $this->getGuzzle($input);
$flag = 0;
$nbEntries = $cursor->count(true);
$timeStart = microtime(true);
$requests = array();
while ($flag == 0) {
try {
foreach ($cursor as $obj) {
$i++;
unset($obj['_id']);
$requests[] = $guzzle->put($input->getArgument('es-type') . '/' . $i, null, json_encode($obj));
if (0 === $i % 180) {
$guzzle->send($requests);
$requests = array();
}
if (0 === $i % 10000) {
$elapsedTime = microtime(true) - $timeStart;
$entriesPerSeconds = floor($i / $elapsedTime);
$output->writeln(date('H:i:s') . "\tProgress: {$i}/{$nbEntries}\t({$entriesPerSeconds}/seconds)\t" . round($i / $nbEntries * 100) . "% \t" . "~" . $this->secsToString(($nbEntries - $i) / $entriesPerSeconds) . " left\tMemory usage : " . (memory_get_usage() >> 20) . "Mo");
}
}
$flag = 1;
} catch (Exception $ex) {
$output->writeln("Something went wrong within MongoDB: " . $ex->getMessage() . ", Retrying ...");
$flag = 0;
$cursor = getCursor($collection);
}
}
$output->writeln("{$i} entries processed. Took " . $this->secsToString(floor(microtime(true) - $timeStart)) . " seconds");
return 0;
}
/**
* run the map/reduce
* @static
* @return void
*/
public static function mapReduce()
{
$map = "function () {\r\n if(arguments.callee.shipcache === undefined)\r\n {\r\n arguments.callee.shipcache = {}\r\n }\r\n if(arguments.callee.shipcache[this.victim.shipTypeID] === undefined)\r\n {\r\n arguments.callee.shipcache[this.victim.shipTypeID] = db.Kingboard_EveItem.findOne({typeID: parseInt(this.victim.shipTypeID)},{'marketGroup.parentGroup.marketGroupName':1});\r\n }\r\n var ship = arguments.callee.shipcache[this.victim.shipTypeID];\r\n var info = {}\r\n info[this.victim.shipType] = 1;\r\n info[\"total\"] = 1;\r\n if(ship != null && ship.marketGroup != null)\r\n emit(ship.marketGroup.parentGroup.marketGroupName, info);\r\n }";
$reduce = "function (k, vals) {\r\n var sums = {}\r\n var total = 0;\r\n vals.forEach(function(info) {\r\n info[\"total\"] = 0;\r\n for (var key in info)\r\n {\r\n if(sums[key] === undefined)\r\n sums[key] = 0;\r\n sums[key] += info[key];\r\n total += info[key];\r\n }\r\n });\r\n sums[\"total\"] = total;\r\n return sums;\r\n }";
// we want the map/reduce to run for as long as it takes
MongoCursor::$timeout = -1;
return King23_Mongo::mapReduce("Kingboard_Kill", __CLASS__, $map, $reduce);
}
/**
*
* @param \MongoDb $db
* @param \Mongodloid_Connection $connection
*/
public function __construct(\MongoDb $db, \Mongodloid_Connection $connection)
{
parent::__construct($db, $connection);
$this->collections = Billrun_Factory::config()->getConfigValue('db.collections', array());
$timeout = Billrun_Factory::config()->getConfigValue('db.timeout', 3600000);
// default 60 minutes
Billrun_Factory::log()->log('Set database cursor timeout to: ' . $timeout, Zend_Log::INFO);
MongoCursor::$timeout = $timeout;
}
/**
* @expectedException MongoCursorTimeoutException
*/
public function testStaticTimeout()
{
$this->markTestSkipped("for now");
return;
MongoCursor::$timeout = 1;
for ($i = 0; $i < 1000; $i++) {
$this->object->insert(array("x" => "sdfjnaireojaerkgmdfkngkdsflngklsgntoigneorisgmsrklgd{$i}", "y" => $i));
}
$rows = $this->object->find(array('$eval' => 'r = 0; cursor = db.c.find(); while (cursor.hasNext()) { x = cursor.next(); for (i=0; i<200; i++) { if (x.name == "joe"+i) { r++; } } } return r;'));
foreach ($rows as $row) {
}
MongoCursor::$timeout = 30000;
}
/**
*
* @param \MongoDb $db
* @param \Mongodloid_Connection $connection
*/
public function __construct(\MongoDb $db, \Mongodloid_Connection $connection)
{
parent::__construct($db, $connection);
// TODO: refatoring the collections to factory (loose coupling)
$this->collections = Billrun_Factory::config()->getConfigValue('db.collections', array());
$timeout = Billrun_Factory::config()->getConfigValue('db.timeout', 3600000);
// default 60 minutes
if ($this->compareClientVersion('1.5.3', '<')) {
Billrun_Factory::log()->log('Set database cursor timeout to: ' . $timeout, Zend_Log::INFO);
@(MongoCursor::$timeout = $timeout);
} else {
// see also bugs:
// https://jira.mongodb.org/browse/PHP-1099
// https://jira.mongodb.org/browse/PHP-1080
$db->setWriteConcern($db->getWriteConcern()['w'], $timeout);
}
}
/**
* Test global timeouts and native mappings
*/
public function testGlobalTimeout()
{
// Create collection
$default_timeout = \MongoCursor::$timeout;
$collection = $this->getTestCollection();
// Create a cursor object
$cursor = $collection->find();
$native_cursor = $cursor->native;
// Assert cursor timeouts are updated
$this->assertEquals($cursor::$timeout, $native_cursor::$timeout);
// Check that static timeouts are binded
$default_timeout = \MongoCursor::$timeout;
$cursor::$timeout = $default_timeout / 2;
$this->assertEquals(\MongoCursor::$timeout, $cursor::$timeout);
$this->assertEquals(\MongoMinify\Cursor::$timeout, $cursor::$timeout);
\MongoCursor::$timeout = 200;
$this->assertEquals(\MongoCursor::$timeout, \MongoMinify\Cursor::$timeout);
$this->assertEquals(\MongoMinify\Cursor::$timeout, 200);
\MongoCursor::$timeout = $default_timeout;
}
/**
*
* @return \MongoCursor
*/
private function getCursor()
{
if ($this->cursor) {
return $this->cursor;
}
$this->cursor = $this->collection->getMongoCollection()->find($this->expression->toArray(), $this->fields);
if ($this->skip) {
$this->cursor->skip($this->skip);
}
if ($this->limit) {
$this->cursor->limit($this->limit);
}
if ($this->options['batchSize']) {
$this->cursor->batchSize($this->options['batchSize']);
}
if ($this->options['clientTimeout']) {
$this->cursor->timeout($this->options['clientTimeout']);
}
if ($this->options['serverTimeout']) {
$this->cursor->maxTimeMS($this->options['clientTimeout']);
}
if ($this->sort) {
$this->cursor->sort($this->sort);
}
if ($this->hint) {
$this->cursor->hint($this->hint);
}
// log request
if ($this->client->hasLogger()) {
$this->client->getLogger()->debug(get_called_class() . ': ' . json_encode(array('collection' => $this->collection->getName(), 'query' => $this->expression->toArray(), 'project' => $this->fields, 'sort' => $this->sort)));
}
$this->cursor->rewind();
// define read preferences
if ($this->readPreference) {
$this->cursor->setReadPreference($this->readPreference['type'], $this->readPreference['tagsets']);
}
return $this->cursor;
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$id = $input->getArgument('collection');
$idmodule = $input->getArgument('module');
$dbname = $input->getArgument('dbname');
$usermail = $input->getArgument('usermail');
if ($id && $idmodule && $dbname && $usermail) {
$orphans = array();
$error = '';
$dm = $this->getContainer()->get('doctrine.odm.mongodb.document_manager');
$dm->getConfiguration()->setDefaultDB($dbname);
$configuration = $dm->getConnection()->getConfiguration();
$configuration->setLoggerCallable(null);
\MongoCursor::$timeout = -1;
$module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule);
if (!$module) {
$error = 'Unable to find Module entity.';
}
if ($module->getType() == 'text') {
$error = 'Module entity: Wrong type.';
}
if (empty($error)) {
/*
* Open the uploaded csv
*/
$csvfile = __DIR__ . '/../Resources/uploads/' . $module->getCollection()->getAlias() . '/' . $module->getAlias() . '.csv';
$handle = fopen($csvfile, "r");
/*
* Get the module properties
*/
$columns = fgetcsv($handle, 0, ";");
$fields = array();
$attributes = $module->getProperties();
foreach ($attributes as $field) {
$fields[] = $field;
}
$s = microtime(true);
$batchSize = 100;
$size = 0;
$rowCount = 0;
$errorCount = 0;
if ($module->getType() == 'image') {
while (($data = fgetcsv($handle, 0, ';')) !== false) {
$num = count($data);
$image = new Image();
$attributes = array();
for ($c = 0; $c < $num; $c++) {
$value = trim($this->data_encode($data[$c]));
//check for int or float value
if (is_numeric($value)) {
$tmp_value = intval($value);
if ($value == $tmp_value) {
$value = $tmp_value;
} else {
$tmp_value = floatval($value);
if ($value == $tmp_value) {
$value = $tmp_value;
}
}
}
//
$attributes[$fields[$c]->getId()] = $value;
switch ($fields[$c]->getType()) {
case 'file':
$image->setPath($value . '');
break;
case 'copyright':
$image->setCopyright($value . '');
break;
case 'idparent':
$image->setIdparent($value . '');
break;
case 'idmodule':
$image->setIdentifier($value . '');
break;
}
}
$image->setProperty($attributes);
$image->setModule($module);
$parent = null;
if ($module->getParent()) {
$parent_q = $dm->createQueryBuilder('PlantnetDataBundle:Plantunit')->field('module.id')->equals($module->getParent()->getId())->field('identifier')->equals($image->getIdparent())->getQuery()->execute();
foreach ($parent_q as $p) {
$parent = $p;
}
}
if ($parent) {
$image->setPlantunit($parent);
$image->setTitle1($parent->getTitle1());
$image->setTitle2($parent->getTitle2());
$image->setTitle3($parent->getTitle3());
$dm->persist($image);
$rowCount++;
$size++;
if (!$parent->getHasimages()) {
$parent->setHasimages(true);
$dm->persist($parent);
$size++;
}
//update Taxons
$taxons = $parent->getTaxonsrefs();
if (count($taxons)) {
foreach ($taxons as $taxon) {
if (!$taxon->getHasimages()) {
$taxon->setHasimages(true);
$dm->persist($taxon);
$size++;
}
if ($taxon->getIssynonym()) {
$taxon_valid = $taxon->getChosen();
if (!$taxon_valid->getHasimages()) {
$taxon_valid->setHasimages(true);
$dm->persist($taxon_valid);
$size++;
}
$parent_taxon_valid = $taxon_valid->getParent();
while ($parent_taxon_valid) {
if (!$parent_taxon_valid->getHasimages()) {
$parent_taxon_valid->setHasimages(true);
$dm->persist($parent_taxon_valid);
$size++;
}
$parent_taxon_valid = $parent_taxon_valid->getParent();
}
}
}
}
if ($size >= $batchSize) {
$dm->flush();
$dm->clear();
gc_collect_cycles();
$module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule);
$size = 0;
}
} else {
$orphans[$image->getIdparent()] = $image->getIdparent();
$errorCount++;
$dm->detach($image);
}
}
$dm->flush();
$dm->clear();
gc_collect_cycles();
$module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule);
} elseif ($module->getType() == 'locality') {
while (($data = fgetcsv($handle, 0, ';')) !== false) {
$geo_error = false;
$num = count($data);
$location = new Location();
$coordinates = new Coordinates();
$attributes = array();
for ($c = 0; $c < $num; $c++) {
$value = trim($this->data_encode($data[$c]));
//check for int or float value
if (is_numeric($value)) {
$tmp_value = intval($value);
if ($value == $tmp_value) {
$value = $tmp_value;
} else {
$tmp_value = floatval($value);
if ($value == $tmp_value) {
$value = $tmp_value;
}
}
}
//
$attributes[$fields[$c]->getId()] = $value;
switch ($fields[$c]->getType()) {
case 'lon':
if (strlen($value) == 0) {
$geo_error = true;
}
$value = str_replace(',', '.', $value);
$value = floatval($value);
$location->setLongitude($value);
$coordinates->setX($value);
break;
case 'lat':
if (strlen($value) == 0) {
$geo_error = true;
}
$value = str_replace(',', '.', $value);
$value = floatval($value);
$location->setLatitude($value);
$coordinates->setY($value);
break;
case 'idparent':
$location->setIdparent($value . '');
break;
case 'idmodule':
$location->setIdentifier($value . '');
break;
}
}
if (!$geo_error) {
$location->setCoordinates($coordinates);
$location->setProperty($attributes);
$location->setModule($module);
$parent = null;
if ($module->getParent()) {
$parent_q = $dm->createQueryBuilder('PlantnetDataBundle:Plantunit')->field('module.id')->equals($module->getParent()->getId())->field('identifier')->equals($location->getIdparent())->getQuery()->execute();
foreach ($parent_q as $p) {
$parent = $p;
}
}
if ($parent) {
$location->setPlantunit($parent);
$location->setTitle1($parent->getTitle1());
$location->setTitle2($parent->getTitle2());
$location->setTitle3($parent->getTitle3());
$dm->persist($location);
$rowCount++;
$size++;
if (!$parent->getHaslocations()) {
$parent->setHaslocations(true);
$dm->persist($parent);
$size++;
}
//update Taxons
$taxons = $parent->getTaxonsrefs();
if (count($taxons)) {
foreach ($taxons as $taxon) {
if (!$taxon->getHaslocations()) {
$taxon->setHaslocations(true);
$dm->persist($taxon);
$size++;
}
if ($taxon->getIssynonym()) {
$taxon_valid = $taxon->getChosen();
if (!$taxon_valid->getHaslocations()) {
$taxon_valid->setHaslocations(true);
$dm->persist($taxon_valid);
$size++;
}
$parent_taxon_valid = $taxon_valid->getParent();
while ($parent_taxon_valid) {
if (!$parent_taxon_valid->getHaslocations()) {
$parent_taxon_valid->setHaslocations(true);
$dm->persist($parent_taxon_valid);
}
$parent_taxon_valid = $parent_taxon_valid->getParent();
}
}
}
}
if ($size >= $batchSize) {
$dm->flush();
$dm->clear();
gc_collect_cycles();
$module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule);
$size = 0;
}
} else {
$orphans[$location->getIdparent()] = $location->getIdparent();
$errorCount++;
$dm->detach($location);
}
} else {
$orphans[$location->getIdparent()] = $location->getIdparent();
$errorCount++;
$dm->detach($location);
}
}
$dm->flush();
$dm->clear();
gc_collect_cycles();
$module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule);
} elseif ($module->getType() == 'other') {
while (($data = fgetcsv($handle, 0, ';')) !== false) {
$num = count($data);
$other = new Other();
$attributes = array();
for ($c = 0; $c < $num; $c++) {
$value = trim($this->data_encode($data[$c]));
//check for int or float value
if (is_numeric($value)) {
$tmp_value = intval($value);
if ($value == $tmp_value) {
$value = $tmp_value;
} else {
$tmp_value = floatval($value);
if ($value == $tmp_value) {
$value = $tmp_value;
}
}
}
//
$attributes[$fields[$c]->getId()] = $value;
switch ($fields[$c]->getType()) {
case 'idparent':
$other->setIdparent($value . '');
break;
case 'idmodule':
$other->setIdentifier($value . '');
break;
}
}
$other->setProperty($attributes);
$other->setModule($module);
$parent = null;
if ($module->getParent()) {
$parent_q = $dm->createQueryBuilder('PlantnetDataBundle:Plantunit')->field('module.id')->equals($module->getParent()->getId())->field('identifier')->equals($other->getIdparent())->getQuery()->execute();
foreach ($parent_q as $p) {
$parent = $p;
}
}
if ($parent) {
$other->setPlantunit($parent);
$dm->persist($other);
$rowCount++;
$size++;
if ($size >= $batchSize) {
$dm->flush();
$dm->clear();
gc_collect_cycles();
$module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule);
$size = 0;
}
} else {
$orphans[$other->getIdparent()] = $other->getIdparent();
$errorCount++;
$dm->detach($other);
}
}
$dm->flush();
$dm->clear();
gc_collect_cycles();
$module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule);
}
fclose($handle);
$module->setNbrows($rowCount);
$module->setUpdating(false);
$dm->persist($module);
$dm->flush();
$dm->clear();
$e = microtime(true);
echo ' Inserted ' . $rowCount . ' objects in ' . ($e - $s) . ' seconds' . PHP_EOL;
if (file_exists($csvfile)) {
unlink($csvfile);
}
$message = 'Importation Success: ' . $rowCount . ' objects imported in ' . ($e - $s) . ' seconds';
$message .= "\n";
$message .= 'Importation Error: ' . $errorCount;
if (count($orphans)) {
$message .= "\n" . 'These "id parent" do not exist:' . "\n";
$message .= implode(', ', $orphans);
}
} else {
$message = $error;
}
$message_mail = \Swift_Message::newInstance()->setSubject('Publish : task ended')->setFrom($this->getContainer()->getParameter('from_email_adress'))->setTo($usermail)->setBody($message . $this->getContainer()->get('templating')->render('PlantnetDataBundle:Backend\\Mail:task.txt.twig'));
$this->getContainer()->get('mailer')->send($message_mail);
$spool = $this->getContainer()->get('mailer')->getTransport()->getSpool();
$transport = $this->getContainer()->get('swiftmailer.transport.real');
$spool->flushQueue($transport);
//
$connection = new \MongoClient();
$db = $connection->{$dbname};
$db->Module->update(array('_id' => new \MongoId($idmodule)), array('$set' => array('updating' => false)));
}
}
/**
* Update a document
*
* @param string $collection Name of the collection
* @param array $options Array of update options
*
* @throws Exception
* @access public
* @return boolean
*/
public function update($collection = '', $options = array())
{
if (empty($collection)) {
throw new \MongoQB\Exception('No Mongo collection selected to
update');
}
if (count($this->updates) === 0) {
throw new \MongoQB\Exception('Nothing to update in Mongo collection or
update is not an array');
}
try {
\MongoCursor::$timeout = -1;
$options = array_merge(array($this->_querySafety => true, 'multiple' => false), $options);
$result = $this->_dbhandle->{$collection}->update($this->wheres, $this->updates, $options);
$this->_clear($collection, 'update');
if ($result['updatedExisting'] > 0) {
return $result['updatedExisting'];
}
// @codeCoverageIgnoreStart
return false;
// @codeCoverageIgnoreEnd
} catch (\MongoCursorException $Exception) {
throw new \MongoQB\Exception('Update of data into MongoDB failed: ' . $Exception->getMessage());
// @codeCoverageIgnoreEnd
}
}
private function update_punit($dm, $module, $dbname, $usermail)
{
\MongoCursor::$timeout = -1;
$idmodule = $module->getId();
$this->reset_taxo($dm, $module);
$csvfile = __DIR__ . '/../Resources/uploads/' . $module->getCollection()->getAlias() . '/' . $module->getAlias() . '.csv';
$handle = fopen($csvfile, "r");
$columns = fgetcsv($handle, 0, ";");
$fields = array();
$attributes = $module->getProperties();
foreach ($attributes as $field) {
$fields[] = $field;
}
//Punit identifiers in the csv file
$csv_ids = array();
while (($data = fgetcsv($handle, 0, ';')) !== false) {
$num = count($data);
for ($c = 0; $c < $num; $c++) {
if ($fields[$c]->getType() == 'idmodule') {
$value = trim($this->data_encode($data[$c]));
$csv_ids[] = $value;
}
}
}
fclose($handle);
//get existing ids in database
$db_ids = array();
$punits = $dm->createQueryBuilder('PlantnetDataBundle:Plantunit')->hydrate(false)->select('identifier')->field('module')->references($module)->getQuery()->execute();
foreach ($punits as $id) {
$db_ids[] = $id['identifier'];
}
$punits = null;
unset($punits);
$ids_to_remove = array_diff($db_ids, $csv_ids);
if (count($ids_to_remove)) {
$sub_ids_to_remove = array_chunk($ids_to_remove, 100);
foreach ($sub_ids_to_remove as $tab_ids) {
//delete
//deletes punits where id is not in csv_ids
$dm->createQueryBuilder('PlantnetDataBundle:Plantunit')->remove()->field('identifier')->in($tab_ids)->getQuery()->execute();
//cascade doesnt work !?
$dm->createQueryBuilder('PlantnetDataBundle:Image')->remove()->field('idparent')->in($tab_ids)->getQuery()->execute();
$dm->createQueryBuilder('PlantnetDataBundle:Location')->remove()->field('idparent')->in($tab_ids)->getQuery()->execute();
$dm->createQueryBuilder('PlantnetDataBundle:Other')->remove()->field('idparent')->in($tab_ids)->getQuery()->execute();
}
}
$csv_ids = null;
unset($csv_ids);
$db_ids = null;
unset($db_ids);
$ids_to_remove = null;
unset($ids_to_remove);
//update / create Punits
$handle = fopen($csvfile, "r");
$columns = fgetcsv($handle, 0, ";");
$batchSize = 100;
$size = 0;
while (($data = fgetcsv($handle, 0, ';')) !== false) {
$csv_id = null;
$num = count($data);
for ($c = 0; $c < $num; $c++) {
if ($fields[$c]->getType() == 'idmodule') {
$value = trim($this->data_encode($data[$c]));
$csv_id = $value;
}
}
if ($csv_id) {
$update = true;
$subupdate = false;
//update
//updates punits where id is in csv_ids
$plantunit = $dm->getRepository('PlantnetDataBundle:Plantunit')->findOneBy(array('module.id' => $module->getId(), 'identifier' => $csv_id));
//create
//creates punits where csv_id is not in id
if (!$plantunit) {
$update = false;
$plantunit = new Plantunit();
$plantunit->setModule($module);
}
$attributes = array();
for ($c = 0; $c < $num; $c++) {
$value = trim($this->data_encode($data[$c]));
//check for int or float value
if (is_numeric($value)) {
$tmp_value = intval($value);
if ($value == $tmp_value) {
$value = $tmp_value;
} else {
$tmp_value = floatval($value);
if ($value == $tmp_value) {
$value = $tmp_value;
}
}
}
//
$attributes[$fields[$c]->getId()] = $value;
switch ($fields[$c]->getType()) {
case 'idmodule':
$plantunit->setIdentifier($value . '');
break;
case 'idparent':
$plantunit->setIdparent($value . '');
break;
case 'title1':
if ($update && $plantunit->getTitle1() != $value . '') {
$subupdate = true;
}
$plantunit->setTitle1($value . '');
break;
case 'title2':
if ($update && $plantunit->getTitle2() != $value . '') {
$subupdate = true;
}
$plantunit->setTitle2($value . '');
break;
case 'title3':
if ($update && $plantunit->getTitle3() != $value . '') {
$subupdate = true;
}
$plantunit->setTitle3($value . '');
break;
}
}
$plantunit->setAttributes($attributes);
$dm->persist($plantunit);
$size++;
//
if ($update && $subupdate) {
$dm->createQueryBuilder('PlantnetDataBundle:Image')->update()->multiple(true)->field('plantunit')->references($plantunit)->field('title1')->set($plantunit->getTitle1())->field('title2')->set($plantunit->getTitle2())->field('title3')->set($plantunit->getTitle3())->getQuery()->execute();
$dm->createQueryBuilder('PlantnetDataBundle:Location')->update()->multiple(true)->field('plantunit')->references($plantunit)->field('title1')->set($plantunit->getTitle1())->field('title2')->set($plantunit->getTitle2())->field('title3')->set($plantunit->getTitle3())->getQuery()->execute();
}
//
if ($size >= $batchSize) {
$dm->flush();
$dm->clear();
gc_collect_cycles();
$module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule);
$size = 0;
}
}
}
fclose($handle);
$dm->flush();
$dm->clear();
gc_collect_cycles();
$module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule);
//nb rows module
$count = $dm->createQueryBuilder('PlantnetDataBundle:Plantunit')->field('module')->references($module)->getQuery()->execute()->count();
$module->setNbrows($count);
$dm->persist($module);
//nb rows children
$children = $module->getChildren();
if (count($children)) {
foreach ($children as $child) {
if ($child->getType() == 'image') {
$count = $dm->createQueryBuilder('PlantnetDataBundle:Image')->field('module')->references($child)->getQuery()->execute()->count();
$child->setNbrows($count);
$dm->persist($child);
} elseif ($child->getType() == 'locality') {
$count = $dm->createQueryBuilder('PlantnetDataBundle:Location')->field('module')->references($child)->getQuery()->execute()->count();
$child->setNbrows($count);
$dm->persist($child);
} elseif ($child->getType() == 'other') {
$count = $dm->createQueryBuilder('PlantnetDataBundle:Other')->field('module')->references($child)->getQuery()->execute()->count();
$child->setNbrows($count);
$dm->persist($child);
}
}
}
//flush it !
$dm->flush();
$dm->clear();
gc_collect_cycles();
$module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule);
//
return $module;
}
/**
* @ApiDoc(
* section="Publish v2 - GeoData. Sub-module entity [type = locality]",
* description="Returns geo data from a 'locality' sub-module entity [GeoJson]",
* statusCodes={
* 200="Returned when: Successful",
* 401="Returned when: Unauthorized client",
* 404="Returned when: Resource not found"
* },
* filters={
* {"name"="project", "dataType"="String", "required"=true, "description"="Project url"},
* {"name"="collection", "dataType"="String", "required"=true, "description"="Collection url"},
* {"name"="module", "dataType"="String", "required"=true, "description"="Module url"},
* {"name"="submodule", "dataType"="String", "required"=true, "description"="'Location' Sub-Module url"}
* }
* )
*
* @Route(
* "/{project}/{collection}/{module}/{submodule}/geodata",
* name="api_submodule_geodata"
* )
* @Method("get")
*/
public function api_submodule_geodataAction($project, $collection, $module, $submodule)
{
ini_set('memory_limit', '-1');
//check project
try {
ControllerHelp::check_enable_project($project, $this->get_prefix(), $this);
} catch (\Exception $e) {
$this->return_404_not_found($e->getMessage());
exit;
}
//init
$dm = $this->get('doctrine.odm.mongodb.document_manager');
$dm->getConfiguration()->setDefaultDB($this->get_prefix() . $project);
$result = array();
//get language config
$config = ControllerHelp::get_config($project, $dm, $this);
$this->check_authorized_client($config);
//data1
$collection = $dm->getRepository('PlantnetDataBundle:Collection')->findOneBy(array('url' => $collection));
if (!$collection || $collection->getDeleting() == true) {
$this->return_404_not_found('Unable to find Collection entity.');
exit;
}
$module = $dm->getRepository('PlantnetDataBundle:Module')->findOneBy(array('url' => $module, 'collection.id' => $collection->getId()));
if (!$module || $module->getType() != 'text' || $module->getDeleting() == true) {
$this->return_404_not_found('Unable to find Module entity.');
exit;
}
$submodule = $dm->getRepository('PlantnetDataBundle:Module')->findOneBy(array('url' => $submodule, 'parent.id' => $module->getId(), 'collection.id' => $collection->getId()));
if (!$submodule || $submodule->getType() == 'text' || $submodule->getDeleting() == true) {
$this->return_404_not_found('Unable to find Sub-module entity.');
exit;
}
if ($submodule->getType() != 'locality') {
$this->return_404_not_found('Unable to find geo data for this Sub-module entity.');
exit;
}
//data2
$display = array();
$field = $module->getProperties();
foreach ($field as $row) {
if ($row->getMain() == true) {
$display[] = $row->getId();
}
}
$field_sub = $submodule->getProperties();
$field_sub_tab = array();
foreach ($field_sub as $f) {
if ($f->getDetails() == true) {
$field_sub_tab[$f->getId()] = StringHelp::cleanToKey($f->getName());
}
}
\MongoCursor::$timeout = -1;
$locations = $dm->createQueryBuilder('PlantnetDataBundle:Location')->hydrate(false)->select('title1')->select('title2')->select('title3')->select('property')->select('latitude')->select('longitude')->select('idparent')->field('module')->references($submodule)->getQuery()->execute()->toArray();
array_walk($locations, function (&$item, $key, $field_sub_tab) {
$l = array('type' => 'Feature', 'geometry' => array('type' => 'Point', 'coordinates' => array($item['longitude'], $item['latitude'])), 'properties' => array('title1' => $item['title1'], 'title2' => $item['title2'], 'title3' => $item['title3'], 'parent_identifier' => $item['idparent']));
foreach ($item['property'] as $key => $value) {
if (array_key_exists($key, $field_sub_tab)) {
$l['properties'][$field_sub_tab[$key]] = $value;
}
}
$item = $l;
}, $field_sub_tab);
$locations = array_values($locations);
//response
$response = new Response(json_encode(array('type' => 'FeatureCollection', 'features' => $locations)));
$response->headers->set('Content-Type', 'application/json');
return $response;
}
echo 'Total : ' . sprintf('%01.6f', microtime(true) - $startTime) . ' seconds.<br />';
exit;*/
if (DO_MYSQL) {
$mysqlConnection = mysql_connect('127.0.0.1', 'root', 'password');
mysql_select_db('cps2_perf_test');
}
if (DO_CPS) {
$cpsConnection->sendRequest(new CPS_Request('clear'));
sleep(1);
}
// $cpsConnection->waitForResponse(false);
if (DO_MONGO) {
$m = new Mongo();
$db = $m->test;
$collection = $db->insert_test;
MongoCursor::$timeout = 120000;
// 2 minutes
}
if (defined('RANDOM_SEED')) {
srand(RANDOM_SEED);
echo 'Seeded with random seed ' . RANDOM_SEED . ', first random value: ' . rand() . '<br />';
}
$randomWords = array();
$wordChars = '0123456789abcdefghijklmnopqrstuvwxyz';
$charCount = strlen($wordChars);
define('WORD_COUNT', 100000);
for ($x = 0; $x < WORD_COUNT; ++$x) {
$size = rand(3, 9);
$word = '';
for ($y = 0; $y < $size; ++$y) {
$word .= $wordChars[rand(0, $charCount - 1)];
public static function group($collection, $keys = [], $query = [], $count = [], $sum = [], $sort = [], $limit = null)
{
global $debug;
// Turn keys into an array if is isn't already an array
if (!is_array($keys)) {
$keys = [$keys];
}
// Start the aggregation pipeline with the query
$pipeline = [];
if (sizeof($query)) {
$pipeline[] = ['$match' => $query];
}
// Create the group by using the given key(s)
$ids = [];
foreach ($keys as $key => $value) {
if (is_numeric($key)) {
$ids[] = '$' . $value;
} else {
$ids[$key] = ['$' . $key => '$' . $value];
}
}
if (sizeof($ids) == 1 && isset($ids[0])) {
$ids = $ids[0];
}
$group = [];
$group['_id'] = $ids;
// If no counts or sums are given, assume a count based on the keys for the $group
if (sizeof($count) == 0 && sizeof($sum) == 0) {
$group['count'] = ['$sum' => 1];
}
// Include counts in the $group
if (!is_array($count)) {
$count = [$count];
}
foreach ($count as $s) {
$group[str_replace('.', '_', $s) . 'Count'] = ['$sum' => 1];
}
// Include sums in the $group
if (!is_array($sum)) {
$sum = [$sum];
}
foreach ($sum as $s) {
$group[str_replace('.', '_', $s) . 'Sum'] = ['$sum' => '$' . $s];
}
// Add the group to the pipeline
$pipeline[] = ['$group' => $group];
// $project the keys into the result
$project = [];
$project['_id'] = 0;
foreach ($keys as $key => $value) {
if (is_numeric($key)) {
$project[$value] = '$_id';
} else {
$project[$key] = '$_id.' . $key;
}
}
if (sizeof($count) == 0 && sizeof($sum) == 0) {
$project['count'] = 1;
}
if (sizeof($count) > 0) {
foreach ($count as $s) {
$project[str_replace('.', '_', $s) . 'Count'] = 1;
}
}
if (sizeof($sum) > 0) {
foreach ($sum as $s) {
$project[str_replace('.', '_', $s) . 'Sum'] = 1;
}
}
$pipeline[] = ['$project' => $project];
// Assign the sort to the pipeline
if (sizeof($sort) > 0) {
$pipeline[] = ['$sort' => $sort];
}
// And add the limit
if ($limit != null) {
$pipeline[] = ['$limit' => (int) $limit];
}
// Prep the cursor
$mdb = new self();
$collection = $mdb->getCollection($collection);
if (!$debug) {
MongoCursor::$timeout = -1;
}
// this should be deprecated but aggregate doesn't have a timeout
// Execute the query
$result = $collection->aggregate($pipeline);
if ($result['ok'] == 1) {
return $result['result'];
}
throw new Exception('pipeline query failure');
}
/**
* setTimeout Method
*
* Sets the MongoCursor timeout so long queries (like map / reduce) can run at will.
* Expressed in milliseconds, for an infinite timeout, set to -1
*
* @param int $ms
* @return boolean
* @access public
*/
public function setTimeout($ms) {
MongoCursor::$timeout = $ms;
return true;
}
/**
* distinct查询
* @param $field
* @param $conds
* @return array
* @throws Exception
*/
public function Distinct($field, $conds)
{
$callback = function ($collection) use($field, $conds) {
return $collection->Distinct($field, $conds);
};
try {
MongoCursor::$timeout = 120000;
$cursor = $this->_doAction($callback);
//$cursor->timeout(120000);
} catch (Exception $ex) {
throw new Exception(__METHOD__ . ' Distinct fail! Exception code: ' . $ex->getCode() . ' msg: ' . $ex->getMessage(), self::ERRNO_FINDONE_FAIL);
}
$ret = $this->UnCompress($cursor);
return $ret;
}
private function delete_glossary($dbname, $id)
{
$error = '';
$dm = $this->getContainer()->get('doctrine.odm.mongodb.document_manager');
$dm->getConfiguration()->setDefaultDB($dbname);
$configuration = $dm->getConnection()->getConfiguration();
$configuration->setLoggerCallable(null);
\MongoCursor::$timeout = -1;
$glossary = $dm->getRepository('PlantnetDataBundle:Glossary')->findOneBy(array('id' => $id));
if (!$glossary) {
$error = 'Unable to find Glossary entity.';
}
/*
* Remove csv directory (and files)
*/
$file = __DIR__ . '/../Resources/uploads/' . $glossary->getCollection()->getAlias() . '/glossary.csv';
if (file_exists($file)) {
unlink($file);
}
$file = __DIR__ . '/../Resources/uploads/' . $glossary->getCollection()->getAlias() . '/glossary_syn.csv';
if (file_exists($file)) {
unlink($file);
}
/*
* Remove upload directory
*/
$dir = $glossary->getUploaddir();
$del_dir = true;
$nb_equal_dirs = 0;
$config = $dm->createQueryBuilder('PlantnetDataBundle:Config')->getQuery()->getSingleResult();
if ($config) {
$original_db = $config->getOriginaldb();
if ($original_db) {
$connection = new \MongoClient();
$dbs_array = array();
$dbs = $connection->admin->command(array('listDatabases' => 1));
foreach ($dbs['databases'] as $db) {
$db_name = $db['name'];
if (substr_count($db_name, $original_db)) {
$dbs_array[] = $db_name;
}
}
if (count($dbs_array) > 1) {
foreach ($dbs_array as $chk_db) {
$nb_equal_dirs += $connection->{$chk_db}->Glossary->find(array('uploaddir' => $dir))->count();
}
}
}
}
if ($nb_equal_dirs > 1) {
$del_dir = false;
}
if ($dir && $del_dir) {
$dir = __DIR__ . '/../../../../web/uploads/' . $dir;
if (file_exists($dir) && is_dir($dir)) {
$files = scandir($dir);
foreach ($files as $file) {
if ($file != '.' && $file != '..') {
unlink($dir . '/' . $file);
}
}
rmdir($dir);
}
}
/*
* Remove Glossary + Cascade
*/
$dm->remove($glossary);
$dm->flush();
$dm->detach($glossary);
}
/**
* 设置一个超时
* @param int $ms
* @return muMongoCursor
*/
public function timeout($ms)
{
$this->oMongoCursor->timeout($ms);
return $this;
}
private function glossarize($action, $dbname, $id_glossary, $usermail)
{
$error = '';
$dm = $this->getContainer()->get('doctrine.odm.mongodb.document_manager');
$dm->getConfiguration()->setDefaultDB($dbname);
$configuration = $dm->getConnection()->getConfiguration();
$configuration->setLoggerCallable(null);
$glossary = $dm->getRepository('PlantnetDataBundle:Glossary')->findOneBy(array('id' => $id_glossary));
if (!$glossary) {
$error = 'Unable to find Glossary entity.';
}
$collection = $glossary->getCollection();
if (!$collection) {
$error = 'Unable to find Glossary entity.';
}
//
if ($action == 'syn') {
$csv = __DIR__ . '/../Resources/uploads/' . $collection->getAlias() . '/glossary_syn.csv';
if (file_exists($csv)) {
$handle = fopen($csv, "r");
$field = fgetcsv($handle, 0, ";");
$csv_error = false;
if (!isset($field[0]) || empty($field[0])) {
$csv_error = true;
}
if (!isset($field[1]) || empty($field[1])) {
$csv_error = true;
}
if (!$csv_error) {
$i = 0;
$batch_size = 100;
\MongoCursor::$timeout = -1;
while (($data = fgetcsv($handle, 0, ';')) !== false) {
$def_word = isset($data[0]) ? trim($data[0]) : '';
$cur_encoding = mb_detect_encoding($def_word);
if ($cur_encoding == "UTF-8" && mb_check_encoding($def_word, "UTF-8")) {
$def_word = $def_word;
} else {
$def_word = utf8_encode($def_word);
}
$syn_word = isset($data[1]) ? trim($data[1]) : '';
$cur_encoding = mb_detect_encoding($syn_word);
if ($cur_encoding == "UTF-8" && mb_check_encoding($syn_word, "UTF-8")) {
$syn_word = $syn_word;
} else {
$syn_word = utf8_encode($syn_word);
}
if ($def_word && $syn_word) {
$definition = $dm->getRepository('PlantnetDataBundle:Definition')->findOneBy(array('glossary.id' => $glossary->getId(), 'name' => $def_word));
if ($definition) {
$i++;
if (!$definition->getHaschildren()) {
$definition->setHaschildren(true);
$dm->persist($definition);
}
$synonym = new Definition();
$synonym->setGlossary($glossary);
$synonym->setParent($definition);
$synonym->setName($syn_word);
$synonym->setDisplayedname($syn_word);
$synonym->setDefinition($definition->getDefinition());
$synonym->setPath($definition->getPath());
$dm->persist($synonym);
if ($i >= $batch_size) {
$dm->flush();
$i = 0;
$dm->clear();
gc_collect_cycles();
$glossary = $dm->getRepository('PlantnetDataBundle:Glossary')->findOneBy(array('id' => $id_glossary));
}
}
}
}
$dm->flush();
$dm->clear();
gc_collect_cycles();
$glossary = $dm->getRepository('PlantnetDataBundle:Glossary')->findOneBy(array('id' => $id_glossary));
} else {
$error = 'Error in Glossary Syn file.';
}
} else {
$error = 'Unable to find Glossary Syn file.';
}
$message = $error;
if (empty($message)) {
$message = 'Synonyms were created successfully.';
}
$message_mail = \Swift_Message::newInstance()->setSubject('Publish : task ended')->setFrom($this->getContainer()->getParameter('from_email_adress'))->setTo($usermail)->setBody($message . $this->getContainer()->get('templating')->render('PlantnetDataBundle:Backend\\Mail:task.txt.twig'));
$this->getContainer()->get('mailer')->send($message_mail);
$spool = $this->getContainer()->get('mailer')->getTransport()->getSpool();
$transport = $this->getContainer()->get('swiftmailer.transport.real');
$spool->flushQueue($transport);
} elseif ($action == 'unsyn') {
$dm->createQueryBuilder('PlantnetDataBundle:Definition')->remove()->field('glossary')->references($glossary)->field('parent')->notEqual(null)->getQuery()->execute();
}
}
/**
* Return result set for current query
*/
public function toCollection(Spot_Query $query, MongoCursor $cursor)
{
$mapper = $query->mapper();
$entityClass = $query->entityName();
if ($cursor instanceof MongoCursor) {
$results = array();
$resultsIdentities = array();
// Set timeout
if (isset($this->options['cursor']['timeout']) && is_int($this->options['cursor']['timeout'])) {
$cursor->timeout($this->options['cursor']['timeout']);
}
// Fetch all results into new entity class
// @todo Move this to collection class so entities will be lazy-loaded by Collection iteration
foreach ($cursor as $document) {
$entity = new $entityClass();
$mapper->data($entity, $document);
// Store in array for Collection
$results[] = $entity;
// Store primary key of each unique record in set
$pk = $mapper->primaryKey($entity);
if (!in_array($pk, $resultsIdentities) && !empty($pk)) {
$resultsIdentities[] = $pk;
}
}
$collectionClass = $mapper->collectionClass();
return new $collectionClass($results, $resultsIdentities);
} else {
$mapper->addError(__METHOD__ . " - Unable to execute query - not a valid MongoCursor");
return array();
}
}
/**
* Return result set for current query
*/
public function toCollection(\Spot\Query $query, \MongoCursor $cursor)
{
$mapper = $query->mapper();
$entityClass = $query->entityName();
if ($cursor instanceof MongoCursor) {
// Set timeout
if (isset($this->options['cursor']['timeout']) && is_int($this->options['cursor']['timeout'])) {
$cursor->timeout($this->options['cursor']['timeout']);
}
return $mapper->collection($entityClass, $cursor);
} else {
$mapper->addError(__METHOD__ . " - Unable to execute query - not a valid MongoCursor");
return array();
}
}
<?php
$connection = new MongoClient();
$collection = $connection->selectDB("getglue")->selectCollection("gg");
MongoCursor::$timeout = -1;
$match1 = array('$match' => array('comment' => array('$ne' => '')));
$group = array('$group' => array('_id' => '$userId', 'total' => array('$sum' => 1)));
$match2 = array('$match' => array('total' => array('$gte' => 5000)));
$sort = array('$sort' => array('total' => -1));
$limit = array('$limit' => 7);
$pipeline = array($match1, $group, $match2, $sort, $limit);
$out = $collection->aggregate($pipeline);
echo json_encode($out, JSON_PRETTY_PRINT);
private function taxonomize($action, $dbname, $id_module, $usermail, $clean)
{
$error = '';
$dm = $this->getContainer()->get('doctrine.odm.mongodb.document_manager');
$dm->getConfiguration()->setDefaultDB($dbname);
$configuration = $dm->getConnection()->getConfiguration();
$configuration->setLoggerCallable(null);
$module = $dm->getRepository('PlantnetDataBundle:Module')->findOneBy(array('id' => $id_module));
if (!$module) {
$error = 'Unable to find Module entity.';
}
if ($clean) {
$csv = __DIR__ . '/../Resources/uploads/' . $module->getCollection()->getAlias() . '/' . $module->getAlias() . '_syn.csv';
if (file_exists($csv)) {
unlink($csv);
}
$csv = __DIR__ . '/../Resources/uploads/' . $module->getCollection()->getAlias() . '/' . $module->getAlias() . '_desc.csv';
if (file_exists($csv)) {
unlink($csv);
}
}
if ($action == 'taxo') {
\MongoCursor::$timeout = -1;
// remove old taxa' refs
$dm->createQueryBuilder('PlantnetDataBundle:Plantunit')->update()->multiple(true)->field('module')->references($module)->field('taxon')->unsetField()->field('taxonsrefs')->unsetField()->getQuery()->execute();
$sub_modules = $module->getChildren();
foreach ($sub_modules as $sub) {
if ($sub->getType() == 'image') {
$dm->createQueryBuilder('PlantnetDataBundle:Image')->update()->multiple(true)->field('module')->references($sub)->field('taxonsrefs')->unsetField()->getQuery()->execute();
} elseif ($sub->getType() == 'locality') {
$dm->createQueryBuilder('PlantnetDataBundle:Location')->update()->multiple(true)->field('module')->references($sub)->field('taxonsrefs')->unsetField()->getQuery()->execute();
}
}
// remove old taxa
$dm->createQueryBuilder('PlantnetDataBundle:Taxon')->remove()->field('module')->references($module)->getQuery()->execute();
}
// load taxonomy data
$taxo = array();
$fields = $module->getProperties();
foreach ($fields as $field) {
if ($field->getTaxolevel()) {
$taxo[$field->getTaxolevel()] = array($field->getId(), $field->getTaxolabel());
}
}
$fields = null;
if (count($taxo)) {
ksort($taxo);
$first_level = key($taxo);
end($taxo);
$last_level = key($taxo);
reset($taxo);
if ($action == 'taxo') {
$connection = new \MongoClient();
$db = $connection->{$dbname};
//populate
$tab_tax = $this->populate($db, $module, $taxo);
//set indexes
if (count($this->tmp_indexes)) {
foreach ($this->tmp_indexes as $i_name => $i_index) {
$db->Plantunit->ensureIndex($i_index, array('name' => $i_name));
}
}
//save
\MongoCursor::$timeout = -1;
$this->save($db, $dbname, $dm, $module, $taxo, $tab_tax);
$tab_tax = null;
unset($tab_tax);
$dm->clear();
gc_collect_cycles();
$module = $dm->getRepository('PlantnetDataBundle:Module')->findOneBy(array('id' => $id_module));
//unset indexes
if (count($this->tmp_indexes)) {
foreach ($this->tmp_indexes as $i_name => $i_index) {
$db->command(array('deleteIndexes' => 'Plantunit', 'index' => $i_name));
}
}
//free
$db = null;
$connection = null;
} elseif ($action == 'syn') {
// synonymy management
$csv = __DIR__ . '/../Resources/uploads/' . $module->getCollection()->getAlias() . '/' . $module->getAlias() . '_syn.csv';
if (file_exists($csv)) {
$cols = array();
$syns = array();
$handle = fopen($csv, "r");
$field = fgetcsv($handle, 0, ";");
foreach ($field as $col) {
$col_name = '';
$cur_encoding = mb_detect_encoding($col);
if ($cur_encoding == "UTF-8" && mb_check_encoding($col, "UTF-8")) {
$col_name = $col;
} else {
$col_name = utf8_encode($col);
}
$cols[] = $col_name;
}
$csv_error = false;
foreach ($taxo as $level => $data) {
$syns[$level] = array('col_non_valid' => '', 'col_valid' => '');
foreach ($cols as $key => $col) {
if (empty($data[1])) {
$csv_error = true;
} else {
if ($col == $data[1]) {
$syns[$level]['col_non_valid'] = $key;
}
if ($col == $data[1] . '_valid') {
$syns[$level]['col_valid'] = $key;
}
}
}
if (empty($syns[$level]['col_non_valid']) && $syns[$level]['col_non_valid'] != 0) {
$csv_error = true;
}
if (empty($syns[$level]['col_valid']) && $syns[$level]['col_valid'] != 0) {
$csv_error = true;
}
}
if (!$csv_error) {
$connection = new \MongoClient();
$db = $connection->{$dbname};
\MongoCursor::$timeout = -1;
while (($data = fgetcsv($handle, 0, ';')) !== false) {
//last non null
$last_non_null_rank_non_valid = 0;
$last_non_null_rank_valid = 0;
foreach ($syns as $level => $tab) {
$checked_value_non_valid = isset($data[$tab['col_non_valid']]) ? trim($data[$tab['col_non_valid']]) : '';
$checked_value_valid = isset($data[$tab['col_valid']]) ? trim($data[$tab['col_valid']]) : '';
if (!empty($checked_value_non_valid)) {
$last_non_null_rank_non_valid = $level;
}
if (!empty($checked_value_valid)) {
$last_non_null_rank_valid = $level;
}
}
//data
$non_valid_identifier = '';
$valid_identifier = '';
$non_valid = array();
$valid = array();
foreach ($syns as $level => $tab) {
$string_non_valid = isset($data[$tab['col_non_valid']]) ? trim($data[$tab['col_non_valid']]) : '';
if (empty($string_non_valid) && $level < $last_non_null_rank_non_valid) {
$string_non_valid = '-';
}
$string_valid = isset($data[$tab['col_valid']]) ? trim($data[$tab['col_valid']]) : '';
if (empty($string_valid) && $level < $last_non_null_rank_valid) {
$string_valid = '-';
}
$cur_encoding = mb_detect_encoding($string_non_valid);
if ($cur_encoding == "UTF-8" && mb_check_encoding($string_non_valid, "UTF-8")) {
$string_non_valid = $string_non_valid;
} else {
$string_non_valid = utf8_encode($string_non_valid);
}
$cur_encoding = mb_detect_encoding($string_valid);
if ($cur_encoding == "UTF-8" && mb_check_encoding($string_valid, "UTF-8")) {
$string_valid = $string_valid;
} else {
$string_valid = utf8_encode($string_valid);
}
if ($non_valid_identifier && !empty($string_non_valid)) {
$non_valid_identifier .= ' - ';
}
$non_valid_identifier .= $string_non_valid;
if ($valid_identifier && !empty($string_valid)) {
$valid_identifier .= ' - ';
}
$valid_identifier .= $string_valid;
$non_valid[$level] = null;
if (!empty($string_non_valid)) {
$non_valid[$level] = array('name' => $string_non_valid, 'identifier' => $non_valid_identifier, 'level' => $level, 'label' => $cols[$tab['col_non_valid']]);
}
$valid[$level] = null;
if (!empty($string_valid)) {
$valid[$level] = array('name' => $string_valid, 'identifier' => $valid_identifier, 'level' => $level, 'label' => $cols[$tab['col_non_valid']]);
}
}
$last_non_valid = null;
foreach ($non_valid as $lvl => $check_taxon) {
if ($check_taxon) {
$tax = $dm->getRepository('PlantnetDataBundle:Taxon')->findOneBy(array('module.id' => $module->getId(), 'identifier' => $check_taxon['identifier']));
if (!$tax) {
$tax = new Taxon();
$tax->setIdentifier($check_taxon['identifier']);
$tax->setName($check_taxon['name']);
$tax->setLabel($check_taxon['label']);
$tax->setLevel($check_taxon['level']);
$tax->setModule($module);
$tax->setNbpunits(0);
$tax->setIssynonym(false);
if ($last_non_valid) {
$tax->setParent($last_non_valid);
if ($last_non_valid->getHaschildren() != true) {
$last_non_valid->setHaschildren(true);
$dm->persist($last_non_valid);
}
}
$dm->persist($tax);
$dm->flush();
}
$last_non_valid = $tax;
}
}
$last_valid = null;
foreach ($valid as $lvl => $check_taxon) {
if ($check_taxon) {
$tax = $dm->getRepository('PlantnetDataBundle:Taxon')->findOneBy(array('module.id' => $module->getId(), 'identifier' => $check_taxon['identifier']));
if (!$tax) {
$tax = new Taxon();
$tax->setIdentifier($check_taxon['identifier']);
$tax->setName($check_taxon['name']);
$tax->setLabel($check_taxon['label']);
$tax->setLevel($check_taxon['level']);
$tax->setModule($module);
$tax->setNbpunits(0);
$tax->setIssynonym(false);
if ($last_valid) {
$tax->setParent($last_valid);
if ($last_valid->getHaschildren() != true) {
$last_valid->setHaschildren(true);
$dm->persist($last_valid);
}
}
$dm->persist($tax);
$dm->flush();
}
$last_valid = $tax;
}
}
$exists = false;
if ($last_non_valid->getIssynonym() && $last_non_valid->getChosen()) {
if ($last_non_valid->getChosen()->getId() == $last_valid->getId()) {
$exists = true;
}
}
if (!$exists && $last_valid && $last_non_valid && $last_valid->getIdentifier() != $last_non_valid->getIdentifier()) {
$last_non_valid->setIssynonym(true);
$last_non_valid->setChosen($last_valid);
if ($last_non_valid->getHaschildren()) {
$last_valid->setHaschildren(true);
}
$last_valid->setHassynonyms(true);
//
$has_images = $last_non_valid->getHasimages();
$has_locations = $last_non_valid->getHaslocations();
$nb_to_switch = $last_non_valid->getNbpunits();
$parent_tmp = $last_non_valid->getParent();
while ($parent_tmp) {
$parent_tmp->setNbpunits($parent_tmp->getNbpunits() - $nb_to_switch);
$dm->persist($parent_tmp);
$parent_tmp = $parent_tmp->getParent();
}
$last_valid->setNbpunits($last_valid->getNbpunits() + $nb_to_switch);
if ($has_images) {
$last_valid->setHasimages(true);
}
if ($has_locations) {
$last_valid->setHaslocations(true);
}
$dm->persist($last_valid);
$parent_tmp = $last_valid->getParent();
while ($parent_tmp) {
$parent_tmp->setNbpunits($parent_tmp->getNbpunits() + $nb_to_switch);
if ($has_images) {
$parent_tmp->setHasimages(true);
}
if ($has_locations) {
$parent_tmp->setHaslocations(true);
}
$dm->persist($parent_tmp);
$parent_tmp = $parent_tmp->getParent();
}
$dm->persist($last_non_valid);
$dm->persist($last_valid);
$dm->flush();
$db->Plantunit->update(array('module.$id' => new \MongoId($module->getId()), 'taxonsrefs' => array('$elemMatch' => array('$id' => new \MongoId($last_non_valid->getId())))), array('$addToSet' => array('taxonsrefs' => array('$ref' => 'Taxon', '$id' => new \MongoId($last_valid->getId()), '$db' => $dbname))), array('multiple' => true));
}
$dm->clear();
gc_collect_cycles();
$module = $dm->getRepository('PlantnetDataBundle:Module')->findOneBy(array('id' => $id_module));
}
} else {
$error = 'CSV file contains some errors...';
}
}
} elseif ($action == 'desc') {
// description management
$csv = __DIR__ . '/../Resources/uploads/' . $module->getCollection()->getAlias() . '/' . $module->getAlias() . '_desc.csv';
if (file_exists($csv)) {
$cols = array();
$taxa = array();
$desc = array();
$handle = fopen($csv, "r");
$field = fgetcsv($handle, 0, ";");
foreach ($field as $col) {
$col_name = '';
$cur_encoding = mb_detect_encoding($col);
if ($cur_encoding == "UTF-8" && mb_check_encoding($col, "UTF-8")) {
$col_name = $col;
} else {
$col_name = utf8_encode($col);
}
$cols[] = $col_name;
}
$csv_error = false;
$used_key = array();
foreach ($taxo as $level => $data) {
$taxa[$level] = array('col' => '');
foreach ($cols as $key => $col) {
if ($col == $data[1]) {
$taxa[$level]['col'] = $key;
$used_key[] = $key;
}
}
if (empty($taxa[$level]['col']) && $taxa[$level]['col'] != 0) {
$csv_error = true;
}
}
foreach ($cols as $key => $col) {
if (!in_array($key, $used_key)) {
$desc[$key] = $col;
}
}
$used_key = null;
if (!count($desc)) {
$csv_error = true;
}
if (!$csv_error) {
$i = 0;
$batch_size = 100;
\MongoCursor::$timeout = -1;
while (($data = fgetcsv($handle, 0, ';')) !== false) {
$identifier = '';
$tax = null;
foreach ($taxa as $level => $tab) {
$string = isset($data[$tab['col']]) ? trim($data[$tab['col']]) : '';
$cur_encoding = mb_detect_encoding($string);
if ($cur_encoding == "UTF-8" && mb_check_encoding($string, "UTF-8")) {
$string = $string;
} else {
$string = utf8_encode($string);
}
if ($identifier && !empty($string)) {
$identifier .= ' - ';
}
$identifier .= $string;
if (!empty($string)) {
$tax = array('name' => $string, 'identifier' => $identifier, 'level' => $level, 'label' => $cols[$tab['col']]);
}
}
if ($tax) {
$description = array();
$nb = 0;
foreach ($desc as $key => $val) {
$string = isset($data[$key]) ? trim($data[$key]) : '';
$cur_encoding = mb_detect_encoding($string);
if ($cur_encoding == "UTF-8" && mb_check_encoding($string, "UTF-8")) {
$string = $string;
} else {
$string = utf8_encode($string);
}
if (!empty($string)) {
if (empty($val)) {
$val = $nb + '';
$nb++;
}
$description[$val] = $string;
}
}
$tax['description'] = $description;
$taxon = $dm->getRepository('PlantnetDataBundle:Taxon')->findOneBy(array('module.id' => $module->getId(), 'identifier' => $tax['identifier']));
if ($taxon) {
$i++;
$taxon->setAttributes($tax['description']);
$dm->persist($taxon);
if ($i >= $batch_size) {
$dm->flush();
$i = 0;
$dm->clear();
gc_collect_cycles();
$module = $dm->getRepository('PlantnetDataBundle:Module')->findOneBy(array('id' => $id_module));
}
}
}
}
$dm->flush();
$dm->clear();
gc_collect_cycles();
$module = $dm->getRepository('PlantnetDataBundle:Module')->findOneBy(array('id' => $id_module));
} else {
$error = 'CSV file contains some errors...';
}
}
} elseif ($action == 'undesc') {
\MongoCursor::$timeout = -1;
// remove taxa' descriptions
$dm->createQueryBuilder('PlantnetDataBundle:Taxon')->update()->multiple(true)->field('module')->references($module)->field('attributes')->unsetField()->getQuery()->execute();
}
$message = $error;
if (empty($message)) {
$message = 'Taxa were created successfully.';
}
$message_mail = \Swift_Message::newInstance()->setSubject('Publish : task ended')->setFrom($this->getContainer()->getParameter('from_email_adress'))->setTo($usermail)->setBody($message . $this->getContainer()->get('templating')->render('PlantnetDataBundle:Backend\\Mail:task.txt.twig'));
$this->getContainer()->get('mailer')->send($message_mail);
$spool = $this->getContainer()->get('mailer')->getTransport()->getSpool();
$transport = $this->getContainer()->get('swiftmailer.transport.real');
$spool->flushQueue($transport);
}
$connection = new \MongoClient();
$db = $connection->{$dbname};
$db->Module->update(array('_id' => new \MongoId($id_module)), array('$set' => array('updating' => false)));
}
public function auth($username, $password, $db = "admin")
{
if ($db === "") {
if (!$this->_mongoAuth && $this->_mongoDb) {
$db = $this->_mongoDb;
} else {
$db = "admin";
}
}
$server = $this->_mongoHost . ":" . $this->_mongoPort;
if (!$this->_mongoPort) {
$server = $this->_mongoHost;
}
try {
$options = $this->_mongoOptions;
if ($this->_mongoAuth) {
$options["username"] = $username;
$options["password"] = $password;
$options["db"] = $db;
}
$this->_mongo = new RMongo($server, $options);
$this->_mongo->setSlaveOkay(true);
} catch (Exception $e) {
if (preg_match("/authenticate/i", $e->getMessage())) {
return false;
}
echo "Unable to connect MongoDB, please check your configurations. MongoDB said:" . $e->getMessage() . ".";
exit;
}
// changing timeout to the new value
MongoCursor::$timeout = $this->_mongoTimeout;
//auth by mongo
if ($this->_mongoAuth) {
// "authenticate" can only be used between 1.0.1 - 1.2.11
if (RMongo::compareVersion("1.0.1") >= 0 && RMongo::compareVersion("1.2.11") < 0) {
$dbs = $db;
if (!is_array($dbs)) {
$dbs = preg_split("/\\s*,\\s*/", $dbs);
}
foreach ($dbs as $db) {
$ret = $this->_mongo->selectDb($db)->authenticate($username, $password);
if (!$ret["ok"]) {
return false;
}
}
}
} else {
if ($this->_controlAuth) {
if (!isset($this->_controlUsers[$username]) || $this->_controlUsers[$username] != $password) {
return false;
}
//authenticate
if (!empty($this->_mongoUser)) {
// "authenticate" can only be used between 1.0.1 - 1.2.11
if (RMongo::compareVersion("1.0.1") >= 0 && RMongo::compareVersion("1.2.11") < 0) {
return $this->_mongo->selectDB($db)->authenticate($this->_mongoUser, $this->_mongoPass);
}
}
} else {
//authenticate
if (!empty($this->_mongoUser)) {
// "authenticate" can only be used between 1.0.1 - 1.2.11
if (RMongo::compareVersion("1.0.1") >= 0 && RMongo::compareVersion("1.2.11") < 0) {
return $this->_mongo->selectDB($db)->authenticate($this->_mongoUser, $this->_mongoPass);
}
}
}
}
return true;
}
public function timeout($ms)
{
parent::timeout($ms);
return $this;
}
/**
* Wrapper method for MongoCursor::timeout().
*
* @see http://php.net/manual/en/mongocursor.timeout.php
* @param integer $ms
* @return self
*/
public function timeout($ms)
{
$this->timeout = (int) $ms;
$this->mongoCursor->timeout($ms);
return $this;
}
public function auth($username, $password, $db = "admin")
{
if ($db === "") {
if (!$this->_mongoAuth && $this->_mongoDb) {
$db = $this->_mongoDb;
} else {
$db = "admin";
}
}
$server = $this->_mongoHost . ":" . $this->_mongoPort;
if (!$this->_mongoPort) {
$server = $this->_mongoHost;
}
try {
$this->_mongo = new Mongo($server, $this->_mongoOptions);
$this->_mongo->setSlaveOkay(true);
} catch (Exception $e) {
echo "Unable to connect MongoDB, please check your configurations. MongoDB said:" . $e->getMessage() . ".";
exit;
}
// changing timeout to the new value
MongoCursor::$timeout = $this->_mongoTimeout;
//auth by mongo
if ($this->_mongoAuth) {
$dbs = $db;
if (!is_array($dbs)) {
$dbs = preg_split("/\\s*,\\s*/", $dbs);
}
foreach ($dbs as $db) {
$ret = $this->_mongo->selectDb($db)->authenticate($username, $password);
if (!$ret["ok"]) {
return false;
}
}
} else {
if ($this->_controlAuth) {
if (!isset($this->_controlUsers[$username]) || $this->_controlUsers[$username] != $password) {
return false;
}
//authenticate
if (!empty($this->_mongoUser)) {
return $this->_mongo->selectDB($db)->authenticate($this->_mongoUser, $this->_mongoPass);
}
} else {
//authenticate
if (!empty($this->_mongoUser)) {
return $this->_mongo->selectDB($db)->authenticate($this->_mongoUser, $this->_mongoPass);
}
}
}
return true;
}
/**
* Constructor
*/
public function __construct()
{
global $configArray;
global $basePath;
$this->dataSourceSettings = parse_ini_file("{$basePath}/conf/datasources.ini", true);
$this->formats = parse_ini_file("{$basePath}/conf/{$configArray['OAI-PMH']['format_definitions']}", true);
$this->sets = parse_ini_file("{$basePath}/conf/{$configArray['OAI-PMH']['set_definitions']}", true);
date_default_timezone_set($configArray['Site']['timezone']);
$this->log = new Logger();
$mongo = new MongoClient($configArray['Mongo']['url']);
$this->db = $mongo->selectDB($configArray['Mongo']['database']);
MongoCursor::$timeout = isset($configArray['Mongo']['cursor_timeout']) ? $configArray['Mongo']['cursor_timeout'] : 300000;
}
public function setTimeout($timeout)
{
if ($this->_db->compareClientVersion('1.5.3', '<')) {
@(MongoCursor::$timeout = (int) $timeout);
} else {
// see bugs:
// https://jira.mongodb.org/browse/PHP-1099
// https://jira.mongodb.org/browse/PHP-1080
}
}
public function setTimeout($timeout)
{
MongoCursor::$timeout = (int) $timeout;
}
public static function getDistinctCount($groupByColumn, $parameters = [])
{
global $mdb, $debug, $longQueryMS;
$hashKey = "distinctCount::{$groupByColumn}:" . serialize($parameters);
$result = RedisCache::get($hashKey);
if ($result != null) {
return $result;
}
if ($parameters == []) {
$type = $groupByColumn == 'solarSystemID' || $groupByColumn == 'regionID' ? "system.{$groupByColumn}" : "involved.{$groupByColumn}";
$result = $mdb->getCollection('oneWeek')->distinct($type);
RedisCache::set($hashKey, sizeof($result), 3600);
return sizeof($result);
}
$query = MongoFilter::buildQuery($parameters);
if (!$mdb->exists('oneWeek', $query)) {
return [];
}
$andQuery = MongoFilter::buildQuery($parameters, false);
if ($groupByColumn == 'solarSystemID' || $groupByColumn == 'regionID') {
$keyField = "system.{$groupByColumn}";
} else {
$keyField = "involved.{$groupByColumn}";
}
$id = $type = null;
if ($groupByColumn == 'solarSystemID' || $groupByColumn == 'regionID') {
$type = "system.{$groupByColumn}";
}
if ($type == null) {
$type = "involved.{$groupByColumn}";
}
$timer = new Timer();
$pipeline = [];
$pipeline[] = ['$match' => $query];
if ($groupByColumn != 'solarSystemID' && $groupByColumn != 'regionID') {
$pipeline[] = ['$unwind' => '$involved'];
}
if ($type != null && $id != null) {
$pipeline[] = ['$match' => [$type => $id]];
}
$pipeline[] = ['$match' => [$keyField => ['$ne' => null]]];
$pipeline[] = ['$match' => $andQuery];
$pipeline[] = ['$group' => ['_id' => '$' . $type, 'foo' => ['$sum' => 1]]];
$pipeline[] = ['$group' => ['_id' => 'total', 'value' => ['$sum' => 1]]];
if (!$debug) {
MongoCursor::$timeout = -1;
}
$result = $mdb->getCollection('oneWeek')->aggregateCursor($pipeline);
$result = iterator_to_array($result);
$time = $timer->stop();
if ($time > $longQueryMS) {
Log::log("Distinct Long query ({$time}ms): {$hashKey}");
}
$retValue = sizeof($result) == 0 ? 0 : $result[0]['value'];
RedisCache::set($hashKey, $retValue, 3600);
return $retValue;
}
