public function execute(InputInterface $input, OutputInterface $output) { $mongoDbName = $input->getArgument('mongo-db'); $mongoCollectionName = $input->getArgument('mongo-collection'); $mongoHost = $input->getArgument('mongo-host'); $mongoPort = $input->getArgument('mongo-port'); $mongoUserName = $input->getOption('mongo-user'); $mongoPassword = $input->getOption('mongo-password'); $options = array(); if (!empty($mongoUserName)) { $options['username'] = $mongoUserName; } if (!empty($mongoPassword)) { $options['password'] = $mongoPassword; } $m = new MongoClient("mongodb://{$mongoHost}:{$mongoPort}", $options); $db = $m->{$mongoDbName}; \MongoCursor::$timeout = -1; $output->writeln("Connected to the database <info>{$mongoDbName}</info>"); $collection = $db->{$mongoCollectionName}; $cursor = $this->getCursor($collection); $i = 0; $output->writeln("Removing elastic-search index..."); $this->doDeleteIndex($input, $output); $output->writeln("Creating elastic-search index..."); $this->doCreateIndex($input, $output); $output->writeln("Setting-up elastic-search mapping..."); $this->doSetupMapping($input, $output); $output->writeln("\nIndexing..."); $guzzle = $this->getGuzzle($input); $flag = 0; $nbEntries = $cursor->count(true); $timeStart = microtime(true); $requests = array(); while ($flag == 0) { try { foreach ($cursor as $obj) { $i++; unset($obj['_id']); $requests[] = $guzzle->put($input->getArgument('es-type') . '/' . $i, null, json_encode($obj)); if (0 === $i % 180) { $guzzle->send($requests); $requests = array(); } if (0 === $i % 10000) { $elapsedTime = microtime(true) - $timeStart; $entriesPerSeconds = floor($i / $elapsedTime); $output->writeln(date('H:i:s') . "\tProgress: {$i}/{$nbEntries}\t({$entriesPerSeconds}/seconds)\t" . round($i / $nbEntries * 100) . "% \t" . "~" . $this->secsToString(($nbEntries - $i) / $entriesPerSeconds) . " left\tMemory usage : " . (memory_get_usage() >> 20) . "Mo"); } } $flag = 1; } catch (Exception $ex) { $output->writeln("Something went wrong within MongoDB: " . $ex->getMessage() . ", Retrying ..."); $flag = 0; $cursor = getCursor($collection); } } $output->writeln("{$i} entries processed. Took " . $this->secsToString(floor(microtime(true) - $timeStart)) . " seconds"); return 0; }
/** * run the map/reduce * @static * @return void */ public static function mapReduce() { $map = "function () {\r\n if(arguments.callee.shipcache === undefined)\r\n {\r\n arguments.callee.shipcache = {}\r\n }\r\n if(arguments.callee.shipcache[this.victim.shipTypeID] === undefined)\r\n {\r\n arguments.callee.shipcache[this.victim.shipTypeID] = db.Kingboard_EveItem.findOne({typeID: parseInt(this.victim.shipTypeID)},{'marketGroup.parentGroup.marketGroupName':1});\r\n }\r\n var ship = arguments.callee.shipcache[this.victim.shipTypeID];\r\n var info = {}\r\n info[this.victim.shipType] = 1;\r\n info[\"total\"] = 1;\r\n if(ship != null && ship.marketGroup != null)\r\n emit(ship.marketGroup.parentGroup.marketGroupName, info);\r\n }"; $reduce = "function (k, vals) {\r\n var sums = {}\r\n var total = 0;\r\n vals.forEach(function(info) {\r\n info[\"total\"] = 0;\r\n for (var key in info)\r\n {\r\n if(sums[key] === undefined)\r\n sums[key] = 0;\r\n sums[key] += info[key];\r\n total += info[key];\r\n }\r\n });\r\n sums[\"total\"] = total;\r\n return sums;\r\n }"; // we want the map/reduce to run for as long as it takes MongoCursor::$timeout = -1; return King23_Mongo::mapReduce("Kingboard_Kill", __CLASS__, $map, $reduce); }
/** * * @param \MongoDb $db * @param \Mongodloid_Connection $connection */ public function __construct(\MongoDb $db, \Mongodloid_Connection $connection) { parent::__construct($db, $connection); $this->collections = Billrun_Factory::config()->getConfigValue('db.collections', array()); $timeout = Billrun_Factory::config()->getConfigValue('db.timeout', 3600000); // default 60 minutes Billrun_Factory::log()->log('Set database cursor timeout to: ' . $timeout, Zend_Log::INFO); MongoCursor::$timeout = $timeout; }
/** * @expectedException MongoCursorTimeoutException */ public function testStaticTimeout() { $this->markTestSkipped("for now"); return; MongoCursor::$timeout = 1; for ($i = 0; $i < 1000; $i++) { $this->object->insert(array("x" => "sdfjnaireojaerkgmdfkngkdsflngklsgntoigneorisgmsrklgd{$i}", "y" => $i)); } $rows = $this->object->find(array('$eval' => 'r = 0; cursor = db.c.find(); while (cursor.hasNext()) { x = cursor.next(); for (i=0; i<200; i++) { if (x.name == "joe"+i) { r++; } } } return r;')); foreach ($rows as $row) { } MongoCursor::$timeout = 30000; }
/** * * @param \MongoDb $db * @param \Mongodloid_Connection $connection */ public function __construct(\MongoDb $db, \Mongodloid_Connection $connection) { parent::__construct($db, $connection); // TODO: refatoring the collections to factory (loose coupling) $this->collections = Billrun_Factory::config()->getConfigValue('db.collections', array()); $timeout = Billrun_Factory::config()->getConfigValue('db.timeout', 3600000); // default 60 minutes if ($this->compareClientVersion('1.5.3', '<')) { Billrun_Factory::log()->log('Set database cursor timeout to: ' . $timeout, Zend_Log::INFO); @(MongoCursor::$timeout = $timeout); } else { // see also bugs: // https://jira.mongodb.org/browse/PHP-1099 // https://jira.mongodb.org/browse/PHP-1080 $db->setWriteConcern($db->getWriteConcern()['w'], $timeout); } }
/** * Test global timeouts and native mappings */ public function testGlobalTimeout() { // Create collection $default_timeout = \MongoCursor::$timeout; $collection = $this->getTestCollection(); // Create a cursor object $cursor = $collection->find(); $native_cursor = $cursor->native; // Assert cursor timeouts are updated $this->assertEquals($cursor::$timeout, $native_cursor::$timeout); // Check that static timeouts are binded $default_timeout = \MongoCursor::$timeout; $cursor::$timeout = $default_timeout / 2; $this->assertEquals(\MongoCursor::$timeout, $cursor::$timeout); $this->assertEquals(\MongoMinify\Cursor::$timeout, $cursor::$timeout); \MongoCursor::$timeout = 200; $this->assertEquals(\MongoCursor::$timeout, \MongoMinify\Cursor::$timeout); $this->assertEquals(\MongoMinify\Cursor::$timeout, 200); \MongoCursor::$timeout = $default_timeout; }
/** * * @return \MongoCursor */ private function getCursor() { if ($this->cursor) { return $this->cursor; } $this->cursor = $this->collection->getMongoCollection()->find($this->expression->toArray(), $this->fields); if ($this->skip) { $this->cursor->skip($this->skip); } if ($this->limit) { $this->cursor->limit($this->limit); } if ($this->options['batchSize']) { $this->cursor->batchSize($this->options['batchSize']); } if ($this->options['clientTimeout']) { $this->cursor->timeout($this->options['clientTimeout']); } if ($this->options['serverTimeout']) { $this->cursor->maxTimeMS($this->options['clientTimeout']); } if ($this->sort) { $this->cursor->sort($this->sort); } if ($this->hint) { $this->cursor->hint($this->hint); } // log request if ($this->client->hasLogger()) { $this->client->getLogger()->debug(get_called_class() . ': ' . json_encode(array('collection' => $this->collection->getName(), 'query' => $this->expression->toArray(), 'project' => $this->fields, 'sort' => $this->sort))); } $this->cursor->rewind(); // define read preferences if ($this->readPreference) { $this->cursor->setReadPreference($this->readPreference['type'], $this->readPreference['tagsets']); } return $this->cursor; }
protected function execute(InputInterface $input, OutputInterface $output) { $id = $input->getArgument('collection'); $idmodule = $input->getArgument('module'); $dbname = $input->getArgument('dbname'); $usermail = $input->getArgument('usermail'); if ($id && $idmodule && $dbname && $usermail) { $orphans = array(); $error = ''; $dm = $this->getContainer()->get('doctrine.odm.mongodb.document_manager'); $dm->getConfiguration()->setDefaultDB($dbname); $configuration = $dm->getConnection()->getConfiguration(); $configuration->setLoggerCallable(null); \MongoCursor::$timeout = -1; $module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule); if (!$module) { $error = 'Unable to find Module entity.'; } if ($module->getType() == 'text') { $error = 'Module entity: Wrong type.'; } if (empty($error)) { /* * Open the uploaded csv */ $csvfile = __DIR__ . '/../Resources/uploads/' . $module->getCollection()->getAlias() . '/' . $module->getAlias() . '.csv'; $handle = fopen($csvfile, "r"); /* * Get the module properties */ $columns = fgetcsv($handle, 0, ";"); $fields = array(); $attributes = $module->getProperties(); foreach ($attributes as $field) { $fields[] = $field; } $s = microtime(true); $batchSize = 100; $size = 0; $rowCount = 0; $errorCount = 0; if ($module->getType() == 'image') { while (($data = fgetcsv($handle, 0, ';')) !== false) { $num = count($data); $image = new Image(); $attributes = array(); for ($c = 0; $c < $num; $c++) { $value = trim($this->data_encode($data[$c])); //check for int or float value if (is_numeric($value)) { $tmp_value = intval($value); if ($value == $tmp_value) { $value = $tmp_value; } else { $tmp_value = floatval($value); if ($value == $tmp_value) { $value = $tmp_value; } } } // $attributes[$fields[$c]->getId()] = $value; switch ($fields[$c]->getType()) { case 'file': $image->setPath($value . ''); break; case 'copyright': $image->setCopyright($value . ''); break; case 'idparent': $image->setIdparent($value . ''); break; case 'idmodule': $image->setIdentifier($value . ''); break; } } $image->setProperty($attributes); $image->setModule($module); $parent = null; if ($module->getParent()) { $parent_q = $dm->createQueryBuilder('PlantnetDataBundle:Plantunit')->field('module.id')->equals($module->getParent()->getId())->field('identifier')->equals($image->getIdparent())->getQuery()->execute(); foreach ($parent_q as $p) { $parent = $p; } } if ($parent) { $image->setPlantunit($parent); $image->setTitle1($parent->getTitle1()); $image->setTitle2($parent->getTitle2()); $image->setTitle3($parent->getTitle3()); $dm->persist($image); $rowCount++; $size++; if (!$parent->getHasimages()) { $parent->setHasimages(true); $dm->persist($parent); $size++; } //update Taxons $taxons = $parent->getTaxonsrefs(); if (count($taxons)) { foreach ($taxons as $taxon) { if (!$taxon->getHasimages()) { $taxon->setHasimages(true); $dm->persist($taxon); $size++; } if ($taxon->getIssynonym()) { $taxon_valid = $taxon->getChosen(); if (!$taxon_valid->getHasimages()) { $taxon_valid->setHasimages(true); $dm->persist($taxon_valid); $size++; } $parent_taxon_valid = $taxon_valid->getParent(); while ($parent_taxon_valid) { if (!$parent_taxon_valid->getHasimages()) { $parent_taxon_valid->setHasimages(true); $dm->persist($parent_taxon_valid); $size++; } $parent_taxon_valid = $parent_taxon_valid->getParent(); } } } } if ($size >= $batchSize) { $dm->flush(); $dm->clear(); gc_collect_cycles(); $module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule); $size = 0; } } else { $orphans[$image->getIdparent()] = $image->getIdparent(); $errorCount++; $dm->detach($image); } } $dm->flush(); $dm->clear(); gc_collect_cycles(); $module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule); } elseif ($module->getType() == 'locality') { while (($data = fgetcsv($handle, 0, ';')) !== false) { $geo_error = false; $num = count($data); $location = new Location(); $coordinates = new Coordinates(); $attributes = array(); for ($c = 0; $c < $num; $c++) { $value = trim($this->data_encode($data[$c])); //check for int or float value if (is_numeric($value)) { $tmp_value = intval($value); if ($value == $tmp_value) { $value = $tmp_value; } else { $tmp_value = floatval($value); if ($value == $tmp_value) { $value = $tmp_value; } } } // $attributes[$fields[$c]->getId()] = $value; switch ($fields[$c]->getType()) { case 'lon': if (strlen($value) == 0) { $geo_error = true; } $value = str_replace(',', '.', $value); $value = floatval($value); $location->setLongitude($value); $coordinates->setX($value); break; case 'lat': if (strlen($value) == 0) { $geo_error = true; } $value = str_replace(',', '.', $value); $value = floatval($value); $location->setLatitude($value); $coordinates->setY($value); break; case 'idparent': $location->setIdparent($value . ''); break; case 'idmodule': $location->setIdentifier($value . ''); break; } } if (!$geo_error) { $location->setCoordinates($coordinates); $location->setProperty($attributes); $location->setModule($module); $parent = null; if ($module->getParent()) { $parent_q = $dm->createQueryBuilder('PlantnetDataBundle:Plantunit')->field('module.id')->equals($module->getParent()->getId())->field('identifier')->equals($location->getIdparent())->getQuery()->execute(); foreach ($parent_q as $p) { $parent = $p; } } if ($parent) { $location->setPlantunit($parent); $location->setTitle1($parent->getTitle1()); $location->setTitle2($parent->getTitle2()); $location->setTitle3($parent->getTitle3()); $dm->persist($location); $rowCount++; $size++; if (!$parent->getHaslocations()) { $parent->setHaslocations(true); $dm->persist($parent); $size++; } //update Taxons $taxons = $parent->getTaxonsrefs(); if (count($taxons)) { foreach ($taxons as $taxon) { if (!$taxon->getHaslocations()) { $taxon->setHaslocations(true); $dm->persist($taxon); $size++; } if ($taxon->getIssynonym()) { $taxon_valid = $taxon->getChosen(); if (!$taxon_valid->getHaslocations()) { $taxon_valid->setHaslocations(true); $dm->persist($taxon_valid); $size++; } $parent_taxon_valid = $taxon_valid->getParent(); while ($parent_taxon_valid) { if (!$parent_taxon_valid->getHaslocations()) { $parent_taxon_valid->setHaslocations(true); $dm->persist($parent_taxon_valid); } $parent_taxon_valid = $parent_taxon_valid->getParent(); } } } } if ($size >= $batchSize) { $dm->flush(); $dm->clear(); gc_collect_cycles(); $module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule); $size = 0; } } else { $orphans[$location->getIdparent()] = $location->getIdparent(); $errorCount++; $dm->detach($location); } } else { $orphans[$location->getIdparent()] = $location->getIdparent(); $errorCount++; $dm->detach($location); } } $dm->flush(); $dm->clear(); gc_collect_cycles(); $module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule); } elseif ($module->getType() == 'other') { while (($data = fgetcsv($handle, 0, ';')) !== false) { $num = count($data); $other = new Other(); $attributes = array(); for ($c = 0; $c < $num; $c++) { $value = trim($this->data_encode($data[$c])); //check for int or float value if (is_numeric($value)) { $tmp_value = intval($value); if ($value == $tmp_value) { $value = $tmp_value; } else { $tmp_value = floatval($value); if ($value == $tmp_value) { $value = $tmp_value; } } } // $attributes[$fields[$c]->getId()] = $value; switch ($fields[$c]->getType()) { case 'idparent': $other->setIdparent($value . ''); break; case 'idmodule': $other->setIdentifier($value . ''); break; } } $other->setProperty($attributes); $other->setModule($module); $parent = null; if ($module->getParent()) { $parent_q = $dm->createQueryBuilder('PlantnetDataBundle:Plantunit')->field('module.id')->equals($module->getParent()->getId())->field('identifier')->equals($other->getIdparent())->getQuery()->execute(); foreach ($parent_q as $p) { $parent = $p; } } if ($parent) { $other->setPlantunit($parent); $dm->persist($other); $rowCount++; $size++; if ($size >= $batchSize) { $dm->flush(); $dm->clear(); gc_collect_cycles(); $module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule); $size = 0; } } else { $orphans[$other->getIdparent()] = $other->getIdparent(); $errorCount++; $dm->detach($other); } } $dm->flush(); $dm->clear(); gc_collect_cycles(); $module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule); } fclose($handle); $module->setNbrows($rowCount); $module->setUpdating(false); $dm->persist($module); $dm->flush(); $dm->clear(); $e = microtime(true); echo ' Inserted ' . $rowCount . ' objects in ' . ($e - $s) . ' seconds' . PHP_EOL; if (file_exists($csvfile)) { unlink($csvfile); } $message = 'Importation Success: ' . $rowCount . ' objects imported in ' . ($e - $s) . ' seconds'; $message .= "\n"; $message .= 'Importation Error: ' . $errorCount; if (count($orphans)) { $message .= "\n" . 'These "id parent" do not exist:' . "\n"; $message .= implode(', ', $orphans); } } else { $message = $error; } $message_mail = \Swift_Message::newInstance()->setSubject('Publish : task ended')->setFrom($this->getContainer()->getParameter('from_email_adress'))->setTo($usermail)->setBody($message . $this->getContainer()->get('templating')->render('PlantnetDataBundle:Backend\\Mail:task.txt.twig')); $this->getContainer()->get('mailer')->send($message_mail); $spool = $this->getContainer()->get('mailer')->getTransport()->getSpool(); $transport = $this->getContainer()->get('swiftmailer.transport.real'); $spool->flushQueue($transport); // $connection = new \MongoClient(); $db = $connection->{$dbname}; $db->Module->update(array('_id' => new \MongoId($idmodule)), array('$set' => array('updating' => false))); } }
/** * Update a document * * @param string $collection Name of the collection * @param array $options Array of update options * * @throws Exception * @access public * @return boolean */ public function update($collection = '', $options = array()) { if (empty($collection)) { throw new \MongoQB\Exception('No Mongo collection selected to update'); } if (count($this->updates) === 0) { throw new \MongoQB\Exception('Nothing to update in Mongo collection or update is not an array'); } try { \MongoCursor::$timeout = -1; $options = array_merge(array($this->_querySafety => true, 'multiple' => false), $options); $result = $this->_dbhandle->{$collection}->update($this->wheres, $this->updates, $options); $this->_clear($collection, 'update'); if ($result['updatedExisting'] > 0) { return $result['updatedExisting']; } // @codeCoverageIgnoreStart return false; // @codeCoverageIgnoreEnd } catch (\MongoCursorException $Exception) { throw new \MongoQB\Exception('Update of data into MongoDB failed: ' . $Exception->getMessage()); // @codeCoverageIgnoreEnd } }
private function update_punit($dm, $module, $dbname, $usermail) { \MongoCursor::$timeout = -1; $idmodule = $module->getId(); $this->reset_taxo($dm, $module); $csvfile = __DIR__ . '/../Resources/uploads/' . $module->getCollection()->getAlias() . '/' . $module->getAlias() . '.csv'; $handle = fopen($csvfile, "r"); $columns = fgetcsv($handle, 0, ";"); $fields = array(); $attributes = $module->getProperties(); foreach ($attributes as $field) { $fields[] = $field; } //Punit identifiers in the csv file $csv_ids = array(); while (($data = fgetcsv($handle, 0, ';')) !== false) { $num = count($data); for ($c = 0; $c < $num; $c++) { if ($fields[$c]->getType() == 'idmodule') { $value = trim($this->data_encode($data[$c])); $csv_ids[] = $value; } } } fclose($handle); //get existing ids in database $db_ids = array(); $punits = $dm->createQueryBuilder('PlantnetDataBundle:Plantunit')->hydrate(false)->select('identifier')->field('module')->references($module)->getQuery()->execute(); foreach ($punits as $id) { $db_ids[] = $id['identifier']; } $punits = null; unset($punits); $ids_to_remove = array_diff($db_ids, $csv_ids); if (count($ids_to_remove)) { $sub_ids_to_remove = array_chunk($ids_to_remove, 100); foreach ($sub_ids_to_remove as $tab_ids) { //delete //deletes punits where id is not in csv_ids $dm->createQueryBuilder('PlantnetDataBundle:Plantunit')->remove()->field('identifier')->in($tab_ids)->getQuery()->execute(); //cascade doesnt work !? $dm->createQueryBuilder('PlantnetDataBundle:Image')->remove()->field('idparent')->in($tab_ids)->getQuery()->execute(); $dm->createQueryBuilder('PlantnetDataBundle:Location')->remove()->field('idparent')->in($tab_ids)->getQuery()->execute(); $dm->createQueryBuilder('PlantnetDataBundle:Other')->remove()->field('idparent')->in($tab_ids)->getQuery()->execute(); } } $csv_ids = null; unset($csv_ids); $db_ids = null; unset($db_ids); $ids_to_remove = null; unset($ids_to_remove); //update / create Punits $handle = fopen($csvfile, "r"); $columns = fgetcsv($handle, 0, ";"); $batchSize = 100; $size = 0; while (($data = fgetcsv($handle, 0, ';')) !== false) { $csv_id = null; $num = count($data); for ($c = 0; $c < $num; $c++) { if ($fields[$c]->getType() == 'idmodule') { $value = trim($this->data_encode($data[$c])); $csv_id = $value; } } if ($csv_id) { $update = true; $subupdate = false; //update //updates punits where id is in csv_ids $plantunit = $dm->getRepository('PlantnetDataBundle:Plantunit')->findOneBy(array('module.id' => $module->getId(), 'identifier' => $csv_id)); //create //creates punits where csv_id is not in id if (!$plantunit) { $update = false; $plantunit = new Plantunit(); $plantunit->setModule($module); } $attributes = array(); for ($c = 0; $c < $num; $c++) { $value = trim($this->data_encode($data[$c])); //check for int or float value if (is_numeric($value)) { $tmp_value = intval($value); if ($value == $tmp_value) { $value = $tmp_value; } else { $tmp_value = floatval($value); if ($value == $tmp_value) { $value = $tmp_value; } } } // $attributes[$fields[$c]->getId()] = $value; switch ($fields[$c]->getType()) { case 'idmodule': $plantunit->setIdentifier($value . ''); break; case 'idparent': $plantunit->setIdparent($value . ''); break; case 'title1': if ($update && $plantunit->getTitle1() != $value . '') { $subupdate = true; } $plantunit->setTitle1($value . ''); break; case 'title2': if ($update && $plantunit->getTitle2() != $value . '') { $subupdate = true; } $plantunit->setTitle2($value . ''); break; case 'title3': if ($update && $plantunit->getTitle3() != $value . '') { $subupdate = true; } $plantunit->setTitle3($value . ''); break; } } $plantunit->setAttributes($attributes); $dm->persist($plantunit); $size++; // if ($update && $subupdate) { $dm->createQueryBuilder('PlantnetDataBundle:Image')->update()->multiple(true)->field('plantunit')->references($plantunit)->field('title1')->set($plantunit->getTitle1())->field('title2')->set($plantunit->getTitle2())->field('title3')->set($plantunit->getTitle3())->getQuery()->execute(); $dm->createQueryBuilder('PlantnetDataBundle:Location')->update()->multiple(true)->field('plantunit')->references($plantunit)->field('title1')->set($plantunit->getTitle1())->field('title2')->set($plantunit->getTitle2())->field('title3')->set($plantunit->getTitle3())->getQuery()->execute(); } // if ($size >= $batchSize) { $dm->flush(); $dm->clear(); gc_collect_cycles(); $module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule); $size = 0; } } } fclose($handle); $dm->flush(); $dm->clear(); gc_collect_cycles(); $module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule); //nb rows module $count = $dm->createQueryBuilder('PlantnetDataBundle:Plantunit')->field('module')->references($module)->getQuery()->execute()->count(); $module->setNbrows($count); $dm->persist($module); //nb rows children $children = $module->getChildren(); if (count($children)) { foreach ($children as $child) { if ($child->getType() == 'image') { $count = $dm->createQueryBuilder('PlantnetDataBundle:Image')->field('module')->references($child)->getQuery()->execute()->count(); $child->setNbrows($count); $dm->persist($child); } elseif ($child->getType() == 'locality') { $count = $dm->createQueryBuilder('PlantnetDataBundle:Location')->field('module')->references($child)->getQuery()->execute()->count(); $child->setNbrows($count); $dm->persist($child); } elseif ($child->getType() == 'other') { $count = $dm->createQueryBuilder('PlantnetDataBundle:Other')->field('module')->references($child)->getQuery()->execute()->count(); $child->setNbrows($count); $dm->persist($child); } } } //flush it ! $dm->flush(); $dm->clear(); gc_collect_cycles(); $module = $dm->getRepository('PlantnetDataBundle:Module')->find($idmodule); // return $module; }
/** * @ApiDoc( * section="Publish v2 - GeoData. Sub-module entity [type = locality]", * description="Returns geo data from a 'locality' sub-module entity [GeoJson]", * statusCodes={ * 200="Returned when: Successful", * 401="Returned when: Unauthorized client", * 404="Returned when: Resource not found" * }, * filters={ * {"name"="project", "dataType"="String", "required"=true, "description"="Project url"}, * {"name"="collection", "dataType"="String", "required"=true, "description"="Collection url"}, * {"name"="module", "dataType"="String", "required"=true, "description"="Module url"}, * {"name"="submodule", "dataType"="String", "required"=true, "description"="'Location' Sub-Module url"} * } * ) * * @Route( * "/{project}/{collection}/{module}/{submodule}/geodata", * name="api_submodule_geodata" * ) * @Method("get") */ public function api_submodule_geodataAction($project, $collection, $module, $submodule) { ini_set('memory_limit', '-1'); //check project try { ControllerHelp::check_enable_project($project, $this->get_prefix(), $this); } catch (\Exception $e) { $this->return_404_not_found($e->getMessage()); exit; } //init $dm = $this->get('doctrine.odm.mongodb.document_manager'); $dm->getConfiguration()->setDefaultDB($this->get_prefix() . $project); $result = array(); //get language config $config = ControllerHelp::get_config($project, $dm, $this); $this->check_authorized_client($config); //data1 $collection = $dm->getRepository('PlantnetDataBundle:Collection')->findOneBy(array('url' => $collection)); if (!$collection || $collection->getDeleting() == true) { $this->return_404_not_found('Unable to find Collection entity.'); exit; } $module = $dm->getRepository('PlantnetDataBundle:Module')->findOneBy(array('url' => $module, 'collection.id' => $collection->getId())); if (!$module || $module->getType() != 'text' || $module->getDeleting() == true) { $this->return_404_not_found('Unable to find Module entity.'); exit; } $submodule = $dm->getRepository('PlantnetDataBundle:Module')->findOneBy(array('url' => $submodule, 'parent.id' => $module->getId(), 'collection.id' => $collection->getId())); if (!$submodule || $submodule->getType() == 'text' || $submodule->getDeleting() == true) { $this->return_404_not_found('Unable to find Sub-module entity.'); exit; } if ($submodule->getType() != 'locality') { $this->return_404_not_found('Unable to find geo data for this Sub-module entity.'); exit; } //data2 $display = array(); $field = $module->getProperties(); foreach ($field as $row) { if ($row->getMain() == true) { $display[] = $row->getId(); } } $field_sub = $submodule->getProperties(); $field_sub_tab = array(); foreach ($field_sub as $f) { if ($f->getDetails() == true) { $field_sub_tab[$f->getId()] = StringHelp::cleanToKey($f->getName()); } } \MongoCursor::$timeout = -1; $locations = $dm->createQueryBuilder('PlantnetDataBundle:Location')->hydrate(false)->select('title1')->select('title2')->select('title3')->select('property')->select('latitude')->select('longitude')->select('idparent')->field('module')->references($submodule)->getQuery()->execute()->toArray(); array_walk($locations, function (&$item, $key, $field_sub_tab) { $l = array('type' => 'Feature', 'geometry' => array('type' => 'Point', 'coordinates' => array($item['longitude'], $item['latitude'])), 'properties' => array('title1' => $item['title1'], 'title2' => $item['title2'], 'title3' => $item['title3'], 'parent_identifier' => $item['idparent'])); foreach ($item['property'] as $key => $value) { if (array_key_exists($key, $field_sub_tab)) { $l['properties'][$field_sub_tab[$key]] = $value; } } $item = $l; }, $field_sub_tab); $locations = array_values($locations); //response $response = new Response(json_encode(array('type' => 'FeatureCollection', 'features' => $locations))); $response->headers->set('Content-Type', 'application/json'); return $response; }
echo 'Total : ' . sprintf('%01.6f', microtime(true) - $startTime) . ' seconds.<br />'; exit;*/ if (DO_MYSQL) { $mysqlConnection = mysql_connect('127.0.0.1', 'root', 'password'); mysql_select_db('cps2_perf_test'); } if (DO_CPS) { $cpsConnection->sendRequest(new CPS_Request('clear')); sleep(1); } // $cpsConnection->waitForResponse(false); if (DO_MONGO) { $m = new Mongo(); $db = $m->test; $collection = $db->insert_test; MongoCursor::$timeout = 120000; // 2 minutes } if (defined('RANDOM_SEED')) { srand(RANDOM_SEED); echo 'Seeded with random seed ' . RANDOM_SEED . ', first random value: ' . rand() . '<br />'; } $randomWords = array(); $wordChars = '0123456789abcdefghijklmnopqrstuvwxyz'; $charCount = strlen($wordChars); define('WORD_COUNT', 100000); for ($x = 0; $x < WORD_COUNT; ++$x) { $size = rand(3, 9); $word = ''; for ($y = 0; $y < $size; ++$y) { $word .= $wordChars[rand(0, $charCount - 1)];
public static function group($collection, $keys = [], $query = [], $count = [], $sum = [], $sort = [], $limit = null) { global $debug; // Turn keys into an array if is isn't already an array if (!is_array($keys)) { $keys = [$keys]; } // Start the aggregation pipeline with the query $pipeline = []; if (sizeof($query)) { $pipeline[] = ['$match' => $query]; } // Create the group by using the given key(s) $ids = []; foreach ($keys as $key => $value) { if (is_numeric($key)) { $ids[] = '$' . $value; } else { $ids[$key] = ['$' . $key => '$' . $value]; } } if (sizeof($ids) == 1 && isset($ids[0])) { $ids = $ids[0]; } $group = []; $group['_id'] = $ids; // If no counts or sums are given, assume a count based on the keys for the $group if (sizeof($count) == 0 && sizeof($sum) == 0) { $group['count'] = ['$sum' => 1]; } // Include counts in the $group if (!is_array($count)) { $count = [$count]; } foreach ($count as $s) { $group[str_replace('.', '_', $s) . 'Count'] = ['$sum' => 1]; } // Include sums in the $group if (!is_array($sum)) { $sum = [$sum]; } foreach ($sum as $s) { $group[str_replace('.', '_', $s) . 'Sum'] = ['$sum' => '$' . $s]; } // Add the group to the pipeline $pipeline[] = ['$group' => $group]; // $project the keys into the result $project = []; $project['_id'] = 0; foreach ($keys as $key => $value) { if (is_numeric($key)) { $project[$value] = '$_id'; } else { $project[$key] = '$_id.' . $key; } } if (sizeof($count) == 0 && sizeof($sum) == 0) { $project['count'] = 1; } if (sizeof($count) > 0) { foreach ($count as $s) { $project[str_replace('.', '_', $s) . 'Count'] = 1; } } if (sizeof($sum) > 0) { foreach ($sum as $s) { $project[str_replace('.', '_', $s) . 'Sum'] = 1; } } $pipeline[] = ['$project' => $project]; // Assign the sort to the pipeline if (sizeof($sort) > 0) { $pipeline[] = ['$sort' => $sort]; } // And add the limit if ($limit != null) { $pipeline[] = ['$limit' => (int) $limit]; } // Prep the cursor $mdb = new self(); $collection = $mdb->getCollection($collection); if (!$debug) { MongoCursor::$timeout = -1; } // this should be deprecated but aggregate doesn't have a timeout // Execute the query $result = $collection->aggregate($pipeline); if ($result['ok'] == 1) { return $result['result']; } throw new Exception('pipeline query failure'); }
/** * setTimeout Method * * Sets the MongoCursor timeout so long queries (like map / reduce) can run at will. * Expressed in milliseconds, for an infinite timeout, set to -1 * * @param int $ms * @return boolean * @access public */ public function setTimeout($ms) { MongoCursor::$timeout = $ms; return true; }
/** * distinct查询 * @param $field * @param $conds * @return array * @throws Exception */ public function Distinct($field, $conds) { $callback = function ($collection) use($field, $conds) { return $collection->Distinct($field, $conds); }; try { MongoCursor::$timeout = 120000; $cursor = $this->_doAction($callback); //$cursor->timeout(120000); } catch (Exception $ex) { throw new Exception(__METHOD__ . ' Distinct fail! Exception code: ' . $ex->getCode() . ' msg: ' . $ex->getMessage(), self::ERRNO_FINDONE_FAIL); } $ret = $this->UnCompress($cursor); return $ret; }
private function delete_glossary($dbname, $id) { $error = ''; $dm = $this->getContainer()->get('doctrine.odm.mongodb.document_manager'); $dm->getConfiguration()->setDefaultDB($dbname); $configuration = $dm->getConnection()->getConfiguration(); $configuration->setLoggerCallable(null); \MongoCursor::$timeout = -1; $glossary = $dm->getRepository('PlantnetDataBundle:Glossary')->findOneBy(array('id' => $id)); if (!$glossary) { $error = 'Unable to find Glossary entity.'; } /* * Remove csv directory (and files) */ $file = __DIR__ . '/../Resources/uploads/' . $glossary->getCollection()->getAlias() . '/glossary.csv'; if (file_exists($file)) { unlink($file); } $file = __DIR__ . '/../Resources/uploads/' . $glossary->getCollection()->getAlias() . '/glossary_syn.csv'; if (file_exists($file)) { unlink($file); } /* * Remove upload directory */ $dir = $glossary->getUploaddir(); $del_dir = true; $nb_equal_dirs = 0; $config = $dm->createQueryBuilder('PlantnetDataBundle:Config')->getQuery()->getSingleResult(); if ($config) { $original_db = $config->getOriginaldb(); if ($original_db) { $connection = new \MongoClient(); $dbs_array = array(); $dbs = $connection->admin->command(array('listDatabases' => 1)); foreach ($dbs['databases'] as $db) { $db_name = $db['name']; if (substr_count($db_name, $original_db)) { $dbs_array[] = $db_name; } } if (count($dbs_array) > 1) { foreach ($dbs_array as $chk_db) { $nb_equal_dirs += $connection->{$chk_db}->Glossary->find(array('uploaddir' => $dir))->count(); } } } } if ($nb_equal_dirs > 1) { $del_dir = false; } if ($dir && $del_dir) { $dir = __DIR__ . '/../../../../web/uploads/' . $dir; if (file_exists($dir) && is_dir($dir)) { $files = scandir($dir); foreach ($files as $file) { if ($file != '.' && $file != '..') { unlink($dir . '/' . $file); } } rmdir($dir); } } /* * Remove Glossary + Cascade */ $dm->remove($glossary); $dm->flush(); $dm->detach($glossary); }
/** * 设置一个超时 * @param int $ms * @return muMongoCursor */ public function timeout($ms) { $this->oMongoCursor->timeout($ms); return $this; }
private function glossarize($action, $dbname, $id_glossary, $usermail) { $error = ''; $dm = $this->getContainer()->get('doctrine.odm.mongodb.document_manager'); $dm->getConfiguration()->setDefaultDB($dbname); $configuration = $dm->getConnection()->getConfiguration(); $configuration->setLoggerCallable(null); $glossary = $dm->getRepository('PlantnetDataBundle:Glossary')->findOneBy(array('id' => $id_glossary)); if (!$glossary) { $error = 'Unable to find Glossary entity.'; } $collection = $glossary->getCollection(); if (!$collection) { $error = 'Unable to find Glossary entity.'; } // if ($action == 'syn') { $csv = __DIR__ . '/../Resources/uploads/' . $collection->getAlias() . '/glossary_syn.csv'; if (file_exists($csv)) { $handle = fopen($csv, "r"); $field = fgetcsv($handle, 0, ";"); $csv_error = false; if (!isset($field[0]) || empty($field[0])) { $csv_error = true; } if (!isset($field[1]) || empty($field[1])) { $csv_error = true; } if (!$csv_error) { $i = 0; $batch_size = 100; \MongoCursor::$timeout = -1; while (($data = fgetcsv($handle, 0, ';')) !== false) { $def_word = isset($data[0]) ? trim($data[0]) : ''; $cur_encoding = mb_detect_encoding($def_word); if ($cur_encoding == "UTF-8" && mb_check_encoding($def_word, "UTF-8")) { $def_word = $def_word; } else { $def_word = utf8_encode($def_word); } $syn_word = isset($data[1]) ? trim($data[1]) : ''; $cur_encoding = mb_detect_encoding($syn_word); if ($cur_encoding == "UTF-8" && mb_check_encoding($syn_word, "UTF-8")) { $syn_word = $syn_word; } else { $syn_word = utf8_encode($syn_word); } if ($def_word && $syn_word) { $definition = $dm->getRepository('PlantnetDataBundle:Definition')->findOneBy(array('glossary.id' => $glossary->getId(), 'name' => $def_word)); if ($definition) { $i++; if (!$definition->getHaschildren()) { $definition->setHaschildren(true); $dm->persist($definition); } $synonym = new Definition(); $synonym->setGlossary($glossary); $synonym->setParent($definition); $synonym->setName($syn_word); $synonym->setDisplayedname($syn_word); $synonym->setDefinition($definition->getDefinition()); $synonym->setPath($definition->getPath()); $dm->persist($synonym); if ($i >= $batch_size) { $dm->flush(); $i = 0; $dm->clear(); gc_collect_cycles(); $glossary = $dm->getRepository('PlantnetDataBundle:Glossary')->findOneBy(array('id' => $id_glossary)); } } } } $dm->flush(); $dm->clear(); gc_collect_cycles(); $glossary = $dm->getRepository('PlantnetDataBundle:Glossary')->findOneBy(array('id' => $id_glossary)); } else { $error = 'Error in Glossary Syn file.'; } } else { $error = 'Unable to find Glossary Syn file.'; } $message = $error; if (empty($message)) { $message = 'Synonyms were created successfully.'; } $message_mail = \Swift_Message::newInstance()->setSubject('Publish : task ended')->setFrom($this->getContainer()->getParameter('from_email_adress'))->setTo($usermail)->setBody($message . $this->getContainer()->get('templating')->render('PlantnetDataBundle:Backend\\Mail:task.txt.twig')); $this->getContainer()->get('mailer')->send($message_mail); $spool = $this->getContainer()->get('mailer')->getTransport()->getSpool(); $transport = $this->getContainer()->get('swiftmailer.transport.real'); $spool->flushQueue($transport); } elseif ($action == 'unsyn') { $dm->createQueryBuilder('PlantnetDataBundle:Definition')->remove()->field('glossary')->references($glossary)->field('parent')->notEqual(null)->getQuery()->execute(); } }
/** * Return result set for current query */ public function toCollection(Spot_Query $query, MongoCursor $cursor) { $mapper = $query->mapper(); $entityClass = $query->entityName(); if ($cursor instanceof MongoCursor) { $results = array(); $resultsIdentities = array(); // Set timeout if (isset($this->options['cursor']['timeout']) && is_int($this->options['cursor']['timeout'])) { $cursor->timeout($this->options['cursor']['timeout']); } // Fetch all results into new entity class // @todo Move this to collection class so entities will be lazy-loaded by Collection iteration foreach ($cursor as $document) { $entity = new $entityClass(); $mapper->data($entity, $document); // Store in array for Collection $results[] = $entity; // Store primary key of each unique record in set $pk = $mapper->primaryKey($entity); if (!in_array($pk, $resultsIdentities) && !empty($pk)) { $resultsIdentities[] = $pk; } } $collectionClass = $mapper->collectionClass(); return new $collectionClass($results, $resultsIdentities); } else { $mapper->addError(__METHOD__ . " - Unable to execute query - not a valid MongoCursor"); return array(); } }
/** * Return result set for current query */ public function toCollection(\Spot\Query $query, \MongoCursor $cursor) { $mapper = $query->mapper(); $entityClass = $query->entityName(); if ($cursor instanceof MongoCursor) { // Set timeout if (isset($this->options['cursor']['timeout']) && is_int($this->options['cursor']['timeout'])) { $cursor->timeout($this->options['cursor']['timeout']); } return $mapper->collection($entityClass, $cursor); } else { $mapper->addError(__METHOD__ . " - Unable to execute query - not a valid MongoCursor"); return array(); } }
<?php $connection = new MongoClient(); $collection = $connection->selectDB("getglue")->selectCollection("gg"); MongoCursor::$timeout = -1; $match1 = array('$match' => array('comment' => array('$ne' => ''))); $group = array('$group' => array('_id' => '$userId', 'total' => array('$sum' => 1))); $match2 = array('$match' => array('total' => array('$gte' => 5000))); $sort = array('$sort' => array('total' => -1)); $limit = array('$limit' => 7); $pipeline = array($match1, $group, $match2, $sort, $limit); $out = $collection->aggregate($pipeline); echo json_encode($out, JSON_PRETTY_PRINT);
private function taxonomize($action, $dbname, $id_module, $usermail, $clean) { $error = ''; $dm = $this->getContainer()->get('doctrine.odm.mongodb.document_manager'); $dm->getConfiguration()->setDefaultDB($dbname); $configuration = $dm->getConnection()->getConfiguration(); $configuration->setLoggerCallable(null); $module = $dm->getRepository('PlantnetDataBundle:Module')->findOneBy(array('id' => $id_module)); if (!$module) { $error = 'Unable to find Module entity.'; } if ($clean) { $csv = __DIR__ . '/../Resources/uploads/' . $module->getCollection()->getAlias() . '/' . $module->getAlias() . '_syn.csv'; if (file_exists($csv)) { unlink($csv); } $csv = __DIR__ . '/../Resources/uploads/' . $module->getCollection()->getAlias() . '/' . $module->getAlias() . '_desc.csv'; if (file_exists($csv)) { unlink($csv); } } if ($action == 'taxo') { \MongoCursor::$timeout = -1; // remove old taxa' refs $dm->createQueryBuilder('PlantnetDataBundle:Plantunit')->update()->multiple(true)->field('module')->references($module)->field('taxon')->unsetField()->field('taxonsrefs')->unsetField()->getQuery()->execute(); $sub_modules = $module->getChildren(); foreach ($sub_modules as $sub) { if ($sub->getType() == 'image') { $dm->createQueryBuilder('PlantnetDataBundle:Image')->update()->multiple(true)->field('module')->references($sub)->field('taxonsrefs')->unsetField()->getQuery()->execute(); } elseif ($sub->getType() == 'locality') { $dm->createQueryBuilder('PlantnetDataBundle:Location')->update()->multiple(true)->field('module')->references($sub)->field('taxonsrefs')->unsetField()->getQuery()->execute(); } } // remove old taxa $dm->createQueryBuilder('PlantnetDataBundle:Taxon')->remove()->field('module')->references($module)->getQuery()->execute(); } // load taxonomy data $taxo = array(); $fields = $module->getProperties(); foreach ($fields as $field) { if ($field->getTaxolevel()) { $taxo[$field->getTaxolevel()] = array($field->getId(), $field->getTaxolabel()); } } $fields = null; if (count($taxo)) { ksort($taxo); $first_level = key($taxo); end($taxo); $last_level = key($taxo); reset($taxo); if ($action == 'taxo') { $connection = new \MongoClient(); $db = $connection->{$dbname}; //populate $tab_tax = $this->populate($db, $module, $taxo); //set indexes if (count($this->tmp_indexes)) { foreach ($this->tmp_indexes as $i_name => $i_index) { $db->Plantunit->ensureIndex($i_index, array('name' => $i_name)); } } //save \MongoCursor::$timeout = -1; $this->save($db, $dbname, $dm, $module, $taxo, $tab_tax); $tab_tax = null; unset($tab_tax); $dm->clear(); gc_collect_cycles(); $module = $dm->getRepository('PlantnetDataBundle:Module')->findOneBy(array('id' => $id_module)); //unset indexes if (count($this->tmp_indexes)) { foreach ($this->tmp_indexes as $i_name => $i_index) { $db->command(array('deleteIndexes' => 'Plantunit', 'index' => $i_name)); } } //free $db = null; $connection = null; } elseif ($action == 'syn') { // synonymy management $csv = __DIR__ . '/../Resources/uploads/' . $module->getCollection()->getAlias() . '/' . $module->getAlias() . '_syn.csv'; if (file_exists($csv)) { $cols = array(); $syns = array(); $handle = fopen($csv, "r"); $field = fgetcsv($handle, 0, ";"); foreach ($field as $col) { $col_name = ''; $cur_encoding = mb_detect_encoding($col); if ($cur_encoding == "UTF-8" && mb_check_encoding($col, "UTF-8")) { $col_name = $col; } else { $col_name = utf8_encode($col); } $cols[] = $col_name; } $csv_error = false; foreach ($taxo as $level => $data) { $syns[$level] = array('col_non_valid' => '', 'col_valid' => ''); foreach ($cols as $key => $col) { if (empty($data[1])) { $csv_error = true; } else { if ($col == $data[1]) { $syns[$level]['col_non_valid'] = $key; } if ($col == $data[1] . '_valid') { $syns[$level]['col_valid'] = $key; } } } if (empty($syns[$level]['col_non_valid']) && $syns[$level]['col_non_valid'] != 0) { $csv_error = true; } if (empty($syns[$level]['col_valid']) && $syns[$level]['col_valid'] != 0) { $csv_error = true; } } if (!$csv_error) { $connection = new \MongoClient(); $db = $connection->{$dbname}; \MongoCursor::$timeout = -1; while (($data = fgetcsv($handle, 0, ';')) !== false) { //last non null $last_non_null_rank_non_valid = 0; $last_non_null_rank_valid = 0; foreach ($syns as $level => $tab) { $checked_value_non_valid = isset($data[$tab['col_non_valid']]) ? trim($data[$tab['col_non_valid']]) : ''; $checked_value_valid = isset($data[$tab['col_valid']]) ? trim($data[$tab['col_valid']]) : ''; if (!empty($checked_value_non_valid)) { $last_non_null_rank_non_valid = $level; } if (!empty($checked_value_valid)) { $last_non_null_rank_valid = $level; } } //data $non_valid_identifier = ''; $valid_identifier = ''; $non_valid = array(); $valid = array(); foreach ($syns as $level => $tab) { $string_non_valid = isset($data[$tab['col_non_valid']]) ? trim($data[$tab['col_non_valid']]) : ''; if (empty($string_non_valid) && $level < $last_non_null_rank_non_valid) { $string_non_valid = '-'; } $string_valid = isset($data[$tab['col_valid']]) ? trim($data[$tab['col_valid']]) : ''; if (empty($string_valid) && $level < $last_non_null_rank_valid) { $string_valid = '-'; } $cur_encoding = mb_detect_encoding($string_non_valid); if ($cur_encoding == "UTF-8" && mb_check_encoding($string_non_valid, "UTF-8")) { $string_non_valid = $string_non_valid; } else { $string_non_valid = utf8_encode($string_non_valid); } $cur_encoding = mb_detect_encoding($string_valid); if ($cur_encoding == "UTF-8" && mb_check_encoding($string_valid, "UTF-8")) { $string_valid = $string_valid; } else { $string_valid = utf8_encode($string_valid); } if ($non_valid_identifier && !empty($string_non_valid)) { $non_valid_identifier .= ' - '; } $non_valid_identifier .= $string_non_valid; if ($valid_identifier && !empty($string_valid)) { $valid_identifier .= ' - '; } $valid_identifier .= $string_valid; $non_valid[$level] = null; if (!empty($string_non_valid)) { $non_valid[$level] = array('name' => $string_non_valid, 'identifier' => $non_valid_identifier, 'level' => $level, 'label' => $cols[$tab['col_non_valid']]); } $valid[$level] = null; if (!empty($string_valid)) { $valid[$level] = array('name' => $string_valid, 'identifier' => $valid_identifier, 'level' => $level, 'label' => $cols[$tab['col_non_valid']]); } } $last_non_valid = null; foreach ($non_valid as $lvl => $check_taxon) { if ($check_taxon) { $tax = $dm->getRepository('PlantnetDataBundle:Taxon')->findOneBy(array('module.id' => $module->getId(), 'identifier' => $check_taxon['identifier'])); if (!$tax) { $tax = new Taxon(); $tax->setIdentifier($check_taxon['identifier']); $tax->setName($check_taxon['name']); $tax->setLabel($check_taxon['label']); $tax->setLevel($check_taxon['level']); $tax->setModule($module); $tax->setNbpunits(0); $tax->setIssynonym(false); if ($last_non_valid) { $tax->setParent($last_non_valid); if ($last_non_valid->getHaschildren() != true) { $last_non_valid->setHaschildren(true); $dm->persist($last_non_valid); } } $dm->persist($tax); $dm->flush(); } $last_non_valid = $tax; } } $last_valid = null; foreach ($valid as $lvl => $check_taxon) { if ($check_taxon) { $tax = $dm->getRepository('PlantnetDataBundle:Taxon')->findOneBy(array('module.id' => $module->getId(), 'identifier' => $check_taxon['identifier'])); if (!$tax) { $tax = new Taxon(); $tax->setIdentifier($check_taxon['identifier']); $tax->setName($check_taxon['name']); $tax->setLabel($check_taxon['label']); $tax->setLevel($check_taxon['level']); $tax->setModule($module); $tax->setNbpunits(0); $tax->setIssynonym(false); if ($last_valid) { $tax->setParent($last_valid); if ($last_valid->getHaschildren() != true) { $last_valid->setHaschildren(true); $dm->persist($last_valid); } } $dm->persist($tax); $dm->flush(); } $last_valid = $tax; } } $exists = false; if ($last_non_valid->getIssynonym() && $last_non_valid->getChosen()) { if ($last_non_valid->getChosen()->getId() == $last_valid->getId()) { $exists = true; } } if (!$exists && $last_valid && $last_non_valid && $last_valid->getIdentifier() != $last_non_valid->getIdentifier()) { $last_non_valid->setIssynonym(true); $last_non_valid->setChosen($last_valid); if ($last_non_valid->getHaschildren()) { $last_valid->setHaschildren(true); } $last_valid->setHassynonyms(true); // $has_images = $last_non_valid->getHasimages(); $has_locations = $last_non_valid->getHaslocations(); $nb_to_switch = $last_non_valid->getNbpunits(); $parent_tmp = $last_non_valid->getParent(); while ($parent_tmp) { $parent_tmp->setNbpunits($parent_tmp->getNbpunits() - $nb_to_switch); $dm->persist($parent_tmp); $parent_tmp = $parent_tmp->getParent(); } $last_valid->setNbpunits($last_valid->getNbpunits() + $nb_to_switch); if ($has_images) { $last_valid->setHasimages(true); } if ($has_locations) { $last_valid->setHaslocations(true); } $dm->persist($last_valid); $parent_tmp = $last_valid->getParent(); while ($parent_tmp) { $parent_tmp->setNbpunits($parent_tmp->getNbpunits() + $nb_to_switch); if ($has_images) { $parent_tmp->setHasimages(true); } if ($has_locations) { $parent_tmp->setHaslocations(true); } $dm->persist($parent_tmp); $parent_tmp = $parent_tmp->getParent(); } $dm->persist($last_non_valid); $dm->persist($last_valid); $dm->flush(); $db->Plantunit->update(array('module.$id' => new \MongoId($module->getId()), 'taxonsrefs' => array('$elemMatch' => array('$id' => new \MongoId($last_non_valid->getId())))), array('$addToSet' => array('taxonsrefs' => array('$ref' => 'Taxon', '$id' => new \MongoId($last_valid->getId()), '$db' => $dbname))), array('multiple' => true)); } $dm->clear(); gc_collect_cycles(); $module = $dm->getRepository('PlantnetDataBundle:Module')->findOneBy(array('id' => $id_module)); } } else { $error = 'CSV file contains some errors...'; } } } elseif ($action == 'desc') { // description management $csv = __DIR__ . '/../Resources/uploads/' . $module->getCollection()->getAlias() . '/' . $module->getAlias() . '_desc.csv'; if (file_exists($csv)) { $cols = array(); $taxa = array(); $desc = array(); $handle = fopen($csv, "r"); $field = fgetcsv($handle, 0, ";"); foreach ($field as $col) { $col_name = ''; $cur_encoding = mb_detect_encoding($col); if ($cur_encoding == "UTF-8" && mb_check_encoding($col, "UTF-8")) { $col_name = $col; } else { $col_name = utf8_encode($col); } $cols[] = $col_name; } $csv_error = false; $used_key = array(); foreach ($taxo as $level => $data) { $taxa[$level] = array('col' => ''); foreach ($cols as $key => $col) { if ($col == $data[1]) { $taxa[$level]['col'] = $key; $used_key[] = $key; } } if (empty($taxa[$level]['col']) && $taxa[$level]['col'] != 0) { $csv_error = true; } } foreach ($cols as $key => $col) { if (!in_array($key, $used_key)) { $desc[$key] = $col; } } $used_key = null; if (!count($desc)) { $csv_error = true; } if (!$csv_error) { $i = 0; $batch_size = 100; \MongoCursor::$timeout = -1; while (($data = fgetcsv($handle, 0, ';')) !== false) { $identifier = ''; $tax = null; foreach ($taxa as $level => $tab) { $string = isset($data[$tab['col']]) ? trim($data[$tab['col']]) : ''; $cur_encoding = mb_detect_encoding($string); if ($cur_encoding == "UTF-8" && mb_check_encoding($string, "UTF-8")) { $string = $string; } else { $string = utf8_encode($string); } if ($identifier && !empty($string)) { $identifier .= ' - '; } $identifier .= $string; if (!empty($string)) { $tax = array('name' => $string, 'identifier' => $identifier, 'level' => $level, 'label' => $cols[$tab['col']]); } } if ($tax) { $description = array(); $nb = 0; foreach ($desc as $key => $val) { $string = isset($data[$key]) ? trim($data[$key]) : ''; $cur_encoding = mb_detect_encoding($string); if ($cur_encoding == "UTF-8" && mb_check_encoding($string, "UTF-8")) { $string = $string; } else { $string = utf8_encode($string); } if (!empty($string)) { if (empty($val)) { $val = $nb + ''; $nb++; } $description[$val] = $string; } } $tax['description'] = $description; $taxon = $dm->getRepository('PlantnetDataBundle:Taxon')->findOneBy(array('module.id' => $module->getId(), 'identifier' => $tax['identifier'])); if ($taxon) { $i++; $taxon->setAttributes($tax['description']); $dm->persist($taxon); if ($i >= $batch_size) { $dm->flush(); $i = 0; $dm->clear(); gc_collect_cycles(); $module = $dm->getRepository('PlantnetDataBundle:Module')->findOneBy(array('id' => $id_module)); } } } } $dm->flush(); $dm->clear(); gc_collect_cycles(); $module = $dm->getRepository('PlantnetDataBundle:Module')->findOneBy(array('id' => $id_module)); } else { $error = 'CSV file contains some errors...'; } } } elseif ($action == 'undesc') { \MongoCursor::$timeout = -1; // remove taxa' descriptions $dm->createQueryBuilder('PlantnetDataBundle:Taxon')->update()->multiple(true)->field('module')->references($module)->field('attributes')->unsetField()->getQuery()->execute(); } $message = $error; if (empty($message)) { $message = 'Taxa were created successfully.'; } $message_mail = \Swift_Message::newInstance()->setSubject('Publish : task ended')->setFrom($this->getContainer()->getParameter('from_email_adress'))->setTo($usermail)->setBody($message . $this->getContainer()->get('templating')->render('PlantnetDataBundle:Backend\\Mail:task.txt.twig')); $this->getContainer()->get('mailer')->send($message_mail); $spool = $this->getContainer()->get('mailer')->getTransport()->getSpool(); $transport = $this->getContainer()->get('swiftmailer.transport.real'); $spool->flushQueue($transport); } $connection = new \MongoClient(); $db = $connection->{$dbname}; $db->Module->update(array('_id' => new \MongoId($id_module)), array('$set' => array('updating' => false))); }
public function auth($username, $password, $db = "admin") { if ($db === "") { if (!$this->_mongoAuth && $this->_mongoDb) { $db = $this->_mongoDb; } else { $db = "admin"; } } $server = $this->_mongoHost . ":" . $this->_mongoPort; if (!$this->_mongoPort) { $server = $this->_mongoHost; } try { $options = $this->_mongoOptions; if ($this->_mongoAuth) { $options["username"] = $username; $options["password"] = $password; $options["db"] = $db; } $this->_mongo = new RMongo($server, $options); $this->_mongo->setSlaveOkay(true); } catch (Exception $e) { if (preg_match("/authenticate/i", $e->getMessage())) { return false; } echo "Unable to connect MongoDB, please check your configurations. MongoDB said:" . $e->getMessage() . "."; exit; } // changing timeout to the new value MongoCursor::$timeout = $this->_mongoTimeout; //auth by mongo if ($this->_mongoAuth) { // "authenticate" can only be used between 1.0.1 - 1.2.11 if (RMongo::compareVersion("1.0.1") >= 0 && RMongo::compareVersion("1.2.11") < 0) { $dbs = $db; if (!is_array($dbs)) { $dbs = preg_split("/\\s*,\\s*/", $dbs); } foreach ($dbs as $db) { $ret = $this->_mongo->selectDb($db)->authenticate($username, $password); if (!$ret["ok"]) { return false; } } } } else { if ($this->_controlAuth) { if (!isset($this->_controlUsers[$username]) || $this->_controlUsers[$username] != $password) { return false; } //authenticate if (!empty($this->_mongoUser)) { // "authenticate" can only be used between 1.0.1 - 1.2.11 if (RMongo::compareVersion("1.0.1") >= 0 && RMongo::compareVersion("1.2.11") < 0) { return $this->_mongo->selectDB($db)->authenticate($this->_mongoUser, $this->_mongoPass); } } } else { //authenticate if (!empty($this->_mongoUser)) { // "authenticate" can only be used between 1.0.1 - 1.2.11 if (RMongo::compareVersion("1.0.1") >= 0 && RMongo::compareVersion("1.2.11") < 0) { return $this->_mongo->selectDB($db)->authenticate($this->_mongoUser, $this->_mongoPass); } } } } return true; }
public function timeout($ms) { parent::timeout($ms); return $this; }
/** * Wrapper method for MongoCursor::timeout(). * * @see http://php.net/manual/en/mongocursor.timeout.php * @param integer $ms * @return self */ public function timeout($ms) { $this->timeout = (int) $ms; $this->mongoCursor->timeout($ms); return $this; }
public function auth($username, $password, $db = "admin") { if ($db === "") { if (!$this->_mongoAuth && $this->_mongoDb) { $db = $this->_mongoDb; } else { $db = "admin"; } } $server = $this->_mongoHost . ":" . $this->_mongoPort; if (!$this->_mongoPort) { $server = $this->_mongoHost; } try { $this->_mongo = new Mongo($server, $this->_mongoOptions); $this->_mongo->setSlaveOkay(true); } catch (Exception $e) { echo "Unable to connect MongoDB, please check your configurations. MongoDB said:" . $e->getMessage() . "."; exit; } // changing timeout to the new value MongoCursor::$timeout = $this->_mongoTimeout; //auth by mongo if ($this->_mongoAuth) { $dbs = $db; if (!is_array($dbs)) { $dbs = preg_split("/\\s*,\\s*/", $dbs); } foreach ($dbs as $db) { $ret = $this->_mongo->selectDb($db)->authenticate($username, $password); if (!$ret["ok"]) { return false; } } } else { if ($this->_controlAuth) { if (!isset($this->_controlUsers[$username]) || $this->_controlUsers[$username] != $password) { return false; } //authenticate if (!empty($this->_mongoUser)) { return $this->_mongo->selectDB($db)->authenticate($this->_mongoUser, $this->_mongoPass); } } else { //authenticate if (!empty($this->_mongoUser)) { return $this->_mongo->selectDB($db)->authenticate($this->_mongoUser, $this->_mongoPass); } } } return true; }
/** * Constructor */ public function __construct() { global $configArray; global $basePath; $this->dataSourceSettings = parse_ini_file("{$basePath}/conf/datasources.ini", true); $this->formats = parse_ini_file("{$basePath}/conf/{$configArray['OAI-PMH']['format_definitions']}", true); $this->sets = parse_ini_file("{$basePath}/conf/{$configArray['OAI-PMH']['set_definitions']}", true); date_default_timezone_set($configArray['Site']['timezone']); $this->log = new Logger(); $mongo = new MongoClient($configArray['Mongo']['url']); $this->db = $mongo->selectDB($configArray['Mongo']['database']); MongoCursor::$timeout = isset($configArray['Mongo']['cursor_timeout']) ? $configArray['Mongo']['cursor_timeout'] : 300000; }
public function setTimeout($timeout) { if ($this->_db->compareClientVersion('1.5.3', '<')) { @(MongoCursor::$timeout = (int) $timeout); } else { // see bugs: // https://jira.mongodb.org/browse/PHP-1099 // https://jira.mongodb.org/browse/PHP-1080 } }
public function setTimeout($timeout) { MongoCursor::$timeout = (int) $timeout; }
public static function getDistinctCount($groupByColumn, $parameters = []) { global $mdb, $debug, $longQueryMS; $hashKey = "distinctCount::{$groupByColumn}:" . serialize($parameters); $result = RedisCache::get($hashKey); if ($result != null) { return $result; } if ($parameters == []) { $type = $groupByColumn == 'solarSystemID' || $groupByColumn == 'regionID' ? "system.{$groupByColumn}" : "involved.{$groupByColumn}"; $result = $mdb->getCollection('oneWeek')->distinct($type); RedisCache::set($hashKey, sizeof($result), 3600); return sizeof($result); } $query = MongoFilter::buildQuery($parameters); if (!$mdb->exists('oneWeek', $query)) { return []; } $andQuery = MongoFilter::buildQuery($parameters, false); if ($groupByColumn == 'solarSystemID' || $groupByColumn == 'regionID') { $keyField = "system.{$groupByColumn}"; } else { $keyField = "involved.{$groupByColumn}"; } $id = $type = null; if ($groupByColumn == 'solarSystemID' || $groupByColumn == 'regionID') { $type = "system.{$groupByColumn}"; } if ($type == null) { $type = "involved.{$groupByColumn}"; } $timer = new Timer(); $pipeline = []; $pipeline[] = ['$match' => $query]; if ($groupByColumn != 'solarSystemID' && $groupByColumn != 'regionID') { $pipeline[] = ['$unwind' => '$involved']; } if ($type != null && $id != null) { $pipeline[] = ['$match' => [$type => $id]]; } $pipeline[] = ['$match' => [$keyField => ['$ne' => null]]]; $pipeline[] = ['$match' => $andQuery]; $pipeline[] = ['$group' => ['_id' => '$' . $type, 'foo' => ['$sum' => 1]]]; $pipeline[] = ['$group' => ['_id' => 'total', 'value' => ['$sum' => 1]]]; if (!$debug) { MongoCursor::$timeout = -1; } $result = $mdb->getCollection('oneWeek')->aggregateCursor($pipeline); $result = iterator_to_array($result); $time = $timer->stop(); if ($time > $longQueryMS) { Log::log("Distinct Long query ({$time}ms): {$hashKey}"); } $retValue = sizeof($result) == 0 ? 0 : $result[0]['value']; RedisCache::set($hashKey, $retValue, 3600); return $retValue; }