/**
* The cursor constructor
* @param string|EMongoDocument $modelClass - The class name for the active record
* @param array|MongoCursor|EMongoCriteria $criteria - Either a condition array (without sort,limit and skip) or a MongoCursor Object
* @param array $fields
*/
public function __construct($modelClass, $criteria = array(), $fields = array())
{
// If $fields has something in it
if (!empty($fields)) {
$this->partial = true;
}
if (is_string($modelClass)) {
$this->modelClass = $modelClass;
$this->model = EMongoDocument::model($this->modelClass);
} elseif ($modelClass instanceof EMongoDocument) {
$this->modelClass = get_class($modelClass);
$this->model = $modelClass;
}
if ($criteria instanceof MongoCursor) {
$this->cursor = $criteria;
$this->cursor->reset();
} elseif ($criteria instanceof EMongoCriteria) {
$this->criteria = $criteria;
$this->cursor = $this->model->getCollection()->find($criteria->condition, $criteria->project)->sort($criteria->sort);
if ($criteria->skip > 0) {
$this->cursor->skip($criteria->skip);
}
if ($criteria->limit > 0) {
$this->cursor->limit($criteria->limit);
}
} else {
// Then we are doing an active query
$this->criteria = $criteria;
$this->cursor = $this->model->getCollection()->find($criteria, $fields);
}
}
/**
* Retrieve the collection object (as provided by x::find())
* @return Collection
*/
public function getNext()
{
$item = $this->cursor->getNext();
$obj = clone $this->collection;
$obj->populate($item);
return $obj;
}
public function execute(InputInterface $input, OutputInterface $output)
{
$mongoDbName = $input->getArgument('mongo-db');
$mongoCollectionName = $input->getArgument('mongo-collection');
$mongoHost = $input->getArgument('mongo-host');
$mongoPort = $input->getArgument('mongo-port');
$mongoUserName = $input->getOption('mongo-user');
$mongoPassword = $input->getOption('mongo-password');
$options = array();
if (!empty($mongoUserName)) {
$options['username'] = $mongoUserName;
}
if (!empty($mongoPassword)) {
$options['password'] = $mongoPassword;
}
$m = new MongoClient("mongodb://{$mongoHost}:{$mongoPort}", $options);
$db = $m->{$mongoDbName};
\MongoCursor::$timeout = -1;
$output->writeln("Connected to the database <info>{$mongoDbName}</info>");
$collection = $db->{$mongoCollectionName};
$cursor = $this->getCursor($collection);
$i = 0;
$output->writeln("Removing elastic-search index...");
$this->doDeleteIndex($input, $output);
$output->writeln("Creating elastic-search index...");
$this->doCreateIndex($input, $output);
$output->writeln("Setting-up elastic-search mapping...");
$this->doSetupMapping($input, $output);
$output->writeln("\nIndexing...");
$guzzle = $this->getGuzzle($input);
$flag = 0;
$nbEntries = $cursor->count(true);
$timeStart = microtime(true);
$requests = array();
while ($flag == 0) {
try {
foreach ($cursor as $obj) {
$i++;
unset($obj['_id']);
$requests[] = $guzzle->put($input->getArgument('es-type') . '/' . $i, null, json_encode($obj));
if (0 === $i % 180) {
$guzzle->send($requests);
$requests = array();
}
if (0 === $i % 10000) {
$elapsedTime = microtime(true) - $timeStart;
$entriesPerSeconds = floor($i / $elapsedTime);
$output->writeln(date('H:i:s') . "\tProgress: {$i}/{$nbEntries}\t({$entriesPerSeconds}/seconds)\t" . round($i / $nbEntries * 100) . "% \t" . "~" . $this->secsToString(($nbEntries - $i) / $entriesPerSeconds) . " left\tMemory usage : " . (memory_get_usage() >> 20) . "Mo");
}
}
$flag = 1;
} catch (Exception $ex) {
$output->writeln("Something went wrong within MongoDB: " . $ex->getMessage() . ", Retrying ...");
$flag = 0;
$cursor = getCursor($collection);
}
}
$output->writeln("{$i} entries processed. Took " . $this->secsToString(floor(microtime(true) - $timeStart)) . " seconds");
return 0;
}
public function limit($itemCountPerPage)
{
if ($this->resource instanceof \MongoCursor) {
return $this->resource->limit($itemCountPerPage);
}
return $this->resource;
}
public function sort($fields)
{
if ($this->count() > 1) {
$this->cursor->sort($fields);
}
return $this;
}
public function getReadPreference()
{
if ($this->cursor) {
return $this->cursor->getReadPreference();
}
return $this->readPreference;
}
/**
* Count the results from the query
*
* + Count the results from the current query: pass false for "all" results (disregard limit/skip)
* + Count results of a separate query: pass an array or JSON string of query parameters
*
* @since 0.3.0
*
* See [\Countable::count]
*
* @link http://www.php.net/manual/en/mongocursor.count.php \MongoCursor::count
*
* @param boolean|array|string $query
*
* @return integer
*
* @throws \Exception
*
* @uses \JSON::encodeMongo
* @uses \JSON::decode
* @uses \Profiler::start
* @uses \Profiler::stop
*/
public function count($query = true)
{
if (is_bool($query)) {
// Profile count operation for cursor
if ($this->getClientInstance()->profiling) {
$this->benchmark = Profiler::start(get_class($this->getClientInstance()) . "::{$this->db}", $this->shellQuery() . ".count(" . JSON::encodeMongo($query) . ")");
}
$this->cursor || $this->load();
$count = $this->cursor->count($query);
} else {
if (is_string($query) && $query[0] == "{") {
$query = JSON::decode($query, true);
}
$query_trans = array();
foreach ($query as $field => $value) {
$query_trans[$this->getFieldName($field)] = $value;
}
$query = $query_trans;
// Profile count operation for collection
if ($this->getClientInstance()->profiling) {
$this->benchmark = Profiler::start(get_class($this->getClientInstance()) . "::{$this->db}", "db.{$this->name}.count(" . ($query ? JSON::encodeMongo($query) : '') . ")");
}
$count = $this->getCollection()->count($query);
}
// End profiling count
if ($this->benchmark) {
// Stop the benchmark
Profiler::stop($this->benchmark);
// Clean benchmark token
$this->benchmark = null;
}
return $count;
}
/**
* Returns the current element
*
* @return array|object
*/
public function current()
{
$values = parent::current();
if (isset($values) && isset($this->collection) && $this->collection->getDocumentClass()) {
$values = $this->collection->asDocument($values, $this->lazy);
}
return $values;
}
public function testDelete()
{
$filename = 'tests/Formelsamling.pdf';
$id = $this->object->put($filename);
$this->object->delete($id);
$file = $this->object->get($id);
$this->assertNull($file);
}
public function current()
{
if ($this->collectionName === null) {
return new $this->objectType(parent::current());
} else {
return new $this->objectType($this->collectionName, parent::current());
}
}
/**
* @see CActiveDataProvider::calculateTotalItemCount()
* @return int
*/
public function calculateTotalItemCount()
{
if (!$this->_builder) {
$criteria = $this->getCriteria();
$this->_builder = new EMongoQueryBuilder($this->model, isset($criteria['condition']) && is_array($criteria['condition']) ? $criteria['condition'] : []);
}
return $this->_builder->count();
}
/**
* run the map/reduce
* @static
* @return void
*/
public static function mapReduce()
{
$map = "function () {\r\n if(arguments.callee.shipcache === undefined)\r\n {\r\n arguments.callee.shipcache = {}\r\n }\r\n if(arguments.callee.shipcache[this.victim.shipTypeID] === undefined)\r\n {\r\n arguments.callee.shipcache[this.victim.shipTypeID] = db.Kingboard_EveItem.findOne({typeID: parseInt(this.victim.shipTypeID)},{'marketGroup.parentGroup.marketGroupName':1});\r\n }\r\n var ship = arguments.callee.shipcache[this.victim.shipTypeID];\r\n var info = {}\r\n info[this.victim.shipType] = 1;\r\n info[\"total\"] = 1;\r\n if(ship != null && ship.marketGroup != null)\r\n emit(ship.marketGroup.parentGroup.marketGroupName, info);\r\n }";
$reduce = "function (k, vals) {\r\n var sums = {}\r\n var total = 0;\r\n vals.forEach(function(info) {\r\n info[\"total\"] = 0;\r\n for (var key in info)\r\n {\r\n if(sums[key] === undefined)\r\n sums[key] = 0;\r\n sums[key] += info[key];\r\n total += info[key];\r\n }\r\n });\r\n sums[\"total\"] = total;\r\n return sums;\r\n }";
// we want the map/reduce to run for as long as it takes
MongoCursor::$timeout = -1;
return King23_Mongo::mapReduce("Kingboard_Kill", __CLASS__, $map, $reduce);
}
/**
* Performs proper skip and limit to get
* data package that can be wraped in paginator
*
* @param int $perPage
* @param int $page
* @param mixed $options Options you want to pass
*/
public function getPaginator($perPage = 10, $page = 1, $options = null)
{
$this->_checkCursor();
$total = $this->_cursor->count();
$this->_cursor->skip(($page - 1) * $perPage)->limit($perPage);
$result = $this->get();
return $this->_createPaginator($result, $total, $perPage, $page, $options);
}
/**
* Create MongoCursor from string query log
*
* @param string $queryString
* @return \MongoCursor|null
*/
protected function getCursorFromQueryLog($queryString)
{
$cursor = null;
$connection = $this->panel->getDb();
$connection->open();
if ($connection->isActive) {
$queryInfo = Json::decode($queryString);
$query = $this->prepareQuery(isset($queryInfo['query']['$query']) ? $queryInfo['query']['$query'] : $queryInfo['query']);
$cursor = new \MongoCursor($connection->mongoClient, $queryInfo['ns'], $query, $queryInfo['fields']);
$cursor->limit($queryInfo['limit']);
$cursor->skip($queryInfo['skip']);
if (isset($queryInfo['query']['$orderby'])) {
$cursor->sort($queryInfo['query']['$orderby']);
}
}
return $cursor;
}
/**
* @see CActiveDataProvider::calculateTotalItemCount()
* @return int
*/
public function calculateTotalItemCount()
{
if (!$this->_cursor) {
$criteria = $this->getCriteria();
$this->_cursor = $this->model->find(isset($criteria['condition']) && is_array($criteria['condition']) ? $criteria['condition'] : array());
}
return $this->_cursor->count();
}
/**
* Ensures that the <code>next</code> points to the correct item, possibly reading from the dbCursor.
*/
private function initializeNextItem()
{
while (!$this->messagesToReturn->valid() && $this->dbCursor->hasNext()) {
$cb = $this->callback;
$this->messagesToReturn = new \ArrayIterator($cb($this->dbCursor->getNext(), $this->actualAggregateIdentifier));
}
$this->next = $this->messagesToReturn->current();
$this->messagesToReturn->next();
}
/**
*
* @param \MongoDb $db
* @param \Mongodloid_Connection $connection
*/
public function __construct(\MongoDb $db, \Mongodloid_Connection $connection)
{
parent::__construct($db, $connection);
$this->collections = Billrun_Factory::config()->getConfigValue('db.collections', array());
$timeout = Billrun_Factory::config()->getConfigValue('db.timeout', 3600000);
// default 60 minutes
Billrun_Factory::log()->log('Set database cursor timeout to: ' . $timeout, Zend_Log::INFO);
MongoCursor::$timeout = $timeout;
}
public function getCollection($collectionName)
{
MongoCursor::$slaveOkay = true;
//if empty($this->dbName) throw error;
$dbName = $this->dbName;
Yii::import('application.extensions.mp.db.mongodb.*');
$this->_collection = new MongoMSCollection($this->_writeServer->{$dbName}, $collectionName);
$this->_collection->addSlaves($this->_readServers);
return $this->_collection;
}
public function testMD5()
{
$mid = $this->object->storeFile("mongo.c");
$gid = $this->object->storeFile("gridfs.c");
$mongo = $this->object->findOne(array("_id" => $mid));
$gridfs = $this->object->findOne(array("_id" => $gid));
$this->assertNotEquals($mongo->file['md5'], $gridfs->file['md5']);
$this->assertEquals($mongo->file['md5'], md5(file_get_contents("mongo.c")));
$this->assertEquals($gridfs->file['md5'], md5(file_get_contents("gridfs.c")));
}
public function testExplainReset()
{
$this->object->insert(array("x" => "abc"), array('safe' => true));
$cursor = $this->object->find();
$qp = $cursor->explain();
$info = $cursor->info();
$this->assertTrue(!array_key_exists('$explain', $info['query']));
$this->assertTrue(array_key_exists('$query', $info['query']));
$doc = $cursor->getNext();
$this->assertEquals('abc', $doc['x'], json_encode($doc));
}
public function testExceptionHost()
{
$host = "";
$this->object->insert(array("_id" => 1), array("safe" => true));
try {
$this->object->insert(array("_id" => 1), array("safe" => true));
} catch (MongoCursorException $e) {
$host = $e->getHost();
}
$this->assertEquals("localhost:27017", $host);
}
public function testExplainLimit()
{
$this->object->drop();
for ($i = 0; $i < 100; $i++) {
$this->object->save(array("x" => $i));
}
$q = array("x" => array('$gt' => 50));
$soft = $this->object->find($q)->limit(20)->explain();
$hard = $this->object->find($q)->limit(-20)->explain();
$this->assertEquals(20, $soft['n']);
$this->assertEquals(20, $hard['n']);
}
public function fetch($key, &$value, $timeout_version = null)
{
MongoCursor::$slaveOkay = true;
$store = self::$_mongodb->findOne(array('key' => $this->create_key($key)));
if (!is_null($store) && $timeout_version < $store['dateline']) {
if ($store['ttl'] > 0 && $store['dateline'] + $store['ttl'] < time()) {
return false;
}
$value = $store['value'];
return true;
}
return false;
}
public function testTimeout3()
{
for ($i = 0; $i < 10000; $i++) {
$this->object->insert(array("name" => "joe" . $i, "interests" => array(rand(), rand(), rand())));
}
$cmd = $this->object->db->selectCollection('$cmd');
$query = 'r = 0; cursor = db.c.find(); while (cursor.hasNext()) { x = cursor.next(); for (i=0; i<200; i++) { if (x.name == "joe"+i) { r++; } } } return r;';
$count = 0;
for ($i = 0; $i < 3; $i++) {
$cursor = $cmd->find(array('$eval' => $query))->limit(-1)->timeout(500);
try {
$x = $cursor->getNext();
$this->assertFalse(true, json_encode($x));
} catch (MongoCursorTimeoutException $e) {
$count++;
}
}
$this->assertEquals(3, $count);
$x = $this->object->findOne();
$this->assertNotNull($x);
$this->assertTrue(array_key_exists('name', $x), json_encode($x));
$this->assertTrue(array_key_exists('interests', $x), json_encode($x));
}
/**
* Apply the limit,skip,sort options to the cursor before iterating on documents
*/
protected function _apply_cursor_controls()
{
if ($this->_Cursor) {
if ($this->_cursor_controls['limit'] !== null) {
$this->_Cursor->limit($this->_cursor_controls['limit']);
}
if ($this->_cursor_controls['skip'] !== null) {
$this->_Cursor->skip($this->_cursor_controls['skip']);
}
if (!empty($this->_cursor_controls['sort'])) {
$this->_Cursor->sort($this->_cursor_controls['sort']);
}
}
}
/**
* Count the results
*
* @since v1.0.0
*/
public function count($foundOnly = FALSE)
{
$count = array();
try {
$count = $this->_cursor->count($foundOnly);
} catch (MongoCursorException $exception) {
show_error($exception->getMessage(), 500);
} catch (MongoConnectionException $exception) {
show_error($exception->getMessage(), 500);
} catch (MongoCursorTimeoutException $exception) {
show_error($exception->getMessage(), 500);
}
return $count;
}
public function configureCursor()
{
if ($this->cursor === null) {
return;
}
if ($this->getOffset() > -1) {
$this->cursor->skip($this->getOffset());
}
if ($this->getLimit() > -1) {
$this->cursor->limit($this->getLimit());
}
if ($this->getOrder() !== null) {
$this->cursor->sort($this->getOrder());
}
}
/**
*
* @param \MongoDb $db
* @param \Mongodloid_Connection $connection
*/
public function __construct(\MongoDb $db, \Mongodloid_Connection $connection)
{
parent::__construct($db, $connection);
// TODO: refatoring the collections to factory (loose coupling)
$this->collections = Billrun_Factory::config()->getConfigValue('db.collections', array());
$timeout = Billrun_Factory::config()->getConfigValue('db.timeout', 3600000);
// default 60 minutes
if ($this->compareClientVersion('1.5.3', '<')) {
Billrun_Factory::log()->log('Set database cursor timeout to: ' . $timeout, Zend_Log::INFO);
@(MongoCursor::$timeout = $timeout);
} else {
// see also bugs:
// https://jira.mongodb.org/browse/PHP-1099
// https://jira.mongodb.org/browse/PHP-1080
$db->setWriteConcern($db->getWriteConcern()['w'], $timeout);
}
}
/**
* Test global timeouts and native mappings
*/
public function testGlobalTimeout()
{
// Create collection
$default_timeout = \MongoCursor::$timeout;
$collection = $this->getTestCollection();
// Create a cursor object
$cursor = $collection->find();
$native_cursor = $cursor->native;
// Assert cursor timeouts are updated
$this->assertEquals($cursor::$timeout, $native_cursor::$timeout);
// Check that static timeouts are binded
$default_timeout = \MongoCursor::$timeout;
$cursor::$timeout = $default_timeout / 2;
$this->assertEquals(\MongoCursor::$timeout, $cursor::$timeout);
$this->assertEquals(\MongoMinify\Cursor::$timeout, $cursor::$timeout);
\MongoCursor::$timeout = 200;
$this->assertEquals(\MongoCursor::$timeout, \MongoMinify\Cursor::$timeout);
$this->assertEquals(\MongoMinify\Cursor::$timeout, 200);
\MongoCursor::$timeout = $default_timeout;
}
public function as_array($key = null, $value = null)
{
$return_value = array();
while ($data = parent::getNext()) {
if ($key === null && $value === null) {
$return_value[] = $this->load_model($data);
} elseif ($key === null) {
if (array_key_exists($value, $data)) {
$return_value[] = $data[$value];
}
} elseif ($value === null) {
if (array_key_exists($key, $data)) {
$return_value[$data[$key]] = $this->load_model($data);
}
} else {
if (array_key_exists($key, $data) && array_key_exists($value, $data)) {
$return_value[(string) $data[$key]] = $data[(string) $value];
}
}
}
return $return_value;
}
