/** * Constructor. * * @param Manager $manager * @param string $filename * @param bool $convertDocuments */ public function __construct($manager, $filename, $convertDocuments = true) { $this->manager = $manager; $this->filename = $filename; $this->converter = $manager->getConverter(); $this->convertDocuments = $convertDocuments; }
/** * @inheritDoc */ public function getRouteCollectionForRequest(Request $request) { if (!$this->manager) { throw new \Exception('Manager must be set to execute query to the elasticsearch'); } $routeCollection = new RouteCollection(); $requestPath = $request->getPathInfo(); $search = new Search(); $search->addQuery(new MatchQuery('url', $requestPath)); $results = $this->manager->execute(array_keys($this->routeMap), $search, Result::RESULTS_OBJECT); try { foreach ($results as $document) { $type = $this->collector->getDocumentType(get_class($document)); if (array_key_exists($type, $this->routeMap)) { $route = new Route($document->url, ['_controller' => $this->routeMap[$type], 'document' => $document, 'type' => $type]); $routeCollection->add('ongr_route_' . $route->getDefault('type'), $route); } else { throw new RouteNotFoundException(sprintf('Route for type %s% cannot be generated.', $type)); } } } catch (\Exception $e) { throw new RouteNotFoundException('Document is not correct or route cannot be generated.'); } return $routeCollection; }
/** * Exports es index to provided file. * * @param Manager $manager * @param string $filename * @param array $types * @param int $chunkSize * @param OutputInterface $output */ public function exportIndex(Manager $manager, $filename, $types, $chunkSize, OutputInterface $output) { $typesMapping = $manager->getMetadataCollector()->getMappings($manager->getConfig()['mappings']); $typesToExport = []; if ($types) { foreach ($types as $type) { if (!array_key_exists($type, $typesMapping)) { throw new \InvalidArgumentException(sprintf('Type "%s" does not exist.', $type)); } $typesToExport[] = $typesMapping[$type]['bundle'] . ':' . $typesMapping[$type]['class']; } } else { foreach ($typesMapping as $type => $typeConfig) { $typesToExport[] = $typeConfig['bundle'] . ':' . $typeConfig['class']; } } $repo = $manager->getRepository($typesToExport); $results = $this->getResults($repo, $chunkSize); $progress = new ProgressBar($output, $results->count()); $progress->setRedrawFrequency(100); $progress->start(); $metadata = ['count' => $results->count(), 'date' => date(\DateTime::ISO8601)]; $writer = $this->getWriter($this->getFilePath($filename), $metadata); foreach ($results as $data) { $doc = array_intersect_key($data, array_flip(['_id', '_type', '_source', 'fields'])); $writer->push($doc); $progress->advance(); } $writer->finalize(); $progress->finish(); $output->writeln(''); }
public function project($event) { $userDocument = new User(); $userDocument->setId("{$event->userId()}"); $userDocument->setPoints(0); $this->manager->persist($userDocument); $this->manager->commit(); }
/** * Test for clearScroll(). */ public function testClearScroll() { $esClient = $this->getMock('Elasticsearch\\Client', ['clearScroll'], [], '', false); $esClient->expects($this->once())->method('clearScroll')->with(['scroll_id' => 'foo']); $metadataCollector = $this->getMockBuilder('ONGR\\ElasticsearchBundle\\Mapping\\MetadataCollector')->disableOriginalConstructor()->getMock(); $converter = $this->getMockBuilder('ONGR\\ElasticsearchBundle\\Result\\Converter')->disableOriginalConstructor()->getMock(); $config = ['readonly' => false]; $manager = new Manager('test', $config, $esClient, ['index' => 'test'], $metadataCollector, $converter); $manager->clearScroll('foo'); }
/** * Test if manager builds correct bulk structure * * @param array $expected * @param array $calls * * @dataProvider getTestBulkData() */ public function testBulk($expected, $calls) { $indices = $this->getMock('Elasticsearch\\Namespaces\\IndicesNamespace', [], [], '', false); $esClient = $this->getMock('Elasticsearch\\Client', [], [], '', false); $esClient->expects($this->once())->method('bulk')->with($expected); $esClient->expects($this->any())->method('indices')->will($this->returnValue($indices)); $metadataCollector = $this->getMockBuilder('ONGR\\ElasticsearchBundle\\Mapping\\MetadataCollector')->disableOriginalConstructor()->getMock(); $converter = $this->getMockBuilder('ONGR\\ElasticsearchBundle\\Result\\Converter')->disableOriginalConstructor()->getMock(); $config = ['readonly' => false]; $manager = new Manager('test', $config, $esClient, ['index' => 'test'], $metadataCollector, $converter); foreach ($calls as list($operation, $type, $query)) { $manager->bulk($operation, $type, $query); } $manager->commit(); }
/** * Exports es index to provided file. * * @param Manager $manager * @param string $filename * @param array $types * @param int $chunkSize * @param OutputInterface $output */ public function exportIndex(Manager $manager, $filename, $types, $chunkSize, OutputInterface $output) { $params = ['search_type' => 'scan', 'scroll' => '5m', 'size' => $chunkSize, 'source' => true, 'body' => ['query' => ['match_all' => []]], 'index' => $manager->getIndexName(), 'type' => $types]; $results = new SearchHitIterator(new SearchResponseIterator($manager->getClient(), $params)); $progress = new ProgressBar($output, $results->count()); $progress->setRedrawFrequency(100); $progress->start(); $metadata = ['count' => $results->count(), 'date' => date(\DateTime::ISO8601)]; $writer = $this->getWriter($this->getFilePath($filename), $metadata); foreach ($results as $data) { $doc = array_intersect_key($data, array_flip(['_id', '_type', '_source', 'fields'])); $writer->push($doc); $progress->advance(); } $writer->finalize(); $progress->finish(); $output->writeln(''); }
/** * Handles the requests and returns the json for the response body * @param Request $request * @param Repository $repository * @param string $action * * @return string * @throws InvalidArgumentException */ public function handleRequest(Request $request, Repository $repository, $action) { $commitSize = $this->manager->getBulkCommitSize(); $documents = $this->requestSerializer->deserializeRequest($request); switch ($action) { case 'create': return $this->create($documents, $repository, $commitSize); break; case 'update': return $this->update($documents, $repository, $commitSize); break; case 'delete': return $this->delete($documents, $repository, $commitSize); break; default: throw new InvalidArgumentException('handleRequest method can only handle `create`, `update` and `delete` actions.'); } }
/** * Advances scan page. * * @return $this */ protected function page() { if ($this->key() == $this->count() || !$this->isScrollable()) { return $this; } $raw = $this->manager->scroll($this->scrollId, $this->scrollDuration, Result::RESULTS_RAW); $this->rewind(); $this->scrollId = $raw['_scroll_id']; $this->documents = $raw['hits']['hits']; return $this; }
/** * @param string $path * * @return array */ protected function getCategoriesByPath($path) { $nodeIds = array_filter(explode('/', $path)); if (count($nodeIds) === 0) { return []; } $search = new Search(); $search->addFilter(new IdsQuery(array_values($nodeIds))); $results = $this->manager->execute(['AppBundle:Category'], $search); $nodes = []; foreach ($results as $document) { $nodes[$document->getId()] = $document; } $sortedNodes = []; foreach ($nodeIds as $nodeId) { if (isset($nodes[$nodeId])) { $sortedNodes[] = $nodes[$nodeId]; } } return $sortedNodes; }
/** * Factory function to create a manager instance. * * @param string $managerName Manager name. * @param array $connection Connection configuration. * @param array $analysis Analyzers, filters and tokenizers config. * @param array $managerConfig Manager configuration. * * @return Manager */ public function createManager($managerName, $connection, $analysis, $managerConfig) { foreach (array_keys($analysis) as $analyzerType) { foreach ($connection['analysis'][$analyzerType] as $name) { $connection['settings']['analysis'][$analyzerType][$name] = $analysis[$analyzerType][$name]; } } unset($connection['analysis']); $mappings = $this->metadataCollector->getClientMapping($managerConfig['mappings']); $client = ClientBuilder::create(); $client->setHosts($connection['hosts']); $client->setTracer($this->tracer); if ($this->logger && $managerConfig['logger']['enabled']) { $client->setLogger($this->logger); } $indexSettings = ['index' => $connection['index_name'], 'body' => array_filter(['settings' => $connection['settings'], 'mappings' => $mappings])]; $manager = new Manager($managerName, $managerConfig, $client->build(), $indexSettings, $this->metadataCollector, $this->converter); $manager->setCommitMode($managerConfig['commit_mode']); $manager->setBulkCommitSize($managerConfig['bulk_size']); return $manager; }
/** * Converts raw array to document. * * @param array $rawData * @param Manager $manager * * @return object * * @throws \LogicException */ public function convertToDocument($rawData, Manager $manager) { $types = $this->metadataCollector->getMappings($manager->getConfig()['mappings']); if (isset($types[$rawData['_type']])) { $metadata = $types[$rawData['_type']]; } else { throw new \LogicException("Got document of unknown type '{$rawData['_type']}'."); } switch (true) { case isset($rawData['_source']): $rawData = array_merge($rawData, $rawData['_source']); break; case isset($rawData['fields']): $rawData = array_merge($rawData, $rawData['fields']); break; default: // Do nothing. break; } $object = $this->assignArrayToObject($rawData, new $metadata['namespace'](), $metadata['aliases']); return $object; }
/** * Validates fields if the allow_extra_fields property * is false or allow_fields are set * * @param Request $request * @param Repository $repository * @param array $data * * @return array */ public function validateFields(Request $request, Repository $repository, $data) { $config = []; $validation = []; foreach ($this->versions as $version) { foreach ($version['endpoints'] as $endpoint) { if ($endpoint['repository'] == $request->attributes->get('repository')) { $config = $endpoint; break; } } if ($config != []) { break; } } if (!$config['allow_extra_fields'] || $config['allow_fields']) { $mapping = $this->manager->getMetadataCollector()->getMapping($repository->getClassName()); $forbiddenFields = $mapping['properties']; if ($config['allow_fields']) { foreach ($config['allow_fields'] as $field) { unset($forbiddenFields[$field]); } } foreach ($data as $parameter => $value) { if (!array_key_exists($parameter, $mapping['properties']) && $parameter != '_id') { $validation['message'] = sprintf('Property `%s` does not exist in the mapping of `%s`.', $parameter, $repository->getType()); return $validation; } if ($config['allow_fields'] && array_key_exists($parameter, $forbiddenFields)) { $validation['message'] = sprintf('You are not allowed to insert or modify the field `%s` in `%s`', $parameter, $repository->getType()); return $validation; } } } return $validation; }
/** * @param Category $document * * @return array */ private function getChildrens($document) { $repository = $this->manager->getRepository('AppBundle:Category'); /** @var Search $search */ $search = $repository->createSearch(); $search->addQuery(new TermQuery('parentKey', $document->id)); $search->setSize(1000); $categories = $repository->execute($search); $tree = []; /** @var Category $category */ foreach ($categories as $category) { $tree[$category->key]['document'] = $category; // $tree[$category->key]['children'] = $this->getChildrens($category); } return $tree; }
/** * Populates elasticsearch with data. * * @param Manager $manager * @param array $data */ protected function populateElasticsearchWithData($manager, array $data) { if (!empty($data)) { foreach ($data as $type => $documents) { foreach ($documents as $document) { $manager->bulk('index', $type, $document); } } $manager->commit(); } }
/** * Executes given search. * * @param Search $search * @param string $resultsType * * @return DocumentIterator|RawIterator|array * * @throws \Exception */ public function execute(Search $search, $resultsType = Result::RESULTS_OBJECT) { return $this->manager->execute([$this->type], $search, $resultsType); }
/** * Returns pre-built category tree. * * Output example: * * [ * 'abc' => [ * 'document' => (object) Category, * ], * 'def' => [ * 'document' => (object) Category, * 'children' => [ * 'ghj' => [ * 'document' => (object) Category, * ], * ], * ], * ] * * @return array */ public function getCategoryTree() { $repository = $this->manager->getRepository('AppBundle:Category'); $categories = $repository->findBy([]); return $this->buildTree($categories); }