/**
* Constructor.
*
* @param Manager $manager
* @param string $filename
* @param bool $convertDocuments
*/
public function __construct($manager, $filename, $convertDocuments = true)
{
$this->manager = $manager;
$this->filename = $filename;
$this->converter = $manager->getConverter();
$this->convertDocuments = $convertDocuments;
}
/**
* @inheritDoc
*/
public function getRouteCollectionForRequest(Request $request)
{
if (!$this->manager) {
throw new \Exception('Manager must be set to execute query to the elasticsearch');
}
$routeCollection = new RouteCollection();
$requestPath = $request->getPathInfo();
$search = new Search();
$search->addQuery(new MatchQuery('url', $requestPath));
$results = $this->manager->execute(array_keys($this->routeMap), $search, Result::RESULTS_OBJECT);
try {
foreach ($results as $document) {
$type = $this->collector->getDocumentType(get_class($document));
if (array_key_exists($type, $this->routeMap)) {
$route = new Route($document->url, ['_controller' => $this->routeMap[$type], 'document' => $document, 'type' => $type]);
$routeCollection->add('ongr_route_' . $route->getDefault('type'), $route);
} else {
throw new RouteNotFoundException(sprintf('Route for type %s% cannot be generated.', $type));
}
}
} catch (\Exception $e) {
throw new RouteNotFoundException('Document is not correct or route cannot be generated.');
}
return $routeCollection;
}
/**
* Exports es index to provided file.
*
* @param Manager $manager
* @param string $filename
* @param array $types
* @param int $chunkSize
* @param OutputInterface $output
*/
public function exportIndex(Manager $manager, $filename, $types, $chunkSize, OutputInterface $output)
{
$typesMapping = $manager->getMetadataCollector()->getMappings($manager->getConfig()['mappings']);
$typesToExport = [];
if ($types) {
foreach ($types as $type) {
if (!array_key_exists($type, $typesMapping)) {
throw new \InvalidArgumentException(sprintf('Type "%s" does not exist.', $type));
}
$typesToExport[] = $typesMapping[$type]['bundle'] . ':' . $typesMapping[$type]['class'];
}
} else {
foreach ($typesMapping as $type => $typeConfig) {
$typesToExport[] = $typeConfig['bundle'] . ':' . $typeConfig['class'];
}
}
$repo = $manager->getRepository($typesToExport);
$results = $this->getResults($repo, $chunkSize);
$progress = new ProgressBar($output, $results->count());
$progress->setRedrawFrequency(100);
$progress->start();
$metadata = ['count' => $results->count(), 'date' => date(\DateTime::ISO8601)];
$writer = $this->getWriter($this->getFilePath($filename), $metadata);
foreach ($results as $data) {
$doc = array_intersect_key($data, array_flip(['_id', '_type', '_source', 'fields']));
$writer->push($doc);
$progress->advance();
}
$writer->finalize();
$progress->finish();
$output->writeln('');
}
public function project($event)
{
$userDocument = new User();
$userDocument->setId("{$event->userId()}");
$userDocument->setPoints(0);
$this->manager->persist($userDocument);
$this->manager->commit();
}
/**
* Test for clearScroll().
*/
public function testClearScroll()
{
$esClient = $this->getMock('Elasticsearch\\Client', ['clearScroll'], [], '', false);
$esClient->expects($this->once())->method('clearScroll')->with(['scroll_id' => 'foo']);
$metadataCollector = $this->getMockBuilder('ONGR\\ElasticsearchBundle\\Mapping\\MetadataCollector')->disableOriginalConstructor()->getMock();
$converter = $this->getMockBuilder('ONGR\\ElasticsearchBundle\\Result\\Converter')->disableOriginalConstructor()->getMock();
$config = ['readonly' => false];
$manager = new Manager('test', $config, $esClient, ['index' => 'test'], $metadataCollector, $converter);
$manager->clearScroll('foo');
}
/**
* Test if manager builds correct bulk structure
*
* @param array $expected
* @param array $calls
*
* @dataProvider getTestBulkData()
*/
public function testBulk($expected, $calls)
{
$indices = $this->getMock('Elasticsearch\\Namespaces\\IndicesNamespace', [], [], '', false);
$esClient = $this->getMock('Elasticsearch\\Client', [], [], '', false);
$esClient->expects($this->once())->method('bulk')->with($expected);
$esClient->expects($this->any())->method('indices')->will($this->returnValue($indices));
$metadataCollector = $this->getMockBuilder('ONGR\\ElasticsearchBundle\\Mapping\\MetadataCollector')->disableOriginalConstructor()->getMock();
$converter = $this->getMockBuilder('ONGR\\ElasticsearchBundle\\Result\\Converter')->disableOriginalConstructor()->getMock();
$config = ['readonly' => false];
$manager = new Manager('test', $config, $esClient, ['index' => 'test'], $metadataCollector, $converter);
foreach ($calls as list($operation, $type, $query)) {
$manager->bulk($operation, $type, $query);
}
$manager->commit();
}
/**
* Exports es index to provided file.
*
* @param Manager $manager
* @param string $filename
* @param array $types
* @param int $chunkSize
* @param OutputInterface $output
*/
public function exportIndex(Manager $manager, $filename, $types, $chunkSize, OutputInterface $output)
{
$params = ['search_type' => 'scan', 'scroll' => '5m', 'size' => $chunkSize, 'source' => true, 'body' => ['query' => ['match_all' => []]], 'index' => $manager->getIndexName(), 'type' => $types];
$results = new SearchHitIterator(new SearchResponseIterator($manager->getClient(), $params));
$progress = new ProgressBar($output, $results->count());
$progress->setRedrawFrequency(100);
$progress->start();
$metadata = ['count' => $results->count(), 'date' => date(\DateTime::ISO8601)];
$writer = $this->getWriter($this->getFilePath($filename), $metadata);
foreach ($results as $data) {
$doc = array_intersect_key($data, array_flip(['_id', '_type', '_source', 'fields']));
$writer->push($doc);
$progress->advance();
}
$writer->finalize();
$progress->finish();
$output->writeln('');
}
/**
* Handles the requests and returns the json for the response body
* @param Request $request
* @param Repository $repository
* @param string $action
*
* @return string
* @throws InvalidArgumentException
*/
public function handleRequest(Request $request, Repository $repository, $action)
{
$commitSize = $this->manager->getBulkCommitSize();
$documents = $this->requestSerializer->deserializeRequest($request);
switch ($action) {
case 'create':
return $this->create($documents, $repository, $commitSize);
break;
case 'update':
return $this->update($documents, $repository, $commitSize);
break;
case 'delete':
return $this->delete($documents, $repository, $commitSize);
break;
default:
throw new InvalidArgumentException('handleRequest method can only handle `create`, `update` and `delete` actions.');
}
}
/**
* Advances scan page.
*
* @return $this
*/
protected function page()
{
if ($this->key() == $this->count() || !$this->isScrollable()) {
return $this;
}
$raw = $this->manager->scroll($this->scrollId, $this->scrollDuration, Result::RESULTS_RAW);
$this->rewind();
$this->scrollId = $raw['_scroll_id'];
$this->documents = $raw['hits']['hits'];
return $this;
}
/**
* @param string $path
*
* @return array
*/
protected function getCategoriesByPath($path)
{
$nodeIds = array_filter(explode('/', $path));
if (count($nodeIds) === 0) {
return [];
}
$search = new Search();
$search->addFilter(new IdsQuery(array_values($nodeIds)));
$results = $this->manager->execute(['AppBundle:Category'], $search);
$nodes = [];
foreach ($results as $document) {
$nodes[$document->getId()] = $document;
}
$sortedNodes = [];
foreach ($nodeIds as $nodeId) {
if (isset($nodes[$nodeId])) {
$sortedNodes[] = $nodes[$nodeId];
}
}
return $sortedNodes;
}
/**
* Factory function to create a manager instance.
*
* @param string $managerName Manager name.
* @param array $connection Connection configuration.
* @param array $analysis Analyzers, filters and tokenizers config.
* @param array $managerConfig Manager configuration.
*
* @return Manager
*/
public function createManager($managerName, $connection, $analysis, $managerConfig)
{
foreach (array_keys($analysis) as $analyzerType) {
foreach ($connection['analysis'][$analyzerType] as $name) {
$connection['settings']['analysis'][$analyzerType][$name] = $analysis[$analyzerType][$name];
}
}
unset($connection['analysis']);
$mappings = $this->metadataCollector->getClientMapping($managerConfig['mappings']);
$client = ClientBuilder::create();
$client->setHosts($connection['hosts']);
$client->setTracer($this->tracer);
if ($this->logger && $managerConfig['logger']['enabled']) {
$client->setLogger($this->logger);
}
$indexSettings = ['index' => $connection['index_name'], 'body' => array_filter(['settings' => $connection['settings'], 'mappings' => $mappings])];
$manager = new Manager($managerName, $managerConfig, $client->build(), $indexSettings, $this->metadataCollector, $this->converter);
$manager->setCommitMode($managerConfig['commit_mode']);
$manager->setBulkCommitSize($managerConfig['bulk_size']);
return $manager;
}
/**
* Converts raw array to document.
*
* @param array $rawData
* @param Manager $manager
*
* @return object
*
* @throws \LogicException
*/
public function convertToDocument($rawData, Manager $manager)
{
$types = $this->metadataCollector->getMappings($manager->getConfig()['mappings']);
if (isset($types[$rawData['_type']])) {
$metadata = $types[$rawData['_type']];
} else {
throw new \LogicException("Got document of unknown type '{$rawData['_type']}'.");
}
switch (true) {
case isset($rawData['_source']):
$rawData = array_merge($rawData, $rawData['_source']);
break;
case isset($rawData['fields']):
$rawData = array_merge($rawData, $rawData['fields']);
break;
default:
// Do nothing.
break;
}
$object = $this->assignArrayToObject($rawData, new $metadata['namespace'](), $metadata['aliases']);
return $object;
}
/**
* Validates fields if the allow_extra_fields property
* is false or allow_fields are set
*
* @param Request $request
* @param Repository $repository
* @param array $data
*
* @return array
*/
public function validateFields(Request $request, Repository $repository, $data)
{
$config = [];
$validation = [];
foreach ($this->versions as $version) {
foreach ($version['endpoints'] as $endpoint) {
if ($endpoint['repository'] == $request->attributes->get('repository')) {
$config = $endpoint;
break;
}
}
if ($config != []) {
break;
}
}
if (!$config['allow_extra_fields'] || $config['allow_fields']) {
$mapping = $this->manager->getMetadataCollector()->getMapping($repository->getClassName());
$forbiddenFields = $mapping['properties'];
if ($config['allow_fields']) {
foreach ($config['allow_fields'] as $field) {
unset($forbiddenFields[$field]);
}
}
foreach ($data as $parameter => $value) {
if (!array_key_exists($parameter, $mapping['properties']) && $parameter != '_id') {
$validation['message'] = sprintf('Property `%s` does not exist in the mapping of `%s`.', $parameter, $repository->getType());
return $validation;
}
if ($config['allow_fields'] && array_key_exists($parameter, $forbiddenFields)) {
$validation['message'] = sprintf('You are not allowed to insert or modify the field `%s` in `%s`', $parameter, $repository->getType());
return $validation;
}
}
}
return $validation;
}
/**
* @param Category $document
*
* @return array
*/
private function getChildrens($document)
{
$repository = $this->manager->getRepository('AppBundle:Category');
/** @var Search $search */
$search = $repository->createSearch();
$search->addQuery(new TermQuery('parentKey', $document->id));
$search->setSize(1000);
$categories = $repository->execute($search);
$tree = [];
/** @var Category $category */
foreach ($categories as $category) {
$tree[$category->key]['document'] = $category;
// $tree[$category->key]['children'] = $this->getChildrens($category);
}
return $tree;
}
/**
* Populates elasticsearch with data.
*
* @param Manager $manager
* @param array $data
*/
protected function populateElasticsearchWithData($manager, array $data)
{
if (!empty($data)) {
foreach ($data as $type => $documents) {
foreach ($documents as $document) {
$manager->bulk('index', $type, $document);
}
}
$manager->commit();
}
}
/**
* Executes given search.
*
* @param Search $search
* @param string $resultsType
*
* @return DocumentIterator|RawIterator|array
*
* @throws \Exception
*/
public function execute(Search $search, $resultsType = Result::RESULTS_OBJECT)
{
return $this->manager->execute([$this->type], $search, $resultsType);
}
/**
* Returns pre-built category tree.
*
* Output example:
*
* [
* 'abc' => [
* 'document' => (object) Category,
* ],
* 'def' => [
* 'document' => (object) Category,
* 'children' => [
* 'ghj' => [
* 'document' => (object) Category,
* ],
* ],
* ],
* ]
*
* @return array
*/
public function getCategoryTree()
{
$repository = $this->manager->getRepository('AppBundle:Category');
$categories = $repository->findBy([]);
return $this->buildTree($categories);
}