public function init()
{
// Check for authentication
if (!empty($this->loader['username']) && !empty($this->loader['password'])) {
$auth = $this->loader['username'] . ':' . $this->loader['password'] . '@';
$parts = parse_url($this->loader['host']);
if (empty($parts['scheme'])) {
$this->loader['host'] = 'http://' . $auth . $this->loader['host'];
} elseif ($parts['scheme'] == 'https') {
$schemeless_url = str_replace('https://', '', $this->loader['host']);
$this->loader['host'] = 'https://' . $auth . $schemeless_url;
} else {
$schemeless_url = str_replace('http://', '', $this->loader['host']);
$this->loader['host'] = 'http://' . $auth . $schemeless_url;
}
}
$this->type = $this->loader['es_type'];
$this->client = new Client(['host' => $this->loader['host'], 'port' => $this->loader['port']]);
$this->index = new Index($this->client, $this->loader['es_index']);
if (!$this->index->exists()) {
$this->index->create();
}
$this->log("The ElasticSearch client is configured to write to the index " . $this->loader['es_index'] . " with the " . $this->loader['es_type'] . " type.", 'info');
$this->type = $this->index->getType($this->loader['es_type']);
// Define mapping
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($this->type);
// Set mapping
$mapping->setProperties(array(self::$ETL_TIMESTAMP => array('type' => 'date')));
// Send mapping to type
$mapping->send();
}
protected function execute(InputInterface $input, OutputInterface $output)
{
ini_set('memory_limit', -1);
$output->writeln('--- vidal:autocomplete_clinic started');
$em = $this->getContainer()->get('doctrine')->getManager('drug');
$names = $em->getRepository('VidalDrugBundle:ClPhGroups')->getNames();
$elasticaClient = new \Elastica\Client();
$elasticaIndex = $elasticaClient->getIndex('website');
$elasticaType = $elasticaIndex->getType('autocomplete_clinic');
# delete if exists
if ($elasticaType->exists()) {
$elasticaType->delete();
}
# Define mapping
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($elasticaType);
# Set mapping
$mapping->setProperties(array('id' => array('type' => 'integer', 'include_in_all' => FALSE), 'name' => array('type' => 'string', 'include_in_all' => TRUE)));
# Send mapping to type
$mapping->send();
# записываем на сервер документы автодополнения
$documents = array();
for ($i = 0, $c = count($names); $i < $c; $i++) {
$documents[] = new \Elastica\Document($i + 1, array('name' => $names[$i]));
if ($i && $i % 500 == 0) {
$elasticaType->addDocuments($documents);
$elasticaType->getIndex()->refresh();
$documents = array();
$output->writeln("... + {$i}");
}
}
$elasticaType->addDocuments($documents);
$elasticaType->getIndex()->refresh();
$output->writeln("+++ vidal:autocomplete_clinic loaded {$i} documents!");
}
protected function execute(InputInterface $input, OutputInterface $output)
{
ini_set('memory_limit', -1);
$output->writeln('--- vidal:autocomplete_product started');
$em = $this->getContainer()->get('doctrine')->getManager('drug');
$names = array();
$products = $em->createQuery("\n\t\t\tSELECT p.ProductID, p.RusName, p.ZipInfo\n\t\t\tFROM VidalDrugBundle:Product p\n\t\t\tWHERE p.MarketStatusID IN (1,2,7)\n\t\t\t\tAND p.ProductTypeCode IN ('DRUG','GOME')\n\t\t\t\tAND p.inactive = FALSE\n\t\t\tORDER BY p.RusName ASC\n\t\t")->getResult();
foreach ($products as $p) {
$names[] = $p['ProductID'] . ' ' . $this->strip($p['RusName']);
}
$elasticaClient = new \Elastica\Client();
$elasticaIndex = $elasticaClient->getIndex('website');
$elasticaType = $elasticaIndex->getType('autocomplete_product');
# delete if exists
if ($elasticaType->exists()) {
$elasticaType->delete();
}
# Define mapping
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($elasticaType);
# Set mapping
$mapping->setProperties(array('name' => array('type' => 'string', 'include_in_all' => TRUE)));
# Send mapping to type
$mapping->send();
$total = count($products);
for ($i = 0; $i < $total; $i++) {
$id = $products[$i]['ProductID'];
$name = $this->strip($products[$i]['RusName']) . ' ' . $id;
$document = new \Elastica\Document($id, array('name' => $name));
$elasticaType->addDocument($document);
$elasticaType->getIndex()->refresh();
}
$output->writeln("+++ vidal:autocomplete_product loaded {$i} documents!");
}
protected function execute(InputInterface $input, OutputInterface $output)
{
ini_set('memory_limit', -1);
$output->writeln('--- vidal:autocomplete_article started');
$em = $this->getContainer()->get('doctrine')->getManager('drug');
$articles = $em->getRepository('VidalDrugBundle:Article')->findActive();
$elasticaClient = new \Elastica\Client();
$elasticaIndex = $elasticaClient->getIndex('website');
$elasticaType = $elasticaIndex->getType('autocomplete_article');
# delete if exists
if ($elasticaType->exists()) {
$elasticaType->delete();
}
# Define mapping
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($elasticaType);
# Set mapping
$mapping->setProperties(array('title' => array('type' => 'string', 'include_in_all' => TRUE)));
# Send mapping to type
$mapping->send();
# записываем на сервер документы автодополнения
$documents = array();
for ($i = 0; $i < count($articles); $i++) {
$documents[] = new \Elastica\Document($i + 1, array('title' => mb_strtolower($articles[$i]->getTitle(), 'utf-8')));
if ($i && $i % 500 == 0) {
$elasticaType->addDocuments($documents);
$elasticaType->getIndex()->refresh();
$documents = array();
}
}
$elasticaType->addDocuments($documents);
$elasticaType->getIndex()->refresh();
$output->writeln("+++ vidal:autocomplete_article completed!");
}
private function prepareSearchData()
{
$client = $this->_getClient();
$index = $client->getIndex('has_child_test');
$index->create(array(), true);
$parentType = $index->getType('parent');
$childType = $index->getType('child');
$childMapping = new \Elastica\Type\Mapping($childType);
$childMapping->setParent('parent');
$childMapping->send();
$altType = $index->getType('alt');
$altDoc = new Document('alt1', array('name' => 'altname'));
$altType->addDocument($altDoc);
$parent1 = new Document('parent1', array('id' => 'parent1', 'user' => 'parent1', 'email' => '*****@*****.**'));
$parentType->addDocument($parent1);
$parent2 = new Document('parent2', array('id' => 'parent2', 'user' => 'parent2', 'email' => '*****@*****.**'));
$parentType->addDocument($parent2);
$child1 = new Document('child1', array('id' => 'child1', 'user' => 'child1', 'email' => '*****@*****.**'));
$child1->setParent('parent1');
$childType->addDocument($child1);
$child2 = new Document('child2', array('id' => 'child2', 'user' => 'child2', 'email' => '*****@*****.**'));
$child2->setParent('parent2');
$childType->addDocument($child2);
$child3 = new Document('child3', array('id' => 'child3', 'user' => 'child3', 'email' => '*****@*****.**', 'alt' => array(array('name' => 'testname'))));
$child3->setParent('parent2');
$childType->addDocument($child3);
$index->refresh();
return $index;
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$output->writeln('--- veterinar:autocomplete started');
$em = $this->getContainer()->get('doctrine')->getManager('veterinar');
$productNames = $em->getRepository('VidalVeterinarBundle:Product')->findProductNames();
$names = array_unique($productNames);
sort($names);
$elasticaClient = new \Elastica\Client();
$elasticaIndex = $elasticaClient->getIndex('website');
$elasticaType = $elasticaIndex->getType('veterinar_autocomplete');
if ($elasticaType->exists()) {
$elasticaType->delete();
}
// Define mapping
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($elasticaType);
// Set mapping
$mapping->setProperties(array('id' => array('type' => 'integer', 'include_in_all' => FALSE), 'name' => array('type' => 'string', 'include_in_all' => TRUE)));
// Send mapping to type
$mapping->send();
# записываем на сервер документы автодополнения
$documents = array();
for ($i = 0; $i < count($names); $i++) {
$documents[] = new \Elastica\Document($i + 1, array('name' => $names[$i]));
if ($i && $i % 500 == 0) {
$elasticaType->addDocuments($documents);
$elasticaType->getIndex()->refresh();
}
}
$output->writeln("+++ veterinar:autocomplete loaded {$i} documents!");
}
protected function execute(InputInterface $input, OutputInterface $output)
{
ini_set('memory_limit', -1);
$output->writeln('--- vidal:autocomplete started');
$em = $this->getContainer()->get('doctrine')->getManager('drug');
$productNames = $em->getRepository('VidalDrugBundle:Product')->findAutocomplete();
$moleculeNames = $em->getRepository('VidalDrugBundle:Molecule')->findAutocomplete();
$companyNames = $em->getRepository('VidalDrugBundle:Company')->findAutocomplete();
$atcNames = $em->getRepository('VidalDrugBundle:ATC')->findAutocomplete();
$elasticaClient = new \Elastica\Client();
$elasticaIndex = $elasticaClient->getIndex('website');
$elasticaType = $elasticaIndex->getType('autocomplete');
if ($elasticaType->exists()) {
$elasticaType->delete();
}
// Define mapping
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($elasticaType);
// Set mapping
$mapping->setProperties(array('name' => array('type' => 'string', 'include_in_all' => TRUE), 'type' => array('type' => 'string', 'include_in_all' => FALSE)));
// Send mapping to type
$mapping->send();
# записываем в ElasticSearch документы автодополнения
$this->save($productNames, 'product', $output, $elasticaType);
$this->save($moleculeNames, 'molecule', $output, $elasticaType);
$this->save($companyNames, 'company', $output, $elasticaType);
$this->save($atcNames, 'atc', $output, $elasticaType);
$output->writeln("+++ vidal:autocomplete completed!");
}
protected function execute(InputInterface $input, OutputInterface $output)
{
ini_set('memory_limit', -1);
$output->writeln('--- vidal:autocomplete_document started');
$em = $this->getContainer()->get('doctrine')->getManager('drug');
$names = array();
$documents = $em->createQuery("\n\t\t\tSELECT d.DocumentID, d.RusName\n\t\t\tFROM VidalDrugBundle:Document d\n\t\t\tWHERE d.CountryEditionCode = 'RUS'\n\t\t\tORDER BY d.RusName ASC\n\t\t")->getResult();
foreach ($documents as $document) {
$names[] = $document['DocumentID'] . ' ' . $this->strip($document['RusName']);
}
$elasticaClient = new \Elastica\Client();
$elasticaIndex = $elasticaClient->getIndex('website');
$elasticaType = $elasticaIndex->getType('autocomplete_document');
# delete if exists
if ($elasticaType->exists()) {
$elasticaType->delete();
}
# Define mapping
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($elasticaType);
# Set mapping
$mapping->setProperties(array('name' => array('type' => 'string', 'include_in_all' => TRUE)));
# Send mapping to type
$mapping->send();
for ($i = 0; $i < count($names); $i++) {
$document = new \Elastica\Document(null, array('name' => $names[$i]));
$elasticaType->addDocument($document);
$elasticaType->getIndex()->refresh();
if ($i && $i % 500 == 0) {
$output->writeln("... + {$i}");
}
}
$output->writeln("+++ vidal:autocomplete_document loaded {$i} documents!");
}
/**
* @param array $properties
* @return \Elastica\Response
*/
public function createMapping(array $properties)
{
$elasticaType = $this->index->getType($this->typeName);
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($elasticaType);
$mapping->setProperties($properties);
return $mapping->send()->isOk();
}
/**
* @return \Elastica\Type\Mapping
*/
public function send_search_mapping()
{
$type = ElasticSearch::instance()->get_type($this->_search_type());
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($type);
// Set mapping
$mapping->setProperties(array('id' => array('type' => 'integer', 'include_in_all' => FALSE), 'content' => array('type' => 'string', 'include_in_all' => TRUE)));
// Send mapping to type
$mapping->send();
}
public function createMapping()
{
// Define mapping
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($this->trackIndex->getType('track'));
// Set mapping
$mapping->setProperties(['id' => ['type' => 'integer', 'include_in_all' => false], 'album' => ['type' => 'object', 'properties' => ['id' => ['type' => 'integer', 'include_in_all' => true], 'title' => ['type' => 'string', 'include_in_all' => true]]], 'name' => ['type' => 'string', 'include_in_all' => true], 'name_not_analyzed' => ['type' => 'string', "index" => "not_analyzed"], 'playList' => ['type' => 'nested', 'properties' => ['id' => ['type' => 'integer', 'include_in_all' => true], 'name' => ['type' => 'string', 'include_in_all' => true]]], 'genre' => ['type' => 'object', 'properties' => ['id' => ['type' => 'integer', 'include_in_all' => true], 'name' => ['type' => 'string', 'include_in_all' => true]]], 'mediaType' => ['type' => 'object', 'properties' => ['id' => ['type' => 'integer', 'include_in_all' => true], 'name' => ['type' => 'string', 'include_in_all' => true]]], 'composer' => ['type' => 'string', 'include_in_all' => true]]);
// Send mapping to type
$mapping->send();
}
protected function execute(Console\Input\InputInterface $input, Console\Output\OutputInterface $output)
{
$this->output = $output;
$selectedIndexes = [$input->getArgument('index')];
if (empty($selectedIndexes)) {
$selectedIndexes = $this->askIndexes();
}
foreach ($selectedIndexes as $indexName) {
$index = $this->elastica->getIndex($indexName);
$indexAnalyzers = isset($this->indices[$indexName]['analyzers']) ? $this->indices[$indexName]['analyzers'] : [];
$indexFilters = isset($this->indices[$indexName]['filters']) ? $this->indices[$indexName]['filters'] : [];
$analyzers = [];
foreach ($indexAnalyzers as $analyzerName) {
$analyzers[$analyzerName] = $this->analyzers[$analyzerName];
}
$filters = [];
foreach ($indexFilters as $filterName) {
$filters[$filterName] = $this->filters[$filterName];
}
try {
$index->create(['analysis' => ['analyzer' => $analyzers, 'filter' => $filters]]);
$output->writeln(sprintf('Index <info>%s</info> successfully created.', $indexName));
} catch (\Elastica\Exception\ResponseException $e) {
if (strpos($e->getMessage(), 'IndexAlreadyExistsException') !== FALSE) {
$output->writeln(sprintf('<error>Index %s already exists. Please drop it first.</error>', $indexName));
}
}
$types = isset($this->indices[$indexName]['types']) ? $this->indices[$indexName]['types'] : [];
foreach ($types as $typeName) {
$properties = $this->types[$typeName]['properties'];
$params = $this->types[$typeName]['params'];
$elasticaType = $index->getType($typeName);
$elasticaMapping = new \Elastica\Type\Mapping($elasticaType, $properties);
if (is_array($params)) {
foreach ($params as $param) {
$elasticaMapping->setParam($param);
}
}
// Send mapping to type
try {
$res = $elasticaMapping->send();
$output->writeln(sprintf('Type <info>%s</info> successfully created in index <info>%s</info>', $typeName, $indexName));
} catch (\Elastica\Exception\ResponseException $e) {
if (strpos($e->getMessage(), 'nested: NullPointerException;') !== FALSE) {
$output->writeln(sprintf('<error>%s. Probably bad mapping.</error>', $e->getMessage()));
}
}
}
}
}
protected function execute(InputInterface $input, OutputInterface $output)
{
ini_set('memory_limit', -1);
$output->writeln('--- evrika:elastic_autocomlete started');
$em = $this->getContainer()->get('doctrine')->getManager();
$publications = $em->getRepository('EvrikaMainBundle:Publication')->findAutocomplete();
$events = $em->getRepository('EvrikaMainBundle:Event')->findAutocomplete();
for ($i = 0; $i < count($publications); $i++) {
$publications[$i]['title'] = trim($publications[$i]['title']);
$publications[$i]['type'] = 'publication';
}
for ($i = 0; $i < count($events); $i++) {
$events[$i]['title'] = trim($events[$i]['title']);
$events[$i]['type'] = 'event';
}
$articles = array_merge($publications, $events);
usort($articles, function ($a, $b) {
return strcmp(trim($a['title'], '"'), trim($b['title'], '"'));
});
$elasticaClient = new \Elastica\Client();
$elasticaIndex = $elasticaClient->getIndex('evrika');
$elasticaType = $elasticaIndex->getType('autocomplete');
if ($elasticaType->exists()) {
$elasticaType->delete();
}
// Define mapping
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($elasticaType);
// Set mapping
$mapping->setProperties(array('id' => array('type' => 'integer', 'include_in_all' => false), 'type' => array('type' => 'string', 'include_in_all' => false), 'title' => array('type' => 'string', 'include_in_all' => true, 'index_analyzer' => 'indexAnalyzer', 'search_analyzer' => 'searchAnalyzer')));
// Send mapping to type
$mapping->send();
# записываем на сервер документы автодополнения
$documents = array();
$count = count($articles);
for ($i = 0; $i < $count; $i++) {
$index = $articles[$i]['type'] == 'event' ? $articles[$i]['id'] + 999999 : $articles[$i]['id'];
$documents[] = new \Elastica\Document($index, array('id' => $articles[$i]['id'], 'type' => $articles[$i]['type'], 'title' => $articles[$i]['title']));
if ($i && $i % 100 == 0) {
$elasticaType->addDocuments($documents);
$elasticaType->getIndex()->refresh();
$documents = array();
}
}
$output->writeln("+++ evrika:elastic_autocomlete completed!");
}
protected function execute(Console\Input\InputInterface $input, Console\Output\OutputInterface $output)
{
$this->output = $output;
$selectedTypes = $this->askTypes();
$indexesForTypes = $this->getIndexesForTypes($selectedTypes);
$selectedIndexes = $this->askIndexes($indexesForTypes);
foreach ($selectedTypes as $typeName) {
$typeIndexes = $indexesForTypes[$typeName];
foreach ($selectedIndexes as $indexName) {
if (in_array($indexName, $typeIndexes)) {
$output->writeln(sprintf('Creating type <info>%s</info> in index <info>%s</info>', $typeName, $indexName));
$properties = $this->types[$typeName]['properties'];
$params = $this->types[$typeName]['params'];
$index = $this->elastica->getIndex($indexName);
$elasticaType = $index->getType($typeName);
$elasticaMapping = new \Elastica\Type\Mapping($elasticaType, $properties);
if (is_array($params)) {
foreach ($params as $param) {
$elasticaMapping->setParam($param);
}
}
// Send mapping to type
try {
$res = $elasticaMapping->send();
$output->writeln(sprintf('Type <info>%s</info> successfully created in index <info>%s</info>', $typeName, $indexName));
} catch (\Elastica\Exception\ResponseException $e) {
if (strpos($e->getMessage(), 'IndexMissingException') !== FALSE) {
$output->writeln(sprintf('<error>Index %1$s is missing. Please create index %1$s first.</error>', $indexName));
} elseif (strpos($e->getMessage(), 'nested: NullPointerException;') !== FALSE) {
$output->writeln(sprintf('<error>%s. Probably bad mapping.</error>', $e->getMessage()));
}
}
}
}
}
}
function createFileMapping()
{
// create a type
$index = $this->index;
$type = $index->getType('file');
// Define mapping
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($type);
$mapping->setParam('index_analyzer', 'indexAnalyzer');
$mapping->setParam('search_analyzer', 'searchAnalyzer');
// Define boost field
//$mapping->setParam('_boost', array('name' => 'chinese_name', 'null_value' => ''));
// Set mapping
$mapping->setProperties(array('id' => array('type' => 'integer', 'include_in_all' => TRUE), 'dataset_id' => array('type' => 'integer'), 'name' => array('type' => 'string'), 'size' => array('type' => 'long'), 'description' => array('type' => 'string'), 'filetype' => array('type' => 'string'), 'format' => array('type' => 'string')));
$mapping->send();
}
public function __construct()
{
//Parsing the yml config file
$yml = Yaml::parse(file_get_contents("config.yml"), false, false, true);
$this->_config = json_decode(json_encode($yml));
//Setting up PDO
try {
$this->_bdd = new \PDO(sprintf('mysql:dbname=%s;host=%s', $this->_config->mysql->bdd, $this->_config->mysql->host), $this->_config->mysql->user, $this->_config->mysql->pass);
$this->_bdd->setAttribute(\PDO::ATTR_DEFAULT_FETCH_MODE, \PDO::FETCH_OBJ);
} catch (PDOException $exception) {
echo "Unable to start database, have you imported the sample database?\n";
exit;
}
//Create the elastica client
$this->_client = new \Elastica\Client();
//Load the existing index or overwriting it if needed
$elasticaIndex = $this->_client->getIndex($this->_config->elastic->index, true);
//Setting up elasticSearch indexes
$elasticaIndex->create(array('number_of_shards' => 4, 'number_of_replicas' => 1, 'analysis' => array('analyzer' => array('indexAnalyzer' => array('type' => 'custom', 'tokenizer' => 'standard', 'filter' => array('standard', 'lowercase', 'asciifolding')), 'searchAnalyzer' => array('type' => 'custom', 'tokenizer' => 'standard', 'filter' => array('standard', 'lowercase', 'asciifolding', 'mySnowball')), 'keylower' => array('tokenizer' => 'keyword', 'filter' => 'lowercase')), 'filter' => array('mySnowball' => array('type' => 'snowball', 'language' => 'French')))), true);
//Initialyse the dataType
$elasticaType = $elasticaIndex->getType('persons');
//Mapping some properties to the type
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($elasticaType);
$mapping->setParam('index_analyzer', 'indexAnalyzer');
$mapping->setParam('search_analyzer', 'searchAnalyzer');
//Setting up the fields of the user
//Here a user have a last/first name, a phone number and multiples adresses
$mapping->setProperties(array('id' => array('type' => 'integer', 'include_in_all' => false), 'persons' => array('type' => 'object', 'properties' => array('id' => array('type' => 'integer', 'include_in_all' => false), 'lastname' => array('type' => 'string', 'include_in_all' => true), 'firstname' => array('type' => 'string', 'include_in_all' => true), 'email' => array('type' => 'string', 'include_in_all' => true), 'address' => array('type' => 'string', 'include_in_all' => true), 'city' => array('type' => 'string', 'include_in_all' => true), 'country' => array('type' => 'string', 'include_in_all' => true), 'company' => array('type' => 'string', 'include_in_all' => true))), 'tstamp' => array('type' => 'date', 'include_in_all' => true), 'location' => array('type' => 'geo_point', 'include_in_all' => true), '_boost' => array('type' => 'float', 'include_in_all' => true)));
//Save the data mapping
$mapping->send();
//Starting RabbitMq
$this->connection = new AMQPConnection($this->_config->rabbitmq->host, $this->_config->rabbitmq->port, $this->_config->rabbitmq->user, $this->_config->rabbitmq->pass);
$this->channel = $this->connection->channel();
$this->channel->queue_declare('task_queue', false, true, false, false);
}
/**
* Create new ES field mappings for the given fields from the configuration
* TODO align callback names with Config::names so we can simply call this method with the kind string
* @param array $config_fields
* @param string $kind of internal fields: meta|field|taxonomy used to call the right indexer_map filter
*/
static function _map_values($config_fields, $kind)
{
$index = self::_index(false);
$numeric = Config::option('numeric');
$notanalyzed = Config::option('not_analyzed');
foreach ($config_fields as $field) {
// set default
$props = array('type' => 'string');
// detect special field type
if (isset($numeric[$field])) {
$props['type'] = 'float';
} elseif (isset($notanalyzed[$field]) || $kind == 'taxonomy') {
$props['index'] = 'not_analyzed';
} elseif ($field == 'post_date') {
$props['type'] = 'date';
$props['format'] = 'date_time_no_millis';
} else {
$props['index'] = 'analyzed';
}
if ($props['type'] == 'string' && $props['index'] == 'analyzed') {
// provides more accurate searches
// TODO: assumes plugin users are in english
$lang = Config::apply_filters('string_language', 'english');
$props = array('type' => 'multi_field', 'fields' => array($field => $props, $lang => array_merge($props, array('analyzer' => $lang))));
}
// generic filter indexer_map_field| indexer_map_meta | indexer_map_taxonomy
$props = Config::apply_filters('indexer_map_' . $kind, $props, $field);
// also index taxonomy_name field
if ($kind == 'taxonomy') {
$tax_name_props = array('type' => 'string');
$tax_name_props = Config::apply_filters('indexer_map_taxonomy_name', $tax_name_props, $field);
}
foreach (Config::types() as $type) {
$type = $index->getType($type);
$mapping = new \Elastica\Type\Mapping($type);
$mapping->setProperties(array($field => $props));
$mapping->send();
// second mapping for taxonomy_name
if (isset($tax_name_props)) {
$mapping = new \Elastica\Type\Mapping($type);
$mapping->setProperties(array($field . '_name' => $tax_name_props));
$mapping->send();
}
}
}
}
/**
* Reads F.E.S configuration and updates ElasticSearch field mapping information (this can corrupt existing data).
* @internal
**/
static function _map($index = null)
{
$index = $index ?: self::_index(true);
foreach (Config::types() as $postType) {
$type = $index->getType($postType);
$properties = array();
self::_map_values($properties, $type, Config::taxonomies(), 'taxonomy');
self::_map_values($properties, $type, Config::fields(), 'field');
self::_map_values($properties, $type, Config::meta_fields(), 'meta');
$properties = Config::apply_filters('indexer_map', $properties, $postType);
$mapping = new \Elastica\Type\Mapping($type, $properties);
$mapping->send();
}
}
/**
* Creates the "mapping" for this searchtype, on the elasticsearch server
*
* TODO: it would be good to be able to make elasticsearch just delete the contents of one
* searchtype, perhaps identified by its mapping? But I haven't been able to figure out how
* to do that, so instead you can only delete the whole index at once.
* @param string $type
*/
private static function set_mapping($type)
{
// usr,interaction_instance,interaction_forum_post,view,group,artefact
$ES_class = 'ElasticsearchType_' . $type;
if ($ES_class::$mappingconf === false) {
return false;
}
$elasticaClient = self::make_client();
$elasticaIndex = $elasticaClient->getIndex(self::get_write_indexname());
$elasticaAnalyzer = get_config_plugin('search', 'elasticsearch', 'analyzer');
if (!isset($elasticaAnalyzer)) {
$elasticaAnalyzer = 'mahara_analyzer';
}
// Load type
$elasticaType = $elasticaIndex->getType($type);
// Define mapping
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($elasticaType);
// we use mahara_analyzer created through elastica
$mapping->setParam('index_analyzer', $elasticaAnalyzer);
$mapping->setParam('search_analyzer', $elasticaAnalyzer);
// Define boost field
//$mapping->setParam('_boost', array('name' => '_boost', 'null_value' => 1.0));
// Set mapping
$mapping->setProperties($ES_class::$mappingconf);
// Send mapping to type
$mapping->send();
}
/**
* @dataProvider providerTransport
* @group benchmark
*/
public function testGetMapping(array $config, $transport)
{
$this->_checkTransport($config, $transport);
$client = $this->_getClient($config);
$index = $client->getIndex('benchmark');
$index->create(array(), true);
$type = $index->getType('mappingTest');
// Define mapping
$mapping = new \Elastica\Type\Mapping();
$mapping->setParam('_boost', array('name' => '_boost', 'null_value' => 1.0));
$mapping->setProperties(array('id' => array('type' => 'integer', 'include_in_all' => false), 'user' => array('type' => 'object', 'properties' => array('name' => array('type' => 'string', 'include_in_all' => true), 'fullName' => array('type' => 'string', 'include_in_all' => true))), 'msg' => array('type' => 'string', 'include_in_all' => true), 'tstamp' => array('type' => 'date', 'include_in_all' => false), 'location' => array('type' => 'geo_point', 'include_in_all' => false), '_boost' => array('type' => 'float', 'include_in_all' => false)));
$type->setMapping($mapping);
$index->refresh();
$times = array();
for ($i = 0; $i < $this->_max; ++$i) {
$response = $type->request('_mapping', \Elastica\Request::GET);
$times[] = $response->getQueryTime();
}
self::logResults('get mapping', $transport, $times);
}
/**
*
* This function creates the mapping on particular type/module and field.
* Ths can be used when user changes the field settings (like boost level) in Studio.
* index must exist before calling this function.
*
* @param $module module name
* @param $fieldDefs field name of the module
*
* @return boolean true if mapping successfully created, false otherwise
*/
private function setFieldMapping($module, $fieldDefs)
{
$properties = $this->constructIndexMappingProperties($fieldDefs);
// add module field which is automatically added in the indexed documents
$properties['module'] = array('index' => 'not_analyzed', 'type' => 'string');
if (is_array($properties) && count($properties) > 0) {
$indexList = $this->sse->getAllIndexes($module);
foreach ($indexList as $indexName) {
$index = new \Elastica\Index($this->sse->getClient(), $indexName);
$GLOBALS['log']->debug("elastic field mapping module={$module}, index={$indexName}");
$type = new \Elastica\Type($index, $module);
$mapping = new \Elastica\Type\Mapping($type, $properties);
$mapping->setProperties($properties);
try {
$mapping->send();
} catch (\Elastica\Exception\ResponseException $e) {
$GLOBALS['log']->fatal("Elastic exception when creating mapping, message= " . $e->getMessage());
return false;
}
}
}
return true;
}
/**
* Return default search fields mapping for node translations
*
* @param \Elastica\Index $index
* @param string $lang
*
* @return \Elastica\Type\Mapping
*/
protected function getMapping(\Elastica\Index $index, $lang = 'en')
{
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($index->getType($this->indexType . '_' . $lang));
$mapping->setParam('analyzer', 'index_analyzer_' . $lang);
$mapping->setParam('_boost', array('name' => '_boost', 'null_value' => 1.0));
$mapping->setProperties(array('node_id' => array('type' => 'integer', 'include_in_all' => false, 'index' => 'not_analyzed'), 'nodetranslation_id' => array('type' => 'integer', 'include_in_all' => false, 'index' => 'not_analyzed'), 'nodeversion_id' => array('type' => 'integer', 'include_in_all' => false, 'index' => 'not_analyzed'), 'title' => array('type' => 'string', 'include_in_all' => true), 'lang' => array('type' => 'string', 'include_in_all' => true, 'index' => 'not_analyzed'), 'slug' => array('type' => 'string', 'include_in_all' => false, 'index' => 'not_analyzed'), 'type' => array('type' => 'string', 'include_in_all' => false, 'index' => 'not_analyzed'), 'page_class' => array('type' => 'string', 'include_in_all' => false, 'index' => 'not_analyzed'), 'contentanalyzer' => array('type' => 'string', 'include_in_all' => true, 'index' => 'not_analyzed'), 'content' => array('type' => 'string', 'include_in_all' => true), 'created' => array('type' => 'date', 'include_in_all' => false, 'index' => 'not_analyzed'), 'updated' => array('type' => 'date', 'include_in_all' => false, 'index' => 'not_analyzed'), 'view_roles' => array('type' => 'string', 'include_in_all' => true, 'index' => 'not_analyzed', 'index_name' => 'view_role'), '_boost' => array('type' => 'float', 'include_in_all' => false)));
return $mapping;
}
/**
* Reads F.E.S configuration and updates ElasticSearch field mapping information (this can corrupt existing data).
* @internal
**/
static function _map()
{
$index = self::_index(false);
foreach (Config::types() as $type) {
$type = $index->getType($type);
$properties = array();
self::_map_values($properties, $type, Config::taxonomies(), 'taxonomy');
self::_map_values($properties, $type, Config::fields(), 'field');
self::_map_values($properties, $type, Config::meta_fields(), 'meta');
$mapping = new \Elastica\Type\Mapping($type, $properties);
$mapping->send();
}
}
/**
* @param bool|null $recreate
*/
protected function _createIndex($recreate = null)
{
$this->getIndex()->create($this->_indexParams, $recreate);
$mapping = new Elastica\Type\Mapping($this->getType(), $this->_mapping);
$mapping->setSource(array('enabled' => $this->_source));
$mapping->send();
}
/**
* Creates or updates Elasticsearch index.
*
* @link http://www.elasticsearch.org/guide/reference/mapping/core-types.html
* @link http://www.elasticsearch.org/guide/reference/mapping/multi-field-type.html
* @return NanoWebG_ElasticSearch_Model_Resource_Engine_Elasticsearch_Client
* @throws Exception
*/
protected function _prepareIndex()
{
try {
$indexSettings = array('settings' => $this->_getIndexSettings());
$index = $this->getIndex($this->_index);
// Using custom _indexExists() method because of a bug in index creation when
// calling exists() method on $index (ignores settings).
if (!$this->_indexExists($this->_index)) {
$indexSettings['settings']['number_of_shards'] = (int) $this->getConfig('number_of_shards');
$index->create($indexSettings);
} else {
$index->setSettings(array('settings' => array('number_of_replicas' => $indexSettings['settings']['number_of_replicas'])));
}
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($index->getType('product'));
$mapping->setProperties($this->_getIndexProperties());
Mage::dispatchEvent('nanowebg_elasticsearch_mapping_send_before', array('client' => $this, 'mapping' => $mapping));
$mapping->send();
} catch (Exception $e) {
Mage::getSingleton('adminhtml/session')->addError($e->getMessage());
throw $e;
}
return $this;
}
/**
* @param \Elastica\Type $type
* @param bool $reset
*/
public function createElasticType($type = null, $reset = false)
{
if (method_exists($this->owner, 'createElasticType')) {
$this->owner->createElasticType($type, $reset);
return;
}
if ($type && $type->exists() && $reset) {
$type->delete();
}
!$type && ($type = $this->getElasticIndex()->getType($this->elasticTypeName));
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($type)->setProperties($this->elasticProperties())->setParam('dynamic', 'strict')->send();
}
/**
* @param AJXP_Node $ajxpNode
* @throws Exception
*/
public function createIndexedDocument($ajxpNode)
{
$ajxpNode->loadNodeInfo();
/*
$ext = strtolower(pathinfo($ajxpNode->getLabel(), PATHINFO_EXTENSION));
$parseContent = $this->indexContent;
if ($parseContent && $ajxpNode->bytesize > $this->getFilteredOption("PARSE_CONTENT_MAX_SIZE")) {
$parseContent = false;
}
if ($parseContent && in_array($ext, explode(",",$this->getFilteredOption("PARSE_CONTENT_HTML")))) {
$doc = @Zend_Search_Lucene_Document_Html::loadHTMLFile($ajxpNode->getUrl());
} elseif ($parseContent && $ext == "docx" && class_exists("Zend_Search_Lucene_Document_Docx")) {
$realFile = call_user_func(array($ajxpNode->wrapperClassName, "getRealFSReference"), $ajxpNode->getUrl());
$doc = @Zend_Search_Lucene_Document_Docx::loadDocxFile($realFile);
} elseif ($parseContent && $ext == "docx" && class_exists("Zend_Search_Lucene_Document_Pptx")) {
$realFile = call_user_func(array($ajxpNode->wrapperClassName, "getRealFSReference"), $ajxpNode->getUrl());
$doc = @Zend_Search_Lucene_Document_Pptx::loadPptxFile($realFile);
} elseif ($parseContent && $ext == "xlsx" && class_exists("Zend_Search_Lucene_Document_Xlsx")) {
$realFile = call_user_func(array($ajxpNode->wrapperClassName, "getRealFSReference"), $ajxpNode->getUrl());
$doc = @Zend_Search_Lucene_Document_Xlsx::loadXlsxFile($realFile);
} else {
$doc = new Zend_Search_Lucene_Document();
}
if($doc == null) throw new Exception("Could not load document");
*/
$mapping_properties = array();
/* we construct the array that will contain all the data for the document we create */
$data = array();
$data["node_url"] = $ajxpNode->getUrl();
$data["node_path"] = str_replace("/", "AJXPFAKESEP", $ajxpNode->getPath());
$data["basename"] = basename($ajxpNode->getPath());
$data["ajxp_node"] = "yes";
$data["ajxp_scope"] = "shared";
$data["serialized_metadata"] = base64_encode(serialize($ajxpNode->metadata));
if (isset($ajxpNode->indexableMetaKeys["shared"])) {
foreach ($ajxpNode->indexableMetaKeys["shared"] as $sharedField) {
if ($ajxpNode->{$sharedField}) {
$data[$sharedField] = $ajxpNode->{$sharedField};
}
}
}
foreach ($this->metaFields as $field) {
if ($ajxpNode->{$field} != null) {
$data["ajxp_meta_{$field}"] = $ajxpNode->{$field};
}
}
/*
We want some fields not to be analyzed when they are indexed.
To achieve this we have to dynamically define a mapping.
When you define a mapping you have to set the type of data you will put
in each field and you can define other parameters.
Here we want some fields not to be analyzed so we just precise the
property index and set it to "not_analyzed".
*/
foreach ($data as $key => $value) {
if ($key == "node_url" || ($key = "node_path")) {
$mapping_properties[$key] = array("type" => "string", "index" => "not_analyzed");
} else {
$type = gettype($value);
if ($type != "integer" && $type != "boolean" && $type != "double") {
$type = "string";
}
$mapping_properties[$key] = array("type" => $type, "index" => "simple");
}
}
$mapping = new Elastica\Type\Mapping();
$mapping->setType($this->currentType);
$mapping->setProperties($mapping_properties);
$mapping->send();
$doc = new Elastica\Document($this->nextId, $data);
$this->nextId++;
/*if (isSet($ajxpNode->indexableMetaKeys["user"]) && count($ajxpNode->indexableMetaKeys["user"]) && AuthService::usersEnabled() && AuthService::getLoggedUser() != null) {
$privateDoc = new Zend_Search_Lucene_Document();
$privateDoc->addField(Zend_Search_Lucene_Field::Keyword("node_url", $ajxpNode->getUrl()), SystemTextEncoding::getEncoding());
$privateDoc->addField(Zend_Search_Lucene_Field::Keyword("node_path", str_replace("/", "AJXPFAKESEP", $ajxpNode->getPath())), SystemTextEncoding::getEncoding());
$privateDoc->addField(Zend_Search_Lucene_Field::Keyword("ajxp_scope", "user"));
$privateDoc->addField(Zend_Search_Lucene_Field::Keyword("ajxp_user", AuthService::getLoggedUser()->getId()));
foreach ($ajxpNode->indexableMetaKeys["user"] as $userField) {
if ($ajxpNode->$userField) {
$privateDoc->addField(Zend_search_Lucene_Field::keyword($userField, $ajxpNode->$userField));
}
}
$privateDoc->addField(Zend_Search_Lucene_Field::Binary("serialized_metadata", $serializedMeta));
$index->addDocument($privateDoc);
}
/*
if ($parseContent && in_array($ext, explode(",",$this->getFilteredOption("PARSE_CONTENT_TXT")))) {
$doc->addField(Zend_Search_Lucene_Field::unStored("body", file_get_contents($ajxpNode->getUrl())));
}
$unoconv = $this->getFilteredOption("UNOCONV");
if ($parseContent && !empty($unoconv) && in_array($ext, array("doc", "odt", "xls", "ods"))) {
$targetExt = "txt";
$pipe = false;
if (in_array($ext, array("xls", "ods"))) {
$targetExt = "csv";
} else if (in_array($ext, array("odp", "ppt"))) {
$targetExt = "pdf";
$pipe = true;
}
$realFile = call_user_func(array($ajxpNode->wrapperClassName, "getRealFSReference"), $ajxpNode->getUrl());
$unoconv = "HOME=".AJXP_Utils::getAjxpTmpDir()." ".$unoconv." --stdout -f $targetExt ".escapeshellarg($realFile);
if ($pipe) {
$newTarget = str_replace(".$ext", ".pdf", $realFile);
$unoconv.= " > $newTarget";
register_shutdown_function("unlink", $newTarget);
}
$output = array();
exec($unoconv, $output, $return);
if (!$pipe) {
$out = implode("\n", $output);
$enc = 'ISO-8859-1';
$asciiString = iconv($enc, 'ASCII//TRANSLIT//IGNORE', $out);
$doc->addField(Zend_Search_Lucene_Field::unStored("body", $asciiString));
} else {
$ext = "pdf";
}
}
$pdftotext = $this->getFilteredOption("PDFTOTEXT");
if ($parseContent && !empty($pdftotext) && in_array($ext, array("pdf"))) {
$realFile = call_user_func(array($ajxpNode->wrapperClassName, "getRealFSReference"), $ajxpNode->getUrl());
if ($pipe && isset($newTarget) && is_file($newTarget)) {
$realFile = $newTarget;
}
$cmd = $pdftotext." ".escapeshellarg($realFile)." -";
$output = array();
exec($cmd, $output, $return);
$out = implode("\n", $output);
$enc = 'UTF8';
$asciiString = iconv($enc, 'ASCII//TRANSLIT//IGNORE', $out);
$doc->addField(Zend_Search_Lucene_Field::unStored("body", $asciiString));
}
*/
/* we update the last id in the file */
$file = fopen($this->lastIdPath, "w");
fputs($file, $this->nextId - 1);
fclose($file);
$this->currentType->addDocument($doc);
}
/**
* @param AJXP_Node $ajxpNode
* @throws Exception
*/
public function createIndexedDocument($ajxpNode)
{
$ajxpNode->loadNodeInfo();
$parseContent = $this->indexContent;
if ($parseContent && $ajxpNode->bytesize > $this->getFilteredOption("PARSE_CONTENT_MAX_SIZE")) {
$parseContent = false;
}
$data = array();
$data["node_url"] = $ajxpNode->getUrl();
$data["node_path"] = str_replace("/", "AJXPFAKESEP", $ajxpNode->getPath());
$data["basename"] = basename($ajxpNode->getPath());
$data["ajxp_node"] = "yes";
$data["ajxp_scope"] = "shared";
$data["serialized_metadata"] = base64_encode(serialize($ajxpNode->metadata));
$data["ajxp_modiftime"] = date("Ymd", $ajxpNode->ajxp_modiftime);
$data["ajxp_bytesize"] = $ajxpNode->bytesize;
$ajxpMime = $ajxpNode->ajxp_mime;
if (empty($ajxpMime)) {
$data["ajxp_mime"] = pathinfo($ajxpNode->getLabel(), PATHINFO_EXTENSION);
} else {
$data["ajxp_mime"] = $ajxpNode->ajxp_mime;
}
if (isset($ajxpNode->indexableMetaKeys["shared"])) {
foreach ($ajxpNode->indexableMetaKeys["shared"] as $sharedField) {
if ($ajxpNode->{$sharedField}) {
$data[$sharedField] = $ajxpNode->{$sharedField};
}
}
}
foreach ($this->metaFields as $field) {
if ($ajxpNode->{$field} != null) {
$data["ajxp_meta_{$field}"] = $ajxpNode->{$field};
}
}
if ($parseContent) {
$body = $this->extractIndexableContent($ajxpNode);
if (!empty($body)) {
$data["body"] = $body;
}
}
$mapping = new Elastica\Type\Mapping();
$mapping->setType($this->currentType);
$mapping->setProperties($this->dataToMappingProperties($data));
$mapping->send();
$doc = new Elastica\Document($this->nextId, $data);
$this->currentType->addDocument($doc);
$this->nextId++;
if (isset($ajxpNode->indexableMetaKeys["user"]) && count($ajxpNode->indexableMetaKeys["user"]) && AuthService::usersEnabled() && AuthService::getLoggedUser() != null) {
$userData = array("ajxp_scope" => "user", "user" => AuthService::getLoggedUser()->getId(), "serialized_metadata" => $data["serialized_metadata"], "node_url" => $data["node_url"], "node_path" => $data["node_path"]);
$userData["ajxp_user"] = AuthService::getLoggedUser()->getId();
foreach ($ajxpNode->indexableMetaKeys["user"] as $userField) {
if ($ajxpNode->{$userField}) {
$userData[$userField] = $ajxpNode->{$userField};
}
}
$mapping = new Elastica\Type\Mapping();
$mapping->setType($this->currentType);
$mapping->setProperties($this->dataToMappingProperties($userData));
$mapping->send();
$doc = new Elastica\Document($this->nextId, $userData);
$this->currentType->addDocument($doc);
$this->nextId++;
}
/* we update the last id in the file */
$file = fopen($this->lastIdPath, "w");
fputs($file, $this->nextId - 1);
fclose($file);
}
protected function updateMapping($indexName, $typeName, $mapping = [])
{
$elasticaType = $this->getClient()->getIndex($indexName)->getType($typeName);
// Define mapping
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($elasticaType);
$mapping->setParam('index_analyzer', 'indexAnalyzer');
$mapping->setParam('search_analyzer', 'searchAnalyzer');
// Define boost field
$mapping->setParam('_boost', array('name' => '_boost', 'null_value' => 1.0));
// Set mapping
$mapping->setProperties(array('id' => array('type' => 'integer', 'include_in_all' => false), 'user' => array('type' => 'object', 'properties' => array('name' => array('type' => 'string', 'include_in_all' => true), 'fullName' => array('type' => 'string', 'include_in_all' => true))), 'msg' => array('type' => 'string', 'include_in_all' => true), 'tstamp' => array('type' => 'date', 'include_in_all' => false), 'location' => array('type' => 'geo_point', 'include_in_all' => false), '_boost' => array('type' => 'float', 'include_in_all' => false)));
// Send mapping to type
$mapping->send();
}
protected function mapping()
{
// Set mapping only if mapping is set and if type does not exist
if ($this->mapping && !$this->typeExist($this->current_type)) {
$ESmapping = new \Elastica\Type\Mapping();
$ESmapping->setType($this->elasticaType);
//If Language supported by ES
if (array_key_exists($this->lang, $this->langAnalyzer)) {
$ESmapping->setParam("index_analyzer", $this->lang . "_Analyzer");
$ESmapping->setParam("search_analyzer", $this->lang . "_Analyzer");
} else {
$ESmapping->setParam("index_analyzer", "default");
$ESmapping->setParam("search_analyzer", "default");
}
if ($this->boost == "") {
$this->boost = 1.0;
}
$ESmapping->setParam('_boost', array('name' => 'boost', 'null_value' => $this->boost));
if ($this->sourceExcludes) {
$ESmapping->setSource(array('excludes' => $this->sourceExcludes));
}
// Set mapping
$ESmapping->setProperties($this->mapping);
// Send mapping to type
$ESmapping->send();
}
}
