function createFileMapping()
{
// create a type
$index = $this->index;
$type = $index->getType('file');
// Define mapping
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($type);
$mapping->setParam('index_analyzer', 'indexAnalyzer');
$mapping->setParam('search_analyzer', 'searchAnalyzer');
// Define boost field
//$mapping->setParam('_boost', array('name' => 'chinese_name', 'null_value' => ''));
// Set mapping
$mapping->setProperties(array('id' => array('type' => 'integer', 'include_in_all' => TRUE), 'dataset_id' => array('type' => 'integer'), 'name' => array('type' => 'string'), 'size' => array('type' => 'long'), 'description' => array('type' => 'string'), 'filetype' => array('type' => 'string'), 'format' => array('type' => 'string')));
$mapping->send();
}
protected function execute(Console\Input\InputInterface $input, Console\Output\OutputInterface $output)
{
$this->output = $output;
$selectedIndexes = [$input->getArgument('index')];
if (empty($selectedIndexes)) {
$selectedIndexes = $this->askIndexes();
}
foreach ($selectedIndexes as $indexName) {
$index = $this->elastica->getIndex($indexName);
$indexAnalyzers = isset($this->indices[$indexName]['analyzers']) ? $this->indices[$indexName]['analyzers'] : [];
$indexFilters = isset($this->indices[$indexName]['filters']) ? $this->indices[$indexName]['filters'] : [];
$analyzers = [];
foreach ($indexAnalyzers as $analyzerName) {
$analyzers[$analyzerName] = $this->analyzers[$analyzerName];
}
$filters = [];
foreach ($indexFilters as $filterName) {
$filters[$filterName] = $this->filters[$filterName];
}
try {
$index->create(['analysis' => ['analyzer' => $analyzers, 'filter' => $filters]]);
$output->writeln(sprintf('Index <info>%s</info> successfully created.', $indexName));
} catch (\Elastica\Exception\ResponseException $e) {
if (strpos($e->getMessage(), 'IndexAlreadyExistsException') !== FALSE) {
$output->writeln(sprintf('<error>Index %s already exists. Please drop it first.</error>', $indexName));
}
}
$types = isset($this->indices[$indexName]['types']) ? $this->indices[$indexName]['types'] : [];
foreach ($types as $typeName) {
$properties = $this->types[$typeName]['properties'];
$params = $this->types[$typeName]['params'];
$elasticaType = $index->getType($typeName);
$elasticaMapping = new \Elastica\Type\Mapping($elasticaType, $properties);
if (is_array($params)) {
foreach ($params as $param) {
$elasticaMapping->setParam($param);
}
}
// Send mapping to type
try {
$res = $elasticaMapping->send();
$output->writeln(sprintf('Type <info>%s</info> successfully created in index <info>%s</info>', $typeName, $indexName));
} catch (\Elastica\Exception\ResponseException $e) {
if (strpos($e->getMessage(), 'nested: NullPointerException;') !== FALSE) {
$output->writeln(sprintf('<error>%s. Probably bad mapping.</error>', $e->getMessage()));
}
}
}
}
}
public function __construct()
{
//Parsing the yml config file
$yml = Yaml::parse(file_get_contents("config.yml"), false, false, true);
$this->_config = json_decode(json_encode($yml));
//Setting up PDO
try {
$this->_bdd = new \PDO(sprintf('mysql:dbname=%s;host=%s', $this->_config->mysql->bdd, $this->_config->mysql->host), $this->_config->mysql->user, $this->_config->mysql->pass);
$this->_bdd->setAttribute(\PDO::ATTR_DEFAULT_FETCH_MODE, \PDO::FETCH_OBJ);
} catch (PDOException $exception) {
echo "Unable to start database, have you imported the sample database?\n";
exit;
}
//Create the elastica client
$this->_client = new \Elastica\Client();
//Load the existing index or overwriting it if needed
$elasticaIndex = $this->_client->getIndex($this->_config->elastic->index, true);
//Setting up elasticSearch indexes
$elasticaIndex->create(array('number_of_shards' => 4, 'number_of_replicas' => 1, 'analysis' => array('analyzer' => array('indexAnalyzer' => array('type' => 'custom', 'tokenizer' => 'standard', 'filter' => array('standard', 'lowercase', 'asciifolding')), 'searchAnalyzer' => array('type' => 'custom', 'tokenizer' => 'standard', 'filter' => array('standard', 'lowercase', 'asciifolding', 'mySnowball')), 'keylower' => array('tokenizer' => 'keyword', 'filter' => 'lowercase')), 'filter' => array('mySnowball' => array('type' => 'snowball', 'language' => 'French')))), true);
//Initialyse the dataType
$elasticaType = $elasticaIndex->getType('persons');
//Mapping some properties to the type
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($elasticaType);
$mapping->setParam('index_analyzer', 'indexAnalyzer');
$mapping->setParam('search_analyzer', 'searchAnalyzer');
//Setting up the fields of the user
//Here a user have a last/first name, a phone number and multiples adresses
$mapping->setProperties(array('id' => array('type' => 'integer', 'include_in_all' => false), 'persons' => array('type' => 'object', 'properties' => array('id' => array('type' => 'integer', 'include_in_all' => false), 'lastname' => array('type' => 'string', 'include_in_all' => true), 'firstname' => array('type' => 'string', 'include_in_all' => true), 'email' => array('type' => 'string', 'include_in_all' => true), 'address' => array('type' => 'string', 'include_in_all' => true), 'city' => array('type' => 'string', 'include_in_all' => true), 'country' => array('type' => 'string', 'include_in_all' => true), 'company' => array('type' => 'string', 'include_in_all' => true))), 'tstamp' => array('type' => 'date', 'include_in_all' => true), 'location' => array('type' => 'geo_point', 'include_in_all' => true), '_boost' => array('type' => 'float', 'include_in_all' => true)));
//Save the data mapping
$mapping->send();
//Starting RabbitMq
$this->connection = new AMQPConnection($this->_config->rabbitmq->host, $this->_config->rabbitmq->port, $this->_config->rabbitmq->user, $this->_config->rabbitmq->pass);
$this->channel = $this->connection->channel();
$this->channel->queue_declare('task_queue', false, true, false, false);
}
protected function execute(Console\Input\InputInterface $input, Console\Output\OutputInterface $output)
{
$this->output = $output;
$selectedTypes = $this->askTypes();
$indexesForTypes = $this->getIndexesForTypes($selectedTypes);
$selectedIndexes = $this->askIndexes($indexesForTypes);
foreach ($selectedTypes as $typeName) {
$typeIndexes = $indexesForTypes[$typeName];
foreach ($selectedIndexes as $indexName) {
if (in_array($indexName, $typeIndexes)) {
$output->writeln(sprintf('Creating type <info>%s</info> in index <info>%s</info>', $typeName, $indexName));
$properties = $this->types[$typeName]['properties'];
$params = $this->types[$typeName]['params'];
$index = $this->elastica->getIndex($indexName);
$elasticaType = $index->getType($typeName);
$elasticaMapping = new \Elastica\Type\Mapping($elasticaType, $properties);
if (is_array($params)) {
foreach ($params as $param) {
$elasticaMapping->setParam($param);
}
}
// Send mapping to type
try {
$res = $elasticaMapping->send();
$output->writeln(sprintf('Type <info>%s</info> successfully created in index <info>%s</info>', $typeName, $indexName));
} catch (\Elastica\Exception\ResponseException $e) {
if (strpos($e->getMessage(), 'IndexMissingException') !== FALSE) {
$output->writeln(sprintf('<error>Index %1$s is missing. Please create index %1$s first.</error>', $indexName));
} elseif (strpos($e->getMessage(), 'nested: NullPointerException;') !== FALSE) {
$output->writeln(sprintf('<error>%s. Probably bad mapping.</error>', $e->getMessage()));
}
}
}
}
}
}
/**
* Creates the "mapping" for this searchtype, on the elasticsearch server
*
* TODO: it would be good to be able to make elasticsearch just delete the contents of one
* searchtype, perhaps identified by its mapping? But I haven't been able to figure out how
* to do that, so instead you can only delete the whole index at once.
* @param string $type
*/
private static function set_mapping($type)
{
// usr,interaction_instance,interaction_forum_post,view,group,artefact
$ES_class = 'ElasticsearchType_' . $type;
if ($ES_class::$mappingconf === false) {
return false;
}
$elasticaClient = self::make_client();
$elasticaIndex = $elasticaClient->getIndex(self::get_write_indexname());
$elasticaAnalyzer = get_config_plugin('search', 'elasticsearch', 'analyzer');
if (!isset($elasticaAnalyzer)) {
$elasticaAnalyzer = 'mahara_analyzer';
}
// Load type
$elasticaType = $elasticaIndex->getType($type);
// Define mapping
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($elasticaType);
// we use mahara_analyzer created through elastica
$mapping->setParam('index_analyzer', $elasticaAnalyzer);
$mapping->setParam('search_analyzer', $elasticaAnalyzer);
// Define boost field
//$mapping->setParam('_boost', array('name' => '_boost', 'null_value' => 1.0));
// Set mapping
$mapping->setProperties($ES_class::$mappingconf);
// Send mapping to type
$mapping->send();
}
/**
* @dataProvider providerTransport
* @group benchmark
*/
public function testGetMapping(array $config, $transport)
{
$this->_checkTransport($config, $transport);
$client = $this->_getClient($config);
$index = $client->getIndex('benchmark');
$index->create(array(), true);
$type = $index->getType('mappingTest');
// Define mapping
$mapping = new \Elastica\Type\Mapping();
$mapping->setParam('_boost', array('name' => '_boost', 'null_value' => 1.0));
$mapping->setProperties(array('id' => array('type' => 'integer', 'include_in_all' => false), 'user' => array('type' => 'object', 'properties' => array('name' => array('type' => 'string', 'include_in_all' => true), 'fullName' => array('type' => 'string', 'include_in_all' => true))), 'msg' => array('type' => 'string', 'include_in_all' => true), 'tstamp' => array('type' => 'date', 'include_in_all' => false), 'location' => array('type' => 'geo_point', 'include_in_all' => false), '_boost' => array('type' => 'float', 'include_in_all' => false)));
$type->setMapping($mapping);
$index->refresh();
$times = array();
for ($i = 0; $i < $this->_max; ++$i) {
$response = $type->request('_mapping', \Elastica\Request::GET);
$times[] = $response->getQueryTime();
}
self::logResults('get mapping', $transport, $times);
}
protected function updateMapping($indexName, $typeName, $mapping = [])
{
$elasticaType = $this->getClient()->getIndex($indexName)->getType($typeName);
// Define mapping
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($elasticaType);
$mapping->setParam('index_analyzer', 'indexAnalyzer');
$mapping->setParam('search_analyzer', 'searchAnalyzer');
// Define boost field
$mapping->setParam('_boost', array('name' => '_boost', 'null_value' => 1.0));
// Set mapping
$mapping->setProperties(array('id' => array('type' => 'integer', 'include_in_all' => false), 'user' => array('type' => 'object', 'properties' => array('name' => array('type' => 'string', 'include_in_all' => true), 'fullName' => array('type' => 'string', 'include_in_all' => true))), 'msg' => array('type' => 'string', 'include_in_all' => true), 'tstamp' => array('type' => 'date', 'include_in_all' => false), 'location' => array('type' => 'geo_point', 'include_in_all' => false), '_boost' => array('type' => 'float', 'include_in_all' => false)));
// Send mapping to type
$mapping->send();
}
protected function mapping()
{
// Set mapping only if mapping is set and if type does not exist
if ($this->mapping && !$this->typeExist($this->current_type)) {
$ESmapping = new \Elastica\Type\Mapping();
$ESmapping->setType($this->elasticaType);
//If Language supported by ES
if (array_key_exists($this->lang, $this->langAnalyzer)) {
$ESmapping->setParam("index_analyzer", $this->lang . "_Analyzer");
$ESmapping->setParam("search_analyzer", $this->lang . "_Analyzer");
} else {
$ESmapping->setParam("index_analyzer", "default");
$ESmapping->setParam("search_analyzer", "default");
}
if ($this->boost == "") {
$this->boost = 1.0;
}
$ESmapping->setParam('_boost', array('name' => 'boost', 'null_value' => $this->boost));
if ($this->sourceExcludes) {
$ESmapping->setSource(array('excludes' => $this->sourceExcludes));
}
// Set mapping
$ESmapping->setProperties($this->mapping);
// Send mapping to type
$ESmapping->send();
}
}
/**
* Return default search fields mapping for node translations
*
* @param \Elastica\Index $index
* @param string $lang
*
* @return \Elastica\Type\Mapping
*/
protected function getMapping(\Elastica\Index $index, $lang = 'en')
{
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($index->getType($this->indexType . '_' . $lang));
$mapping->setParam('analyzer', 'index_analyzer_' . $lang);
$mapping->setParam('_boost', array('name' => '_boost', 'null_value' => 1.0));
$mapping->setProperties(array('node_id' => array('type' => 'integer', 'include_in_all' => false, 'index' => 'not_analyzed'), 'nodetranslation_id' => array('type' => 'integer', 'include_in_all' => false, 'index' => 'not_analyzed'), 'nodeversion_id' => array('type' => 'integer', 'include_in_all' => false, 'index' => 'not_analyzed'), 'title' => array('type' => 'string', 'include_in_all' => true), 'lang' => array('type' => 'string', 'include_in_all' => true, 'index' => 'not_analyzed'), 'slug' => array('type' => 'string', 'include_in_all' => false, 'index' => 'not_analyzed'), 'type' => array('type' => 'string', 'include_in_all' => false, 'index' => 'not_analyzed'), 'page_class' => array('type' => 'string', 'include_in_all' => false, 'index' => 'not_analyzed'), 'contentanalyzer' => array('type' => 'string', 'include_in_all' => true, 'index' => 'not_analyzed'), 'content' => array('type' => 'string', 'include_in_all' => true), 'created' => array('type' => 'date', 'include_in_all' => false, 'index' => 'not_analyzed'), 'updated' => array('type' => 'date', 'include_in_all' => false, 'index' => 'not_analyzed'), 'view_roles' => array('type' => 'string', 'include_in_all' => true, 'index' => 'not_analyzed', 'index_name' => 'view_role'), '_boost' => array('type' => 'float', 'include_in_all' => false)));
return $mapping;
}
/**
* prepare elasticsearch index for store
*
* @param integer $storeId
* @return Hackathon_ElasticgentoCore_Model_Resource_Catalog_Product_Indexer_Elasticgento
* @todo implement alias handling for non blocking reindex
*/
protected function _prepareIndex($storeId)
{
if (true === isset($this->_preparedIndexes[$storeId])) {
return $this;
}
//handle index creation / deletition
$idx = $this->_getClient()->getIndex($storeId);
$settings = Mage::getModel('elasticgento/catalog_product_elasticgento_settings')->setStoreId($storeId)->getIndexSettings();
if (false === $idx->exists()) {
$idx->create($settings);
} else {
$idx->delete();
$idx->create($settings);
}
//handle type
//load settings
$typeMappings = $this->getMappings($storeId);
$dynamicTemplates = $this->_getMappingModel($storeId)->getDynamicTemplates();
$type = $this->_getClient()->getIndex($storeId)->getType($this->getEntityType());
$elasticaMapping = new \Elastica\Type\Mapping($type);
$elasticaMapping->setParam('_all', array('enabled' => false));
$elasticaMapping->setParam('dynamic_templates', $dynamicTemplates);
$elasticaMapping->setProperties($typeMappings);
$elasticaMapping->send();
//set index to be prepared
$this->_preparedIndexes[$storeId] = true;
return $this;
}
public function createIndex($index, $config)
{
$response = $index->create($config['config'], $config['options']);
if ($response->isOk()) {
// lets register the mappings
foreach ($config['mappings'] as $type => $type_config) {
$type = $index->getType($type);
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($type);
$mapping->setParam('index_analyzer', 'indexAnalyzer');
$mapping->setParam('search_analyzer', 'searchAnalyzer');
$mapping->setParam('_boost', array('name' => '_boost', 'null_value' => 1.0));
$mapping->setProperties($type_config['properties']);
$mapping->send();
}
}
return $response;
}
