protected function execute(InputInterface $input, OutputInterface $output)
{
$output->writeln('--- veterinar:autocomplete started');
$em = $this->getContainer()->get('doctrine')->getManager('veterinar');
$productNames = $em->getRepository('VidalVeterinarBundle:Product')->findProductNames();
$names = array_unique($productNames);
sort($names);
$elasticaClient = new \Elastica\Client();
$elasticaIndex = $elasticaClient->getIndex('website');
$elasticaType = $elasticaIndex->getType('veterinar_autocomplete');
if ($elasticaType->exists()) {
$elasticaType->delete();
}
// Define mapping
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($elasticaType);
// Set mapping
$mapping->setProperties(array('id' => array('type' => 'integer', 'include_in_all' => FALSE), 'name' => array('type' => 'string', 'include_in_all' => TRUE)));
// Send mapping to type
$mapping->send();
# записываем на сервер документы автодополнения
$documents = array();
for ($i = 0; $i < count($names); $i++) {
$documents[] = new \Elastica\Document($i + 1, array('name' => $names[$i]));
if ($i && $i % 500 == 0) {
$elasticaType->addDocuments($documents);
$elasticaType->getIndex()->refresh();
}
}
$output->writeln("+++ veterinar:autocomplete loaded {$i} documents!");
}
protected function execute(InputInterface $input, OutputInterface $output)
{
ini_set('memory_limit', -1);
$output->writeln('--- vidal:autocomplete_clinic started');
$em = $this->getContainer()->get('doctrine')->getManager('drug');
$names = $em->getRepository('VidalDrugBundle:ClPhGroups')->getNames();
$elasticaClient = new \Elastica\Client();
$elasticaIndex = $elasticaClient->getIndex('website');
$elasticaType = $elasticaIndex->getType('autocomplete_clinic');
# delete if exists
if ($elasticaType->exists()) {
$elasticaType->delete();
}
# Define mapping
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($elasticaType);
# Set mapping
$mapping->setProperties(array('id' => array('type' => 'integer', 'include_in_all' => FALSE), 'name' => array('type' => 'string', 'include_in_all' => TRUE)));
# Send mapping to type
$mapping->send();
# записываем на сервер документы автодополнения
$documents = array();
for ($i = 0, $c = count($names); $i < $c; $i++) {
$documents[] = new \Elastica\Document($i + 1, array('name' => $names[$i]));
if ($i && $i % 500 == 0) {
$elasticaType->addDocuments($documents);
$elasticaType->getIndex()->refresh();
$documents = array();
$output->writeln("... + {$i}");
}
}
$elasticaType->addDocuments($documents);
$elasticaType->getIndex()->refresh();
$output->writeln("+++ vidal:autocomplete_clinic loaded {$i} documents!");
}
protected function execute(InputInterface $input, OutputInterface $output)
{
ini_set('memory_limit', -1);
$output->writeln('--- vidal:autocomplete_document started');
$em = $this->getContainer()->get('doctrine')->getManager('drug');
$names = array();
$documents = $em->createQuery("\n\t\t\tSELECT d.DocumentID, d.RusName\n\t\t\tFROM VidalDrugBundle:Document d\n\t\t\tWHERE d.CountryEditionCode = 'RUS'\n\t\t\tORDER BY d.RusName ASC\n\t\t")->getResult();
foreach ($documents as $document) {
$names[] = $document['DocumentID'] . ' ' . $this->strip($document['RusName']);
}
$elasticaClient = new \Elastica\Client();
$elasticaIndex = $elasticaClient->getIndex('website');
$elasticaType = $elasticaIndex->getType('autocomplete_document');
# delete if exists
if ($elasticaType->exists()) {
$elasticaType->delete();
}
# Define mapping
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($elasticaType);
# Set mapping
$mapping->setProperties(array('name' => array('type' => 'string', 'include_in_all' => TRUE)));
# Send mapping to type
$mapping->send();
for ($i = 0; $i < count($names); $i++) {
$document = new \Elastica\Document(null, array('name' => $names[$i]));
$elasticaType->addDocument($document);
$elasticaType->getIndex()->refresh();
if ($i && $i % 500 == 0) {
$output->writeln("... + {$i}");
}
}
$output->writeln("+++ vidal:autocomplete_document loaded {$i} documents!");
}
protected function execute(InputInterface $input, OutputInterface $output)
{
ini_set('memory_limit', -1);
$output->writeln('--- vidal:autocomplete_article started');
$em = $this->getContainer()->get('doctrine')->getManager('drug');
$articles = $em->getRepository('VidalDrugBundle:Article')->findActive();
$elasticaClient = new \Elastica\Client();
$elasticaIndex = $elasticaClient->getIndex('website');
$elasticaType = $elasticaIndex->getType('autocomplete_article');
# delete if exists
if ($elasticaType->exists()) {
$elasticaType->delete();
}
# Define mapping
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($elasticaType);
# Set mapping
$mapping->setProperties(array('title' => array('type' => 'string', 'include_in_all' => TRUE)));
# Send mapping to type
$mapping->send();
# записываем на сервер документы автодополнения
$documents = array();
for ($i = 0; $i < count($articles); $i++) {
$documents[] = new \Elastica\Document($i + 1, array('title' => mb_strtolower($articles[$i]->getTitle(), 'utf-8')));
if ($i && $i % 500 == 0) {
$elasticaType->addDocuments($documents);
$elasticaType->getIndex()->refresh();
$documents = array();
}
}
$elasticaType->addDocuments($documents);
$elasticaType->getIndex()->refresh();
$output->writeln("+++ vidal:autocomplete_article completed!");
}
protected function execute(InputInterface $input, OutputInterface $output)
{
ini_set('memory_limit', -1);
$output->writeln('--- vidal:autocomplete_product started');
$em = $this->getContainer()->get('doctrine')->getManager('drug');
$names = array();
$products = $em->createQuery("\n\t\t\tSELECT p.ProductID, p.RusName, p.ZipInfo\n\t\t\tFROM VidalDrugBundle:Product p\n\t\t\tWHERE p.MarketStatusID IN (1,2,7)\n\t\t\t\tAND p.ProductTypeCode IN ('DRUG','GOME')\n\t\t\t\tAND p.inactive = FALSE\n\t\t\tORDER BY p.RusName ASC\n\t\t")->getResult();
foreach ($products as $p) {
$names[] = $p['ProductID'] . ' ' . $this->strip($p['RusName']);
}
$elasticaClient = new \Elastica\Client();
$elasticaIndex = $elasticaClient->getIndex('website');
$elasticaType = $elasticaIndex->getType('autocomplete_product');
# delete if exists
if ($elasticaType->exists()) {
$elasticaType->delete();
}
# Define mapping
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($elasticaType);
# Set mapping
$mapping->setProperties(array('name' => array('type' => 'string', 'include_in_all' => TRUE)));
# Send mapping to type
$mapping->send();
$total = count($products);
for ($i = 0; $i < $total; $i++) {
$id = $products[$i]['ProductID'];
$name = $this->strip($products[$i]['RusName']) . ' ' . $id;
$document = new \Elastica\Document($id, array('name' => $name));
$elasticaType->addDocument($document);
$elasticaType->getIndex()->refresh();
}
$output->writeln("+++ vidal:autocomplete_product loaded {$i} documents!");
}
public function init()
{
// Check for authentication
if (!empty($this->loader['username']) && !empty($this->loader['password'])) {
$auth = $this->loader['username'] . ':' . $this->loader['password'] . '@';
$parts = parse_url($this->loader['host']);
if (empty($parts['scheme'])) {
$this->loader['host'] = 'http://' . $auth . $this->loader['host'];
} elseif ($parts['scheme'] == 'https') {
$schemeless_url = str_replace('https://', '', $this->loader['host']);
$this->loader['host'] = 'https://' . $auth . $schemeless_url;
} else {
$schemeless_url = str_replace('http://', '', $this->loader['host']);
$this->loader['host'] = 'http://' . $auth . $schemeless_url;
}
}
$this->type = $this->loader['es_type'];
$this->client = new Client(['host' => $this->loader['host'], 'port' => $this->loader['port']]);
$this->index = new Index($this->client, $this->loader['es_index']);
if (!$this->index->exists()) {
$this->index->create();
}
$this->log("The ElasticSearch client is configured to write to the index " . $this->loader['es_index'] . " with the " . $this->loader['es_type'] . " type.", 'info');
$this->type = $this->index->getType($this->loader['es_type']);
// Define mapping
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($this->type);
// Set mapping
$mapping->setProperties(array(self::$ETL_TIMESTAMP => array('type' => 'date')));
// Send mapping to type
$mapping->send();
}
protected function execute(InputInterface $input, OutputInterface $output)
{
ini_set('memory_limit', -1);
$output->writeln('--- vidal:autocomplete started');
$em = $this->getContainer()->get('doctrine')->getManager('drug');
$productNames = $em->getRepository('VidalDrugBundle:Product')->findAutocomplete();
$moleculeNames = $em->getRepository('VidalDrugBundle:Molecule')->findAutocomplete();
$companyNames = $em->getRepository('VidalDrugBundle:Company')->findAutocomplete();
$atcNames = $em->getRepository('VidalDrugBundle:ATC')->findAutocomplete();
$elasticaClient = new \Elastica\Client();
$elasticaIndex = $elasticaClient->getIndex('website');
$elasticaType = $elasticaIndex->getType('autocomplete');
if ($elasticaType->exists()) {
$elasticaType->delete();
}
// Define mapping
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($elasticaType);
// Set mapping
$mapping->setProperties(array('name' => array('type' => 'string', 'include_in_all' => TRUE), 'type' => array('type' => 'string', 'include_in_all' => FALSE)));
// Send mapping to type
$mapping->send();
# записываем в ElasticSearch документы автодополнения
$this->save($productNames, 'product', $output, $elasticaType);
$this->save($moleculeNames, 'molecule', $output, $elasticaType);
$this->save($companyNames, 'company', $output, $elasticaType);
$this->save($atcNames, 'atc', $output, $elasticaType);
$output->writeln("+++ vidal:autocomplete completed!");
}
/**
* @param array $properties
* @return \Elastica\Response
*/
public function createMapping(array $properties)
{
$elasticaType = $this->index->getType($this->typeName);
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($elasticaType);
$mapping->setProperties($properties);
return $mapping->send()->isOk();
}
public function createMapping()
{
// Define mapping
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($this->trackIndex->getType('track'));
// Set mapping
$mapping->setProperties(['id' => ['type' => 'integer', 'include_in_all' => false], 'album' => ['type' => 'object', 'properties' => ['id' => ['type' => 'integer', 'include_in_all' => true], 'title' => ['type' => 'string', 'include_in_all' => true]]], 'name' => ['type' => 'string', 'include_in_all' => true], 'name_not_analyzed' => ['type' => 'string', "index" => "not_analyzed"], 'playList' => ['type' => 'nested', 'properties' => ['id' => ['type' => 'integer', 'include_in_all' => true], 'name' => ['type' => 'string', 'include_in_all' => true]]], 'genre' => ['type' => 'object', 'properties' => ['id' => ['type' => 'integer', 'include_in_all' => true], 'name' => ['type' => 'string', 'include_in_all' => true]]], 'mediaType' => ['type' => 'object', 'properties' => ['id' => ['type' => 'integer', 'include_in_all' => true], 'name' => ['type' => 'string', 'include_in_all' => true]]], 'composer' => ['type' => 'string', 'include_in_all' => true]]);
// Send mapping to type
$mapping->send();
}
/**
* @return \Elastica\Type\Mapping
*/
public function send_search_mapping()
{
$type = ElasticSearch::instance()->get_type($this->_search_type());
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($type);
// Set mapping
$mapping->setProperties(array('id' => array('type' => 'integer', 'include_in_all' => FALSE), 'content' => array('type' => 'string', 'include_in_all' => TRUE)));
// Send mapping to type
$mapping->send();
}
function createFileMapping()
{
// create a type
$index = $this->index;
$type = $index->getType('file');
// Define mapping
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($type);
$mapping->setParam('index_analyzer', 'indexAnalyzer');
$mapping->setParam('search_analyzer', 'searchAnalyzer');
// Define boost field
//$mapping->setParam('_boost', array('name' => 'chinese_name', 'null_value' => ''));
// Set mapping
$mapping->setProperties(array('id' => array('type' => 'integer', 'include_in_all' => TRUE), 'dataset_id' => array('type' => 'integer'), 'name' => array('type' => 'string'), 'size' => array('type' => 'long'), 'description' => array('type' => 'string'), 'filetype' => array('type' => 'string'), 'format' => array('type' => 'string')));
$mapping->send();
}
protected function execute(InputInterface $input, OutputInterface $output)
{
ini_set('memory_limit', -1);
$output->writeln('--- evrika:elastic_autocomlete started');
$em = $this->getContainer()->get('doctrine')->getManager();
$publications = $em->getRepository('EvrikaMainBundle:Publication')->findAutocomplete();
$events = $em->getRepository('EvrikaMainBundle:Event')->findAutocomplete();
for ($i = 0; $i < count($publications); $i++) {
$publications[$i]['title'] = trim($publications[$i]['title']);
$publications[$i]['type'] = 'publication';
}
for ($i = 0; $i < count($events); $i++) {
$events[$i]['title'] = trim($events[$i]['title']);
$events[$i]['type'] = 'event';
}
$articles = array_merge($publications, $events);
usort($articles, function ($a, $b) {
return strcmp(trim($a['title'], '"'), trim($b['title'], '"'));
});
$elasticaClient = new \Elastica\Client();
$elasticaIndex = $elasticaClient->getIndex('evrika');
$elasticaType = $elasticaIndex->getType('autocomplete');
if ($elasticaType->exists()) {
$elasticaType->delete();
}
// Define mapping
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($elasticaType);
// Set mapping
$mapping->setProperties(array('id' => array('type' => 'integer', 'include_in_all' => false), 'type' => array('type' => 'string', 'include_in_all' => false), 'title' => array('type' => 'string', 'include_in_all' => true, 'index_analyzer' => 'indexAnalyzer', 'search_analyzer' => 'searchAnalyzer')));
// Send mapping to type
$mapping->send();
# записываем на сервер документы автодополнения
$documents = array();
$count = count($articles);
for ($i = 0; $i < $count; $i++) {
$index = $articles[$i]['type'] == 'event' ? $articles[$i]['id'] + 999999 : $articles[$i]['id'];
$documents[] = new \Elastica\Document($index, array('id' => $articles[$i]['id'], 'type' => $articles[$i]['type'], 'title' => $articles[$i]['title']));
if ($i && $i % 100 == 0) {
$elasticaType->addDocuments($documents);
$elasticaType->getIndex()->refresh();
$documents = array();
}
}
$output->writeln("+++ evrika:elastic_autocomlete completed!");
}
public function __construct()
{
//Parsing the yml config file
$yml = Yaml::parse(file_get_contents("config.yml"), false, false, true);
$this->_config = json_decode(json_encode($yml));
//Setting up PDO
try {
$this->_bdd = new \PDO(sprintf('mysql:dbname=%s;host=%s', $this->_config->mysql->bdd, $this->_config->mysql->host), $this->_config->mysql->user, $this->_config->mysql->pass);
$this->_bdd->setAttribute(\PDO::ATTR_DEFAULT_FETCH_MODE, \PDO::FETCH_OBJ);
} catch (PDOException $exception) {
echo "Unable to start database, have you imported the sample database?\n";
exit;
}
//Create the elastica client
$this->_client = new \Elastica\Client();
//Load the existing index or overwriting it if needed
$elasticaIndex = $this->_client->getIndex($this->_config->elastic->index, true);
//Setting up elasticSearch indexes
$elasticaIndex->create(array('number_of_shards' => 4, 'number_of_replicas' => 1, 'analysis' => array('analyzer' => array('indexAnalyzer' => array('type' => 'custom', 'tokenizer' => 'standard', 'filter' => array('standard', 'lowercase', 'asciifolding')), 'searchAnalyzer' => array('type' => 'custom', 'tokenizer' => 'standard', 'filter' => array('standard', 'lowercase', 'asciifolding', 'mySnowball')), 'keylower' => array('tokenizer' => 'keyword', 'filter' => 'lowercase')), 'filter' => array('mySnowball' => array('type' => 'snowball', 'language' => 'French')))), true);
//Initialyse the dataType
$elasticaType = $elasticaIndex->getType('persons');
//Mapping some properties to the type
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($elasticaType);
$mapping->setParam('index_analyzer', 'indexAnalyzer');
$mapping->setParam('search_analyzer', 'searchAnalyzer');
//Setting up the fields of the user
//Here a user have a last/first name, a phone number and multiples adresses
$mapping->setProperties(array('id' => array('type' => 'integer', 'include_in_all' => false), 'persons' => array('type' => 'object', 'properties' => array('id' => array('type' => 'integer', 'include_in_all' => false), 'lastname' => array('type' => 'string', 'include_in_all' => true), 'firstname' => array('type' => 'string', 'include_in_all' => true), 'email' => array('type' => 'string', 'include_in_all' => true), 'address' => array('type' => 'string', 'include_in_all' => true), 'city' => array('type' => 'string', 'include_in_all' => true), 'country' => array('type' => 'string', 'include_in_all' => true), 'company' => array('type' => 'string', 'include_in_all' => true))), 'tstamp' => array('type' => 'date', 'include_in_all' => true), 'location' => array('type' => 'geo_point', 'include_in_all' => true), '_boost' => array('type' => 'float', 'include_in_all' => true)));
//Save the data mapping
$mapping->send();
//Starting RabbitMq
$this->connection = new AMQPConnection($this->_config->rabbitmq->host, $this->_config->rabbitmq->port, $this->_config->rabbitmq->user, $this->_config->rabbitmq->pass);
$this->channel = $this->connection->channel();
$this->channel->queue_declare('task_queue', false, true, false, false);
}
protected function updateMapping($indexName, $typeName, $mapping = [])
{
$elasticaType = $this->getClient()->getIndex($indexName)->getType($typeName);
// Define mapping
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($elasticaType);
$mapping->setParam('index_analyzer', 'indexAnalyzer');
$mapping->setParam('search_analyzer', 'searchAnalyzer');
// Define boost field
$mapping->setParam('_boost', array('name' => '_boost', 'null_value' => 1.0));
// Set mapping
$mapping->setProperties(array('id' => array('type' => 'integer', 'include_in_all' => false), 'user' => array('type' => 'object', 'properties' => array('name' => array('type' => 'string', 'include_in_all' => true), 'fullName' => array('type' => 'string', 'include_in_all' => true))), 'msg' => array('type' => 'string', 'include_in_all' => true), 'tstamp' => array('type' => 'date', 'include_in_all' => false), 'location' => array('type' => 'geo_point', 'include_in_all' => false), '_boost' => array('type' => 'float', 'include_in_all' => false)));
// Send mapping to type
$mapping->send();
}
/**
* Return default search fields mapping for node translations
*
* @param \Elastica\Index $index
* @param string $lang
*
* @return \Elastica\Type\Mapping
*/
protected function getMapping(\Elastica\Index $index, $lang = 'en')
{
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($index->getType($this->indexType . '_' . $lang));
$mapping->setParam('analyzer', 'index_analyzer_' . $lang);
$mapping->setParam('_boost', array('name' => '_boost', 'null_value' => 1.0));
$mapping->setProperties(array('node_id' => array('type' => 'integer', 'include_in_all' => false, 'index' => 'not_analyzed'), 'nodetranslation_id' => array('type' => 'integer', 'include_in_all' => false, 'index' => 'not_analyzed'), 'nodeversion_id' => array('type' => 'integer', 'include_in_all' => false, 'index' => 'not_analyzed'), 'title' => array('type' => 'string', 'include_in_all' => true), 'lang' => array('type' => 'string', 'include_in_all' => true, 'index' => 'not_analyzed'), 'slug' => array('type' => 'string', 'include_in_all' => false, 'index' => 'not_analyzed'), 'type' => array('type' => 'string', 'include_in_all' => false, 'index' => 'not_analyzed'), 'page_class' => array('type' => 'string', 'include_in_all' => false, 'index' => 'not_analyzed'), 'contentanalyzer' => array('type' => 'string', 'include_in_all' => true, 'index' => 'not_analyzed'), 'content' => array('type' => 'string', 'include_in_all' => true), 'created' => array('type' => 'date', 'include_in_all' => false, 'index' => 'not_analyzed'), 'updated' => array('type' => 'date', 'include_in_all' => false, 'index' => 'not_analyzed'), 'view_roles' => array('type' => 'string', 'include_in_all' => true, 'index' => 'not_analyzed', 'index_name' => 'view_role'), '_boost' => array('type' => 'float', 'include_in_all' => false)));
return $mapping;
}
/**
* @param AJXP_Node $ajxpNode
* @throws Exception
*/
public function createIndexedDocument($ajxpNode)
{
$ajxpNode->loadNodeInfo();
/*
$ext = strtolower(pathinfo($ajxpNode->getLabel(), PATHINFO_EXTENSION));
$parseContent = $this->indexContent;
if ($parseContent && $ajxpNode->bytesize > $this->getFilteredOption("PARSE_CONTENT_MAX_SIZE")) {
$parseContent = false;
}
if ($parseContent && in_array($ext, explode(",",$this->getFilteredOption("PARSE_CONTENT_HTML")))) {
$doc = @Zend_Search_Lucene_Document_Html::loadHTMLFile($ajxpNode->getUrl());
} elseif ($parseContent && $ext == "docx" && class_exists("Zend_Search_Lucene_Document_Docx")) {
$realFile = call_user_func(array($ajxpNode->wrapperClassName, "getRealFSReference"), $ajxpNode->getUrl());
$doc = @Zend_Search_Lucene_Document_Docx::loadDocxFile($realFile);
} elseif ($parseContent && $ext == "docx" && class_exists("Zend_Search_Lucene_Document_Pptx")) {
$realFile = call_user_func(array($ajxpNode->wrapperClassName, "getRealFSReference"), $ajxpNode->getUrl());
$doc = @Zend_Search_Lucene_Document_Pptx::loadPptxFile($realFile);
} elseif ($parseContent && $ext == "xlsx" && class_exists("Zend_Search_Lucene_Document_Xlsx")) {
$realFile = call_user_func(array($ajxpNode->wrapperClassName, "getRealFSReference"), $ajxpNode->getUrl());
$doc = @Zend_Search_Lucene_Document_Xlsx::loadXlsxFile($realFile);
} else {
$doc = new Zend_Search_Lucene_Document();
}
if($doc == null) throw new Exception("Could not load document");
*/
$mapping_properties = array();
/* we construct the array that will contain all the data for the document we create */
$data = array();
$data["node_url"] = $ajxpNode->getUrl();
$data["node_path"] = str_replace("/", "AJXPFAKESEP", $ajxpNode->getPath());
$data["basename"] = basename($ajxpNode->getPath());
$data["ajxp_node"] = "yes";
$data["ajxp_scope"] = "shared";
$data["serialized_metadata"] = base64_encode(serialize($ajxpNode->metadata));
if (isset($ajxpNode->indexableMetaKeys["shared"])) {
foreach ($ajxpNode->indexableMetaKeys["shared"] as $sharedField) {
if ($ajxpNode->{$sharedField}) {
$data[$sharedField] = $ajxpNode->{$sharedField};
}
}
}
foreach ($this->metaFields as $field) {
if ($ajxpNode->{$field} != null) {
$data["ajxp_meta_{$field}"] = $ajxpNode->{$field};
}
}
/*
We want some fields not to be analyzed when they are indexed.
To achieve this we have to dynamically define a mapping.
When you define a mapping you have to set the type of data you will put
in each field and you can define other parameters.
Here we want some fields not to be analyzed so we just precise the
property index and set it to "not_analyzed".
*/
foreach ($data as $key => $value) {
if ($key == "node_url" || ($key = "node_path")) {
$mapping_properties[$key] = array("type" => "string", "index" => "not_analyzed");
} else {
$type = gettype($value);
if ($type != "integer" && $type != "boolean" && $type != "double") {
$type = "string";
}
$mapping_properties[$key] = array("type" => $type, "index" => "simple");
}
}
$mapping = new Elastica\Type\Mapping();
$mapping->setType($this->currentType);
$mapping->setProperties($mapping_properties);
$mapping->send();
$doc = new Elastica\Document($this->nextId, $data);
$this->nextId++;
/*if (isSet($ajxpNode->indexableMetaKeys["user"]) && count($ajxpNode->indexableMetaKeys["user"]) && AuthService::usersEnabled() && AuthService::getLoggedUser() != null) {
$privateDoc = new Zend_Search_Lucene_Document();
$privateDoc->addField(Zend_Search_Lucene_Field::Keyword("node_url", $ajxpNode->getUrl()), SystemTextEncoding::getEncoding());
$privateDoc->addField(Zend_Search_Lucene_Field::Keyword("node_path", str_replace("/", "AJXPFAKESEP", $ajxpNode->getPath())), SystemTextEncoding::getEncoding());
$privateDoc->addField(Zend_Search_Lucene_Field::Keyword("ajxp_scope", "user"));
$privateDoc->addField(Zend_Search_Lucene_Field::Keyword("ajxp_user", AuthService::getLoggedUser()->getId()));
foreach ($ajxpNode->indexableMetaKeys["user"] as $userField) {
if ($ajxpNode->$userField) {
$privateDoc->addField(Zend_search_Lucene_Field::keyword($userField, $ajxpNode->$userField));
}
}
$privateDoc->addField(Zend_Search_Lucene_Field::Binary("serialized_metadata", $serializedMeta));
$index->addDocument($privateDoc);
}
/*
if ($parseContent && in_array($ext, explode(",",$this->getFilteredOption("PARSE_CONTENT_TXT")))) {
$doc->addField(Zend_Search_Lucene_Field::unStored("body", file_get_contents($ajxpNode->getUrl())));
}
$unoconv = $this->getFilteredOption("UNOCONV");
if ($parseContent && !empty($unoconv) && in_array($ext, array("doc", "odt", "xls", "ods"))) {
$targetExt = "txt";
$pipe = false;
if (in_array($ext, array("xls", "ods"))) {
$targetExt = "csv";
} else if (in_array($ext, array("odp", "ppt"))) {
$targetExt = "pdf";
$pipe = true;
}
$realFile = call_user_func(array($ajxpNode->wrapperClassName, "getRealFSReference"), $ajxpNode->getUrl());
$unoconv = "HOME=".AJXP_Utils::getAjxpTmpDir()." ".$unoconv." --stdout -f $targetExt ".escapeshellarg($realFile);
if ($pipe) {
$newTarget = str_replace(".$ext", ".pdf", $realFile);
$unoconv.= " > $newTarget";
register_shutdown_function("unlink", $newTarget);
}
$output = array();
exec($unoconv, $output, $return);
if (!$pipe) {
$out = implode("\n", $output);
$enc = 'ISO-8859-1';
$asciiString = iconv($enc, 'ASCII//TRANSLIT//IGNORE', $out);
$doc->addField(Zend_Search_Lucene_Field::unStored("body", $asciiString));
} else {
$ext = "pdf";
}
}
$pdftotext = $this->getFilteredOption("PDFTOTEXT");
if ($parseContent && !empty($pdftotext) && in_array($ext, array("pdf"))) {
$realFile = call_user_func(array($ajxpNode->wrapperClassName, "getRealFSReference"), $ajxpNode->getUrl());
if ($pipe && isset($newTarget) && is_file($newTarget)) {
$realFile = $newTarget;
}
$cmd = $pdftotext." ".escapeshellarg($realFile)." -";
$output = array();
exec($cmd, $output, $return);
$out = implode("\n", $output);
$enc = 'UTF8';
$asciiString = iconv($enc, 'ASCII//TRANSLIT//IGNORE', $out);
$doc->addField(Zend_Search_Lucene_Field::unStored("body", $asciiString));
}
*/
/* we update the last id in the file */
$file = fopen($this->lastIdPath, "w");
fputs($file, $this->nextId - 1);
fclose($file);
$this->currentType->addDocument($doc);
}
private function getType()
{
$index = $this->getConnection()->getIndex('mw_cirrus_versions');
if (!$index->exists()) {
$this->outputIndented("Creating tracking index...");
$index->create(array('number_of_shards' => 1, 'auto_expand_replicas' => '0-2'), true);
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($index->getType('version'));
$mapping->setProperties(array('analysis_maj' => array('type' => 'long', 'include_in_all' => false), 'analysis_min' => array('type' => 'long', 'include_in_all' => false), 'mapping_maj' => array('type' => 'long', 'include_in_all' => false), 'mapping_min' => array('type' => 'long', 'include_in_all' => false), 'shard_count' => array('type' => 'long', 'include_in_all' => false)));
$mapping->send();
$this->output("done\n");
}
return $index->getType('version');
}
public function createIndex($index, $config)
{
$response = $index->create($config['config'], $config['options']);
if ($response->isOk()) {
// lets register the mappings
foreach ($config['mappings'] as $type => $type_config) {
$type = $index->getType($type);
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($type);
$mapping->setParam('index_analyzer', 'indexAnalyzer');
$mapping->setParam('search_analyzer', 'searchAnalyzer');
$mapping->setParam('_boost', array('name' => '_boost', 'null_value' => 1.0));
$mapping->setProperties($type_config['properties']);
$mapping->send();
}
}
return $response;
}
/**
* @param AJXP_Node $ajxpNode
* @throws Exception
*/
public function createIndexedDocument($ajxpNode)
{
$ajxpNode->loadNodeInfo();
$parseContent = $this->indexContent;
if ($parseContent && $ajxpNode->bytesize > $this->getFilteredOption("PARSE_CONTENT_MAX_SIZE")) {
$parseContent = false;
}
$data = array();
$data["node_url"] = $ajxpNode->getUrl();
$data["node_path"] = str_replace("/", "AJXPFAKESEP", $ajxpNode->getPath());
$data["basename"] = basename($ajxpNode->getPath());
$data["ajxp_node"] = "yes";
$data["ajxp_scope"] = "shared";
$data["serialized_metadata"] = base64_encode(serialize($ajxpNode->metadata));
$data["ajxp_modiftime"] = date("Ymd", $ajxpNode->ajxp_modiftime);
$data["ajxp_bytesize"] = $ajxpNode->bytesize;
$ajxpMime = $ajxpNode->ajxp_mime;
if (empty($ajxpMime)) {
$data["ajxp_mime"] = pathinfo($ajxpNode->getLabel(), PATHINFO_EXTENSION);
} else {
$data["ajxp_mime"] = $ajxpNode->ajxp_mime;
}
if (isset($ajxpNode->indexableMetaKeys["shared"])) {
foreach ($ajxpNode->indexableMetaKeys["shared"] as $sharedField) {
if ($ajxpNode->{$sharedField}) {
$data[$sharedField] = $ajxpNode->{$sharedField};
}
}
}
foreach ($this->metaFields as $field) {
if ($ajxpNode->{$field} != null) {
$data["ajxp_meta_{$field}"] = $ajxpNode->{$field};
}
}
if ($parseContent) {
$body = $this->extractIndexableContent($ajxpNode);
if (!empty($body)) {
$data["body"] = $body;
}
}
$mapping = new Elastica\Type\Mapping();
$mapping->setType($this->currentType);
$mapping->setProperties($this->dataToMappingProperties($data));
$mapping->send();
$doc = new Elastica\Document($this->nextId, $data);
$this->currentType->addDocument($doc);
$this->nextId++;
if (isset($ajxpNode->indexableMetaKeys["user"]) && count($ajxpNode->indexableMetaKeys["user"]) && AuthService::usersEnabled() && AuthService::getLoggedUser() != null) {
$userData = array("ajxp_scope" => "user", "user" => AuthService::getLoggedUser()->getId(), "serialized_metadata" => $data["serialized_metadata"], "node_url" => $data["node_url"], "node_path" => $data["node_path"]);
$userData["ajxp_user"] = AuthService::getLoggedUser()->getId();
foreach ($ajxpNode->indexableMetaKeys["user"] as $userField) {
if ($ajxpNode->{$userField}) {
$userData[$userField] = $ajxpNode->{$userField};
}
}
$mapping = new Elastica\Type\Mapping();
$mapping->setType($this->currentType);
$mapping->setProperties($this->dataToMappingProperties($userData));
$mapping->send();
$doc = new Elastica\Document($this->nextId, $userData);
$this->currentType->addDocument($doc);
$this->nextId++;
}
/* we update the last id in the file */
$file = fopen($this->lastIdPath, "w");
fputs($file, $this->nextId - 1);
fclose($file);
}
/**
* Creates or updates Elasticsearch index.
*
* @link http://www.elasticsearch.org/guide/reference/mapping/core-types.html
* @link http://www.elasticsearch.org/guide/reference/mapping/multi-field-type.html
* @return NanoWebG_ElasticSearch_Model_Resource_Engine_Elasticsearch_Client
* @throws Exception
*/
protected function _prepareIndex()
{
try {
$indexSettings = array('settings' => $this->_getIndexSettings());
$index = $this->getIndex($this->_index);
// Using custom _indexExists() method because of a bug in index creation when
// calling exists() method on $index (ignores settings).
if (!$this->_indexExists($this->_index)) {
$indexSettings['settings']['number_of_shards'] = (int) $this->getConfig('number_of_shards');
$index->create($indexSettings);
} else {
$index->setSettings(array('settings' => array('number_of_replicas' => $indexSettings['settings']['number_of_replicas'])));
}
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($index->getType('product'));
$mapping->setProperties($this->_getIndexProperties());
Mage::dispatchEvent('nanowebg_elasticsearch_mapping_send_before', array('client' => $this, 'mapping' => $mapping));
$mapping->send();
} catch (Exception $e) {
Mage::getSingleton('adminhtml/session')->addError($e->getMessage());
throw $e;
}
return $this;
}
/**
* Creates the "mapping" for this searchtype, on the elasticsearch server
*
* TODO: it would be good to be able to make elasticsearch just delete the contents of one
* searchtype, perhaps identified by its mapping? But I haven't been able to figure out how
* to do that, so instead you can only delete the whole index at once.
* @param string $type
*/
private static function set_mapping($type)
{
// usr,interaction_instance,interaction_forum_post,view,group,artefact
$ES_class = 'ElasticsearchType_' . $type;
if ($ES_class::$mappingconf === false) {
return false;
}
$elasticaClient = self::make_client();
$elasticaIndex = $elasticaClient->getIndex(self::get_write_indexname());
$elasticaAnalyzer = get_config_plugin('search', 'elasticsearch', 'analyzer');
if (!isset($elasticaAnalyzer)) {
$elasticaAnalyzer = 'mahara_analyzer';
}
// Load type
$elasticaType = $elasticaIndex->getType($type);
// Define mapping
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($elasticaType);
// we use mahara_analyzer created through elastica
$mapping->setParam('index_analyzer', $elasticaAnalyzer);
$mapping->setParam('search_analyzer', $elasticaAnalyzer);
// Define boost field
//$mapping->setParam('_boost', array('name' => '_boost', 'null_value' => 1.0));
// Set mapping
$mapping->setProperties($ES_class::$mappingconf);
// Send mapping to type
$mapping->send();
}
/**
* @param \Elastica\Type $type
* @param bool $reset
*/
public function createElasticType($type = null, $reset = false)
{
if (method_exists($this->owner, 'createElasticType')) {
$this->owner->createElasticType($type, $reset);
return;
}
if ($type && $type->exists() && $reset) {
$type->delete();
}
!$type && ($type = $this->getElasticIndex()->getType($this->elasticTypeName));
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($type)->setProperties($this->elasticProperties())->setParam('dynamic', 'strict')->send();
}
protected function mapping()
{
// Set mapping only if mapping is set and if type does not exist
if ($this->mapping && !$this->typeExist($this->current_type)) {
$ESmapping = new \Elastica\Type\Mapping();
$ESmapping->setType($this->elasticaType);
//If Language supported by ES
if (array_key_exists($this->lang, $this->langAnalyzer)) {
$ESmapping->setParam("index_analyzer", $this->lang . "_Analyzer");
$ESmapping->setParam("search_analyzer", $this->lang . "_Analyzer");
} else {
$ESmapping->setParam("index_analyzer", "default");
$ESmapping->setParam("search_analyzer", "default");
}
if ($this->boost == "") {
$this->boost = 1.0;
}
$ESmapping->setParam('_boost', array('name' => 'boost', 'null_value' => $this->boost));
if ($this->sourceExcludes) {
$ESmapping->setSource(array('excludes' => $this->sourceExcludes));
}
// Set mapping
$ESmapping->setProperties($this->mapping);
// Send mapping to type
$ESmapping->send();
}
}
public function beginBootstrap()
{
$type = $this->getType();
if ($this->updateMapping) {
$this->logOutput('Updating the index mappings...');
$this->createIndex(true);
} elseif (!$type->getIndex()->exists()) {
$this->createIndex(false);
}
$settings = $type->getIndex()->getSettings();
$settings->setRefreshInterval(-1);
$term = new Elastica\Query\Term();
$term->setTerm('wiki', wfWikiId());
$query = new \Elastica\Query($term);
$type->deleteByQuery($query);
$mapping = new \Elastica\Type\Mapping();
$mapping->setType($type);
$mapping->setProperties(array('wiki' => array('type' => 'string', 'index' => 'not_analyzed'), 'localid' => array('type' => 'string', 'index' => 'not_analyzed'), 'uri' => array('type' => 'string', 'index' => 'not_analyzed'), 'language' => array('type' => 'string', 'index' => 'not_analyzed'), 'group' => array('type' => 'string', 'index' => 'not_analyzed'), 'content' => array('type' => 'string', 'fields' => array('content' => array('type' => 'string', 'index' => 'analyzed', 'term_vector' => 'yes'), 'prefix_complete' => array('type' => 'string', 'index_analyzer' => 'prefix', 'search_analyzer' => 'standard', 'term_vector' => 'yes'), 'case_sensitive' => array('type' => 'string', 'index' => 'analyzed', 'analyzer' => 'casesensitive', 'term_vector' => 'yes')))));
$mapping->send();
$this->waitUntilReady();
}
