function index() { $this->load->library('zend', 'Zend/Feed'); $this->load->library('zend', 'Zend/Search/Lucene'); $this->load->library('zend'); $this->zend->load('Zend/Feed'); $this->zend->load('Zend/Search/Lucene'); //Create index. $index = new Zend_Search_Lucene('C:\\xampp\\xampp\\htdocs\\controle_frota\\lucene\\feeds_index', true); $feeds = array('http://oglobo.globo.com/rss.xml?limite=50'); //grab each feed. foreach ($feeds as $feed) { $channel = Zend_Feed::import($feed); echo $channel->title() . '<br />'; //index each item. foreach ($channel->items as $item) { if ($item->link() && $item->title() && $item->description()) { //create an index doc. $doc = new Zend_Search_Lucene_Document(); $doc->addField(Zend_Search_Lucene_Field::Keyword('link', $this->sanitize($item->link()))); $doc->addField(Zend_Search_Lucene_Field::Text('title', $this->sanitize($item->title()))); $doc->addField(Zend_Search_Lucene_Field::Unstored('contents', $this->sanitize($item->description()))); echo "\tAdding: " . $item->title() . '<br />'; $index->addDocument($doc); } } } $index->commit(); echo $index->count() . ' Documents indexed.<br />'; }
public function edit($needFields = array(), $data = array(), $charset = 'UTF-8') { $index = new Zend_Search_Lucene(ZY_ROOT . '/index'); $doc = new Zend_Search_Lucene_Document(); foreach ($needFields as $key => $field) { switch ($field) { case 'keywords': $doc->addField(Zend_Search_Lucene_Field::Keyword($key, $data[$key], $charset)); break; case 'text': $doc->addField(Zend_Search_Lucene_Field::Text($key, $data[$key], $charset)); break; case 'unindexed': $doc->addField(Zend_Search_Lucene_Field::unindexed($key, $data[$key], $charset)); break; default: $doc->addField(Zend_Search_Lucene_Field::$field($key, $data[$key], $charset)); break; } } $index->addDocument($doc); $index->commit(); $index->optimize(); return TRUE; }
/** * This is the default 'index' action that is invoked * when an action is not explicitly requested by users. */ public function actionIndexing() { ini_set('max_execution_time', 0); ob_start(); $index = new Zend_Search_Lucene(Yii::getPathOfAlias($this->_indexFilesPath), true); $criteria = new CDbCriteria(); $criteria->compare('t.publish', 1); $criteria->order = 'album_id DESC'; //$criteria->limit = 10; $model = Albums::model()->findAll($criteria); foreach ($model as $key => $item) { if ($item->media_id != 0) { $images = Yii::app()->request->baseUrl . '/public/album/' . $item->album_id . '/' . $item->cover->media; } else { $images = ''; } $doc = new Zend_Search_Lucene_Document(); $doc->addField(Zend_Search_Lucene_Field::UnIndexed('id', CHtml::encode($item->album_id), 'utf-8')); $doc->addField(Zend_Search_Lucene_Field::Text('media', CHtml::encode($images), 'utf-8')); $doc->addField(Zend_Search_Lucene_Field::Text('title', CHtml::encode($item->title), 'utf-8')); $doc->addField(Zend_Search_Lucene_Field::Text('body', CHtml::encode(Utility::hardDecode(Utility::softDecode($item->body))), 'utf-8')); $doc->addField(Zend_Search_Lucene_Field::Text('url', CHtml::encode(Utility::getProtocol() . '://' . Yii::app()->request->serverName . Yii::app()->createUrl('album/site/view', array('id' => $item->album_id, 't' => Utility::getUrlTitle($item->title)))), 'utf-8')); $doc->addField(Zend_Search_Lucene_Field::UnIndexed('date', CHtml::encode(Utility::dateFormat($item->creation_date, true) . ' WIB'), 'utf-8')); $doc->addField(Zend_Search_Lucene_Field::UnIndexed('creation', CHtml::encode($item->user->displayname), 'utf-8')); $index->addDocument($doc); } echo 'Album Lucene index created'; $index->commit(); $this->redirect(Yii::app()->createUrl('article/search/indexing')); ob_end_flush(); }
/** * @param $doc \Zend_Search_Lucene_Document */ public function addDocumentToIndex($doc) { if ($doc instanceof \Zend_Search_Lucene_Document) { $this->index->addDocument($doc); \Pimcore\Logger::debug('LuceneSearch: Added to lucene index db entry'); } else { \Pimcore\Logger::error('LuceneSearch: could not parse lucene document '); } }
/** * A refactored method to add the document to the index.. * * @param int $docid * @param string $content * @param string $discussion */ private function addDocument($docid, $content, $discussion, $title, $version) { $doc = new Zend_Search_Lucene_Document(); $doc->addField(Zend_Search_Lucene_Field::Text('DocumentID', PHPLuceneIndexer::longToString($docid))); $doc->addField(Zend_Search_Lucene_Field::Text('Content', $content, 'UTF-8')); $doc->addField(Zend_Search_Lucene_Field::Text('Discussion', $discussion, 'UTF-8')); $doc->addField(Zend_Search_Lucene_Field::Text('Title', $title, 'UTF-8')); $doc->addField(Zend_Search_Lucene_Field::Text('Version', $version, 'UTF-8')); $this->lucene->addDocument($doc); }
/** * Add a document to the index. * * @param Zend_Search_Lucene_Document $document The document to be added. * @return Zend_Search_Lucene */ public function addDocument(Zend_Search_Lucene_Document $document) { // Search for documents with the same Key value. $term = new Zend_Search_Lucene_Index_Term($document->Key, 'Key'); $docIds = $this->termDocs($term); // Delete any documents found. foreach ($docIds as $id) { $this->delete($id); } return parent::addDocument($document); }
public function addDocument(Zend_Search_Lucene_Document $document) { $docRef = $document->docRef; $term = new Zend_Search_Lucene_Index_Term($docRef, 'docRef'); $query = new Zend_Search_Lucene_Search_Query_Term($term); $results = $this->find($query); if (count($results) > 0) { foreach ($results as $result) { $this->delete($result->id); } } return parent::addDocument($document); }
$results = $index->find($term); $query = Zend_Search_Lucene_Search_QueryParser::parse($term); $this->render('/sParameter/search', compact('results', 'term', 'query')); } } /** * Search index creation */ public function actionCreate() { $index = new Zend_Search_Lucene(Yii::getPathOfAlias('application.' . $this->_indexFiles), true); $posts = tAccount::model()->findAll(); foreach ($posts as $post) { $doc = new Zend_Search_Lucene_Document(); $doc->addField(Zend_Search_Lucene_Field::Text('account_no', CHtml::encode($post->account_no), 'utf-8'));
function fill_index() { for ($i = 0; $i < 10; $i++) { $index = new Zend_Search_Lucene('./data/index', true); $index->find("test"); $doc = new Zend_Search_Lucene_Document(); $doc->addField(Zend_Search_Lucene_Field::Text("test", getword())); $doc->addField(Zend_Search_Lucene_Field::UnStored("contents", getword())); $index->addDocument($doc); $index->commit(); $index->getDirectory()->close(); //comment this to see another bug :-| } }
public static function indexationAdd($indexationData) { $directory = Zend_Registry::get('lucene_index'); Zend_Search_Lucene_Analysis_Analyzer::setDefault(new Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8()); $doc = new Zend_Search_Lucene_Document(); $doc->addField(Zend_Search_Lucene_Field::Keyword('pageID', $indexationData['pageID'])); $doc->addField(Zend_Search_Lucene_Field::Keyword('moduleID', $indexationData['moduleID'])); $doc->addField(Zend_Search_Lucene_Field::Keyword('contentID', $indexationData['contentID'])); $doc->addField(Zend_Search_Lucene_Field::Keyword('languageID', $indexationData['languageID'])); $doc->addField(Zend_Search_Lucene_Field::Text('title', Cible_FunctionsGeneral::html2text($indexationData['title']))); $doc->addField(Zend_Search_Lucene_Field::Text('text', Cible_FunctionsGeneral::html2text($indexationData['text']))); $doc->addField(Zend_Search_Lucene_Field::UnIndexed('link', $indexationData['link'])); $doc->addField(Zend_Search_Lucene_Field::UnStored('contents', strtolower(Cible_FunctionsGeneral::removeAccents(Cible_FunctionsGeneral::html2text($indexationData['contents']))))); $newIndex = !is_dir($directory); $index = new Zend_Search_Lucene($directory, $newIndex); $index->addDocument($doc); $index->commit(); }
public function actionCreate() { $index = new Zend_Search_Lucene(Yii::getPathOfAlias('application.' . $this->_indexFiles), true); $items = Items::model()->findAll(); foreach ($items as $item) { $doc = new Zend_Search_Lucene_Document(); Zend_Search_Lucene_Analysis_Analyzer::setDefault(new Zend_Search_Lucene_Analysis_Analyzer_Common_TextNum_CaseInsensitive()); $doc->addField(Zend_Search_Lucene_Field::keyword('part_number', CHtml::encode($item->part_number), 'utf-8')); $doc->addField(Zend_Search_Lucene_Field::Text('name', CHtml::encode($item->name), 'utf-8')); $doc->addField(Zend_Search_Lucene_Field::Text('description', CHtml::encode($item->description), 'utf-8')); $doc->addField(Zend_Search_Lucene_Field::Text('barcode', CHtml::encode($item->barcode), 'utf-8')); $doc->addField(Zend_Search_Lucene_Field::float('available_quantity', CHtml::encode($item->available_quantity), 'utf-8')); $doc->addField(Zend_Search_Lucene_Field::float('current_quantity', CHtml::encode($item->current_quantity), 'utf-8')); $index->addDocument($doc); } $index->commit(); echo 'Lucene index created'; }
private function addEntryToSearchIndex($url, array $content) { $options = $this->getInvokeArg('bootstrap')->getOption('lucene'); $luceneDir = $options['dir']; $doc = new Zend_Search_Lucene_Document(); $doc->addField(Zend_Search_Lucene_Field::Keyword('url', $url)); foreach ($content as $key => $value) { if (!is_array($value) && $value && strlen($value) && !is_numeric($value)) { $doc->addField(Zend_Search_Lucene_Field::Text($key, $value)); } } $newIndex = !is_dir($luceneDir); $index = new Zend_Search_Lucene($luceneDir, $newIndex); $index->addDocument($doc); $index->commit(); // $index->optimize(); }
function createIndex() { global $db; $dir = FULL_PATH . 'Zend/app/tmp/cache'; $iterator = new RecursiveDirectoryIterator($dir); foreach (new RecursiveIteratorIterator($iterator, RecursiveIteratorIterator::CHILD_FIRST) as $file) { if ($file->isDir()) { rmdir($file->getPathname()); } else { unlink($file->getPathname()); } } $indexPath1 = FULL_PATH . 'Zend/app/tmp/cache/index1'; $index1 = new Zend_Search_Lucene($indexPath1, true); $q = "SELECT userid, FirstName, LastName, About, tr_About, BizDesc, tr_BizDesc FROM ! where 1"; $res1 = $db->getAll($q, array(TBL_BORROWER)); foreach ($res1 as $row1) { $doc1 = new Zend_Search_Lucene_Document(); // Add some information $doc1->addField(Zend_Search_Lucene_Field::Keyword('pk', $row1['userid'])); $doc1->addField(Zend_Search_Lucene_Field::UnIndexed('document_id', $row1['userid'])); $doc1->addField(Zend_Search_Lucene_Field::UnStored('document_FirstName', $row1['FirstName']), 'utf-8'); $doc1->addField(Zend_Search_Lucene_Field::UnStored('document_LastName', $row1['LastName']), 'utf-8'); $doc1->addField(Zend_Search_Lucene_Field::UnStored('document_About', $row1['About']), 'utf-8'); $doc1->addField(Zend_Search_Lucene_Field::UnStored('document_tr_About', $row1['tr_About']), 'utf-8'); $doc1->addField(Zend_Search_Lucene_Field::UnStored('document_BizDesc', $row1['BizDesc']), 'utf-8'); $doc1->addField(Zend_Search_Lucene_Field::UnStored('document_tr_BizDesc', $row1['tr_BizDesc']), 'utf-8'); $index1->addDocument($doc1); } $index1->commit(); $indexPath2 = FULL_PATH . 'Zend/app/tmp/cache/index2'; $index2 = new Zend_Search_Lucene($indexPath2, true); $p = "SELECT borrowerid,loanid, loanuse,tr_loanuse FROM ! where 1"; $res2 = $db->getAll($p, array(TBL_LOANAPPLIC)); foreach ($res2 as $row2) { $doc2 = new Zend_Search_Lucene_Document(); // Add some information $doc2->addField(Zend_Search_Lucene_Field::Keyword('pk', $row2['loanid'])); $doc2->addField(Zend_Search_Lucene_Field::UnIndexed('document_id', $row2['borrowerid'])); $doc2->addField(Zend_Search_Lucene_Field::UnStored('document_loanuse', $row2['loanuse']), 'utf-8'); $doc2->addField(Zend_Search_Lucene_Field::UnStored('document_tr_loanuse', $row2['tr_loanuse']), 'utf-8'); $index2->addDocument($doc2); } $index2->commit(); $indexPath3 = FULL_PATH . 'Zend/app/tmp/cache/index3'; $index3 = new Zend_Search_Lucene($indexPath3, true); $r = "SELECT id,receiverid, message, tr_message FROM ! where 1"; $res3 = $db->getAll($r, array('zi_comment')); foreach ($res3 as $row3) { $doc3 = new Zend_Search_Lucene_Document(); // Add some information $doc3->addField(Zend_Search_Lucene_Field::Keyword('pk', $row3['id'])); $doc3->addField(Zend_Search_Lucene_Field::UnIndexed('document_id', $row3['receiverid'])); $doc3->addField(Zend_Search_Lucene_Field::UnStored('document_message', $row3['message']), 'utf-8'); $doc3->addField(Zend_Search_Lucene_Field::UnStored('document_tr_message', $row3['tr_message']), 'utf-8'); $index3->addDocument($doc3); } $index3->commit(); /*$queryStr= "use"; $query = Zend_Search_Lucene_Search_QueryParser::parse($queryStr); $index = Zend_Search_Lucene::open(FULL_PATH.'Zend/app/tmp/cache/index3'); $results = $index->find($query); //print_r($results); $count = 0; foreach ($results as $result) { $data[$count]["userid"] = $result->document_id; $count++; } print_R($data);exit;*/ $count = 0; foreach (new RecursiveIteratorIterator($iterator, RecursiveIteratorIterator::CHILD_FIRST) as $file) { $count++; } return $count; }
/** * This will refresh the search index for this topic (for all message content under this topic) * @param Zend_Search_Lucene $objIndex should be null if we are updating just one -- but for bulk index updates, you can pass in an already loaded index file * @return void */ public function RefreshSearchIndex($objIndex = null) { // Currently only implemented for Forum-based topic/message searches if ($this->TopicLink->TopicLinkTypeId != TopicLinkType::Forum) { return; } if (!$objIndex) { $objIndex = new Zend_Search_Lucene(__SEARCH_INDEXES__ . '/topics'); $blnIndexProvided = false; } else { $blnIndexProvided = true; } // Retrievew the Index Documents (if applicable) to delete them from the index $objSearchTerm = new Zend_Search_Lucene_Index_Term($this->Id, 'db_id'); foreach ($objIndex->termDocs($objSearchTerm) as $intDocId) { $objIndex->delete($intDocId); } // Create the Message Contents for this Topic $strContents = null; foreach ($this->GetMessageArray(QQ::OrderBy(QQN::Message()->ReplyNumber)) as $objMessage) { $strMessage = strip_tags(trim($objMessage->CompiledHtml)); $strMessage = html_entity_decode($strMessage, ENT_QUOTES, 'UTF-8'); $strContents .= $strMessage . "\r\n\r\n"; } // Create the Document $objDocument = new Zend_Search_Lucene_Document(); $objDocument->addField(Zend_Search_Lucene_Field::Keyword('db_id', $this->Id)); $objDocument->addField(Zend_Search_Lucene_Field::UnIndexed('topic_link_id', $this->TopicLinkId)); $objDocument->addField(Zend_Search_Lucene_Field::UnIndexed('topic_link_type_id', $this->TopicLink->TopicLinkTypeId)); $objDocument->addField(Zend_Search_Lucene_Field::UnIndexed('message_count', $this->MessageCount)); $objDocument->addField(Zend_Search_Lucene_Field::UnIndexed('last_post_date', $this->LastPostDate->Timestamp)); $objDocument->addField(Zend_Search_Lucene_Field::Text('title', $this->Name)); $objDocument->addField(Zend_Search_Lucene_Field::UnStored('contents', trim($strContents))); // Add Document to Index $objIndex->addDocument($objDocument); // Only call commit on the index if it was provided for us if (!$blnIndexProvided) { $objIndex->commit(); } }
/** * addToIndex * @param string $strIndexPath * @param string $strKey * @author Thomas Schedler <*****@*****.**> * @version 1.0 */ protected final function addToIndex($strIndexPath, $strKey) { try { if (!is_object($this->objIndex) || !$this->objIndex instanceof Zend_Search_Lucene) { if (count(scandir($strIndexPath)) > 2) { $this->objIndex = Zend_Search_Lucene::open($strIndexPath); } else { $this->objIndex = Zend_Search_Lucene::create($strIndexPath); } } $objDoc = new Zend_Search_Lucene_Document(); $objDoc->addField(Zend_Search_Lucene_Field::keyword('key', $strKey)); $objDoc->addField(Zend_Search_Lucene_Field::unIndexed('date', $this->setup->getPublishDate('d.m.Y'))); $objDoc->addField(Zend_Search_Lucene_Field::unIndexed('rootLevelId', $this->setup->getRootLevelId())); /** * index fields */ foreach ($this->setup->FieldNames() as $strField => $intFieldType) { $objField = $this->setup->getField($strField); if (is_object($objField) && $objField->idSearchFieldTypes != Search::FIELD_TYPE_NONE) { $strValue = ''; if (is_array($objField->getValue()) && $objField->sqlSelect != '') { $arrIds = $objField->getValue(); $sqlSelect = $objField->sqlSelect; if (is_array($arrIds)) { if (count($arrIds) > 0) { $strReplaceWhere = ''; foreach ($arrIds as $strId) { $strReplaceWhere .= $strId . ','; } $strReplaceWhere = trim($strReplaceWhere, ','); $objReplacer = new Replacer(); $sqlSelect = $objReplacer->sqlReplacer($sqlSelect, $this->setup->getLanguageId(), $this->setup->getRootLevelId(), ' AND tbl.id IN (' . $strReplaceWhere . ')'); $objCategoriesData = $this->core->dbh->query($sqlSelect)->fetchAll(Zend_Db::FETCH_OBJ); if (count($objCategoriesData) > 0) { foreach ($objCategoriesData as $objCategories) { $strValue .= $objCategories->title . ', '; } $strValue = rtrim($strValue, ', '); } } } } else { $strValue = $objField->getValue(); } if ($strValue != '') { switch ($objField->idSearchFieldTypes) { case Search::FIELD_TYPE_KEYWORD: $objDoc->addField(Zend_Search_Lucene_Field::keyword($strField, $strValue, $this->core->sysConfig->encoding->default)); break; case Search::FIELD_TYPE_UNINDEXED: $objDoc->addField(Zend_Search_Lucene_Field::unIndexed($strField, $strValue, $this->core->sysConfig->encoding->default)); break; case Search::FIELD_TYPE_BINARY: $objDoc->addField(Zend_Search_Lucene_Field::binary($strField, $strValue, $this->core->sysConfig->encoding->default)); break; case Search::FIELD_TYPE_TEXT: $objDoc->addField(Zend_Search_Lucene_Field::text($strField, $strValue, $this->core->sysConfig->encoding->default)); break; case Search::FIELD_TYPE_UNSTORED: $objDoc->addField(Zend_Search_Lucene_Field::unStored($strField, strip_tags($strValue), $this->core->sysConfig->encoding->default)); break; } } } } // Add document to the index. $this->objIndex->addDocument($objDoc); $this->objIndex->optimize(); } catch (Exception $exc) { $this->core->logger->err($exc); } }
/** * Search index creation */ public function actionCreate() { /** * Если это не AJAX-запрос - посылаем: **/ if (!Yii::app()->getRequest()->getIsPostRequest() && !Yii::app()->getRequest()->getIsAjaxRequest()) { throw new CHttpException(404, Yii::t('ZendSearchModule.zendsearch', 'Page was not found!')); } try { // Папка для хранения индекса поиска $indexFiles = Yii::app()->getModule('zendsearch')->indexFiles; // Модели, включенные в индекс $searchModels = Yii::app()->getModule('zendsearch')->searchModels; // Лимит количества символов к описанию превью найденной страницы $limit = 600; SetLocale(LC_ALL, 'ru_RU.UTF-8'); $analyzer = new Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8Num_CaseInsensitive(); Zend_Search_Lucene_Analysis_Analyzer::setDefault($analyzer); $index = new Zend_Search_Lucene(Yii::getPathOfAlias('application.' . $indexFiles), true); $messages = []; if (extension_loaded('iconv') === true) { // Пробежаться по всем моделям и добавить их в индекс foreach ($searchModels as $modelName => $model) { if (!empty($model['path'])) { Yii::import($model['path']); } if (!isset($model['module'])) { $messages[] = Yii::t('ZendSearchModule.zendsearch', 'Update config file or module, Module index not found for model "{model}"!', ['{model}' => $modelName]); } elseif (is_file(Yii::getPathOfAlias($model['path']) . '.php') && Yii::app()->hasModule($model['module'])) { $criteria = isset($model['criteria']) ? $model['criteria'] : []; $searchNodes = $modelName::model()->findAll(new CDbCriteria($criteria)); foreach ($searchNodes as $node) { $doc = new Zend_Search_Lucene_Document(); $doc->addField(Zend_Search_Lucene_Field::Text('title', CHtml::encode($node->{$model}['titleColumn']), 'UTF-8')); $link = str_replace('{' . $model['linkColumn'] . '}', $node->{$model}['linkColumn'], $model['linkPattern']); $doc->addField(Zend_Search_Lucene_Field::Text('link', $link, 'UTF-8')); $contentColumns = explode(',', $model['textColumns']); $i = 0; foreach ($contentColumns as $column) { $content = $this->cleanContent($node->{$column}); if ($i == 0) { $doc->addField(Zend_Search_Lucene_Field::Text('content', $content, 'UTF-8')); $description = $this->cleanContent($this->previewContent($node->{$column}, $limit)); $doc->addField(Zend_Search_Lucene_Field::Text('description', $description, 'UTF-8')); } else { $doc->addField(Zend_Search_Lucene_Field::Text('content' . $i, $content, 'UTF-8')); } $i++; } $index->addDocument($doc); } $index->optimize(); $index->commit(); } else { $messages[] = Yii::t('ZendSearchModule.zendsearch', 'Module "{module}" not installed!', ['{module}' => $model['module']]); } } } else { $messages[] = Yii::t('ZendSearchModule.zendsearch', 'This module require "Iconv" extension!'); } Yii::app()->ajax->raw(empty($messages) ? Yii::t('ZendSearchModule.zendsearch', 'Index updated successfully!') : Yii::t('ZendSearchModule.zendsearch', 'There is an error!') . ': ' . implode("\n", $messages)); } catch (Exception $e) { Yii::app()->ajax->raw(Yii::t('ZendSearchModule.zendsearch', 'There is an error!') . ":\n" . $e->getMessage()); } }
function insert($val = array()) { $this->link(); $index = new Zend_Search_Lucene($this->dir); $analyzerObj = new search_instance_analyzer_cjk(); $analyzerObj->addPreFilter(new search_instance_analyzer_filter_goods()); $analyzerObj->addPreFilter(new search_instance_analyzer_filter_cjk()); $analyzerObj->addFilter(new search_instance_token_filter_lowercase()); Zend_Search_Lucene_Analysis_Analyzer::setDefault($analyzerObj); $doc = new Zend_Search_Lucene_Document(); $doc->addField(Zend_Search_Lucene_Field::Text('goods_id', $val['goods_id'])); if (isset($val['product'][0]['price']['price']['price'])) { $pric = $val['product'][0]['price']['price']['price']; } else { if (is_array($val['product'])) { foreach ($val['product'] as $kp => $vp) { $pric = $vp['price']['price']['price']; } } } $doc->addField(Zend_Search_Lucene_Field::UnStored('cat_id', $val['category']['cat_id'])); $doc->addField(Zend_Search_Lucene_Field::UnStored('brand_id', $val['brand']['brand_id'])); $doc->addField(Zend_Search_Lucene_Field::UnStored('price', $this->priceChange($pric))); $doc->addField(Zend_Search_Lucene_Field::UnStored('marketable', 'true')); if (isset($val['props'])) { for ($i = 1; $i <= 28; $i++) { $p = 'p_' . $i; $doc->addField(Zend_Search_Lucene_Field::UnStored($p, $val['props'][$p]['value'])); } } if (is_array($val['keywords'])) { foreach ($val['keywords'] as $k => $v) { $keyword .= '#' . $v['keyword'] . '@'; } } if (is_array($val['product'])) { foreach ($val['product'] as $k => $v) { if (is_array($v['spec_desc']['spec_value_id'])) { foreach ($v['spec_desc']['spec_value_id'] as $key => $vals) { $spec .= '#' . $key . $vals . '@'; } } $bn .= '#' . $v['bn'] . '@'; } } //$name = '#'.$val['name'].'@'.$keyword.'#'.$val['bn'].'@'.$bn; $name = '#' . $val['name'] . '@'; $doc->addField(Zend_Search_Lucene_Field::UnStored('title', $name)); $doc->addField(Zend_Search_Lucene_Field::UnStored('keyword', $keyword)); $doc->addField(Zend_Search_Lucene_Field::UnStored('spec', $spec)); $doc->addField(Zend_Search_Lucene_Field::UnStored('bn', '#' . $val['bn'] . '@' . $bn)); $index->addDocument($doc); return $index->commit(); }
public static function saveSearchIndex() { $index = new Zend_Search_Lucene(sfConfig::get('sf_lib_dir') . '/modules/search/tmp/lucene.user.index', true); $doc = new Zend_Search_Lucene_Document(); $doc->addField(Zend_Search_Lucene_Field::Keyword('id', $this->pageDocument->getId())); $doc->addField(Zend_Search_Lucene_Field::Keyword('pageid', $this->pageDocument->getId())); $doc->addField(Zend_Search_Lucene_Field::Keyword('title', $this->pageDocument->getNavigationTitle())); $blobData = $this->pageDocument->getContent(); // $blockContents = $blobData->__toString(); $blockContents = $blobData; $doc->addField(Zend_Search_Lucene_Field::Unstored('contents', $blockContents . ' ' . $this->pageDocument->getNavigationTitle())); $index->addDocument($doc); $index->commit(); $hits = $index->find(strtolower('maquette')); foreach ($hits as $hit) { // echo $hit->score.'<br/>'; // echo $hit->id; // echo $hit->contents.'<br/>'; echo $hit->pageid; } }
/** * Index an object. * * @param mixed $item */ public function index($item) { /** * Never index users. */ if ($item instanceof NovemberUser) { return; } if (!$this->index) { return; } // Get the document ID. If it doesn't exist, // quit, otherwise we'll never be able to figure out // what to do with it. if (!isset($item->id)) { return; } $type = strtolower(get_class($item)); // $id = $this->convertToChars($item->id).$type; $id = $item->id . $type; $title = isset($item->title) ? $item->title : ''; $name = isset($item->name) ? $item->name : $title; $description = isset($item->description) ? $item->description : ''; // First see if the document exists. If it does, we need to // remove it first. $hits = $this->index->find("identifier:'" . $id . "'"); foreach ($hits as $hit) { /* @var $hit Zend_Search_Lucene_Search_QueryHit */ $this->index->delete($hit->id); } $doc = new Zend_Search_Lucene_Document(); /* @var $doc Zend_Search_Lucene_Document */ $doc->addField(Zend_Search_Lucene_Field::Keyword('identifier', $id)); $doc->addField(Zend_Search_Lucene_Field::Keyword('type', $type)); $doc->addField(Zend_Search_Lucene_Field::Keyword('id', $item->id)); /** * To be indexed, you MUST set the searchableFields array */ if (isset($item->searchableFields)) { if (count($item->searchableFields) == 0) { return; } foreach ($item->searchableFields as $field) { // get the value and add it, as below. $this->addPropertyValueToDocument($doc, $field, $item->{$field}); } } else { // Use reflection class to prevent reflectionObject finding // dynamically assigned properties. /*$obj = new ReflectionClass(get_class($item)); $props = $obj->getProperties(); foreach ($props as $property) { if ($property->isPublic()) { $prop = $property->getName(); if ($prop == 'constraints' || $prop == 'requiredFields' || $prop == 'searchableFields') continue; $value = $item->$prop; if (is_null($value)) continue; $this->addPropertyValueToDocument($doc, $property->getName(), $value); } }*/ return; } $this->log->debug("Indexing document " . $item->id); $this->index->addDocument($doc); }
/** * Reindexes the search engine. */ public function reindex() { Loader::library('3rdparty/Zend/Search/Lucene'); Loader::library('3rdparty/StandardAnalyzer/Analyzer/Standard/English'); $index = new Zend_Search_Lucene(DIR_FILES_CACHE_PAGES, true); //Zend_Search_Lucene_Analysis_Analyzer::setDefault(new Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8Num_CaseInsensitive()); Zend_Search_Lucene_Analysis_Analyzer::setDefault(new StandardAnalyzer_Analyzer_Standard_English()); $db = Loader::db(); $collection_attributes = Loader::model('collection_attributes'); $r = $db->query("select cID from Pages order by cID asc"); $g = Group::getByID(GUEST_GROUP_ID); $nh = Loader::helper('navigation'); while ($row = $r->fetchRow()) { $c = Page::getByID($row['cID'], 'ACTIVE'); if($c->getCollectionAttributeValue('exclude_search_index')) continue; $themeObject = $c->getCollectionThemeObject(); $g->setPermissionsForObject($c); if ($g->canRead()) { $pageID = md5($row['cID']); $doc = new Zend_Search_Lucene_Document(); $doc->addField(Zend_Search_Lucene_Field::Keyword('cIDhash', $pageID)); $doc->addField(Zend_Search_Lucene_Field::Unindexed('cID', $row['cID'])); $doc->addField(Zend_Search_Lucene_Field::Text('cName', $c->getCollectionName(), APP_CHARSET)); $doc->addField(Zend_Search_Lucene_Field::Keyword('ctHandle', $c->getCollectionTypeHandle())); $doc->addField(Zend_Search_Lucene_Field::Text('cDescription', $c->getCollectionDescription(), APP_CHARSET)); $doc->addField(Zend_Search_Lucene_Field::Text('cBody', $this->getBodyContentFromPage($c), APP_CHARSET)); if (is_object($themeObject)) { $doc->addField(Zend_Search_Lucene_Field::Text('cTheme', $themeObject->getThemeHandle())); } $doc->addField(Zend_Search_Lucene_Field::Text( 'cPath', $c->getCollectionPath())); if (count($this->cPathSections) > 0) { foreach($this->cPathSections as $var => $cPath) { $isInSection = (strstr(strtolower($c->getCollectionPath()), $cPath . '/')) ? 'true' : 'false'; $doc->addField(Zend_Search_Lucene_Field::Keyword($var, $isInSection)); } } $attributes=$c->getSetCollectionAttributes(); foreach($attributes as $attribute){ if ($attribute->isCollectionAttributeKeySearchable()) { $doc->addField(Zend_Search_Lucene_Field::Keyword( $attribute->akHandle, $c->getCollectionAttributeValue($attribute) )); } } $index->addDocument($doc); } } $result = new stdClass; $result->count = $index->count(); return $result; }
/** * This is the default 'index' action that is invoked * when an action is not explicitly requested by users. */ public function actionIndexing() { ini_set('max_execution_time', 0); ob_start(); $index = new Zend_Search_Lucene(Yii::getPathOfAlias($this->_indexFilesPath), true); $criteria = new CDbCriteria(); $now = new CDbExpression("NOW()"); $criteria->compare('t.publish', 1); $criteria->compare('date(published_date) <', $now); $criteria->order = 'article_id DESC'; //$criteria->limit = 10; $model = Articles::model()->findAll($criteria); foreach ($model as $key => $item) { if ($item->media_id != 0) { $images = Yii::app()->request->baseUrl . '/public/article/' . $item->article_id . '/' . $item->cover->media; } else { $images = ''; } if (in_array($item->cat_id, array(2, 3, 5, 6, 7, 18))) { $url = Yii::app()->createUrl('article/news/site/view', array('id' => $item->article_id, 't' => Utility::getUrlTitle($item->title))); } else { if (in_array($item->cat_id, array(9))) { $url = Yii::app()->createUrl('article/site/view', array('id' => $item->article_id, 't' => Utility::getUrlTitle($item->title))); } else { if (in_array($item->cat_id, array(10, 15, 16))) { $url = Yii::app()->createUrl('article/archive/site/view', array('id' => $item->article_id, 't' => Utility::getUrlTitle($item->title))); } else { if (in_array($item->cat_id, array(23, 24, 25))) { $url = Yii::app()->createUrl('article/newspaper/site/view', array('id' => $item->article_id, 't' => Utility::getUrlTitle($item->title))); } else { if (in_array($item->cat_id, array(13, 14, 20, 21))) { $url = Yii::app()->createUrl('article/regulation/site/download', array('id' => $item->article_id, 't' => Utility::getUrlTitle($item->title))); } else { if (in_array($item->cat_id, array(19))) { $url = Yii::app()->createUrl('article/announcement/site/download', array('id' => $item->article_id, 't' => Utility::getUrlTitle($item->title))); } } } } } } $doc = new Zend_Search_Lucene_Document(); $doc->addField(Zend_Search_Lucene_Field::UnIndexed('id', CHtml::encode($item->article_id), 'utf-8')); $doc->addField(Zend_Search_Lucene_Field::Keyword('category', CHtml::encode(Phrase::trans($item->cat->name, 2)), 'utf-8')); $doc->addField(Zend_Search_Lucene_Field::Text('media', CHtml::encode($images), 'utf-8')); $doc->addField(Zend_Search_Lucene_Field::Text('title', CHtml::encode($item->title), 'utf-8')); $doc->addField(Zend_Search_Lucene_Field::Text('body', CHtml::encode(Utility::hardDecode(Utility::softDecode($item->body))), 'utf-8')); $doc->addField(Zend_Search_Lucene_Field::Text('url', CHtml::encode(Utility::getProtocol() . '://' . Yii::app()->request->serverName . $url), 'utf-8')); $doc->addField(Zend_Search_Lucene_Field::UnIndexed('date', CHtml::encode(Utility::dateFormat($item->published_date, true) . ' WIB'), 'utf-8')); $doc->addField(Zend_Search_Lucene_Field::UnIndexed('creation', CHtml::encode($item->user->displayname), 'utf-8')); $index->addDocument($doc); } echo 'Artkel Lucene index created'; $index->commit(); $this->redirect(Yii::app()->createUrl('video/search/indexing')); ob_end_flush(); }
/** * addToIndex * @param string $strIndexPath * @param string $strKey * @param PageContainer $objParentPageContainer * @author Thomas Schedler <*****@*****.**> * @version 1.0 */ protected final function addToIndex($strIndexPath, $strKey, $objParentPageContainer = null, $arrParentFolderIds = array()) { try { $this->core->logger->debug('massiveart->generic->data->types->GenericDataTypeAbstract->addToIndex(' . $strIndexPath . ', ' . $strKey . ')'); if (!is_object($this->objIndex) || !$this->objIndex instanceof Zend_Search_Lucene) { if (count(scandir($strIndexPath)) > 2) { $this->objIndex = Zend_Search_Lucene::open($strIndexPath); } else { $this->objIndex = Zend_Search_Lucene::create($strIndexPath); } } Zend_Search_Lucene_Analysis_Analyzer::setDefault(new Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8Num_CaseInsensitive()); $objDoc = new Zend_Search_Lucene_Document(); $objDoc->addField(Zend_Search_Lucene_Field::keyword('key', $strKey)); if ($this->setup->getLanguageFallbackId() > 0 && $this->setup->getLanguageFallbackId() != $this->setup->getLanguageId()) { $objDoc->addField(Zend_Search_Lucene_Field::keyword('languageId', $this->setup->getLanguageFallbackId())); } else { $objDoc->addField(Zend_Search_Lucene_Field::keyword('languageId', $this->setup->getLanguageId())); } $objDoc->addField(Zend_Search_Lucene_Field::keyword('rootLevelId', $this->setup->getRootLevelId())); $objDoc->addField(Zend_Search_Lucene_Field::unIndexed('date', $this->setup->getPublishDate('d.m.Y'))); $objDoc->addField(Zend_Search_Lucene_Field::unIndexed('elementTypeId', $this->setup->getElementTypeId())); if ($objParentPageContainer !== null && $objParentPageContainer instanceof PageContainer) { if (count($objParentPageContainer->getEntries()) > 0) { $objDoc->addField(Zend_Search_Lucene_Field::unIndexed('parentPages', serialize($objParentPageContainer->getEntries()))); $objDoc->addField(Zend_Search_Lucene_Field::keyword('rootLevelId', end($objParentPageContainer->getEntries())->rootLevelId)); } } if (count($arrParentFolderIds) > 0) { $objDoc->addField(Zend_Search_Lucene_Field::unStored('parentFolderIds', implode(',', $arrParentFolderIds))); } /** * index fields */ foreach ($this->setup->FieldNames() as $strField => $intFieldType) { $objField = $this->setup->getField($strField); if (is_object($objField) && $objField->idSearchFieldTypes != Search::FIELD_TYPE_NONE) { $this->indexFieldNow($objField, $strField, $intFieldType, $objField->getValue(), $objDoc); } } foreach ($this->setup->MultiplyRegionIds() as $intRegionId) { $objRegion = $this->setup->getRegion($intRegionId); if ($objRegion instanceof GenericElementRegion) { $intRegionPosition = 0; foreach ($objRegion->RegionInstanceIds() as $intRegionInstanceId) { $intRegionPosition++; foreach ($objRegion->FieldNames() as $strField => $intFieldType) { $objField = $objRegion->getField($strField); if (is_object($objField) && $objField->idSearchFieldTypes != Search::FIELD_TYPE_NONE) { $this->indexFieldNow($objField, $objField->name . '_' . $intRegionPosition, $intFieldType, $objField->getInstanceValue($intRegionInstanceId), $objDoc); } } } } } // Add document to the index. $this->objIndex->addDocument($objDoc); unset($objDoc); $this->objIndex->optimize(); } catch (Exception $exc) { $this->core->logger->err($exc); } }
public function actionDoIndex() { $document = Document::model()->findAll(); Yii::import('application.vendor.*'); require_once 'Zend/Search/Lucene.php'; $index = new Zend_Search_Lucene(Yii::getPathOfAlias('application.index'), true); try { foreach ($document as $doc) { $indexDoc = new Zend_Search_Lucene_Document(); $indexDoc->addField(Zend_Search_Lucene_Field::text("title", CHtml::encode($doc->title), 'utf-8')); $indexDoc->addField(Zend_Search_Lucene_Field::text("content", CHtml::encode($doc->content), 'utf-8')); $indexDoc->addField(Zend_Search_Lucene_Field::text("author", CHtml::encode($doc->author), 'utf-8')); $indexDoc->addField(Zend_Search_Lucene_Field::unIndexed("id", CHtml::encode($doc->id), 'utf-8')); $indexDoc->addField(Zend_Search_Lucene_Field::keyword("url", CHtml::encode($doc->url), 'utf-8')); $index->addDocument($indexDoc); } $index->commit(); Yii::app()->user->setFlash('success', 'Indexing success!'); } catch (Exception $e) { Yii::app()->user->setFlash('error', 'Indexing fail, try again!'); } $this->redirect(array('document/admin')); }
* @package Zend_Search_Lucene * @subpackage Demos * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** * @see Zend_Feed */ require_once 'Zend/Feed.php'; /** * @see Zend_Search_Lucene */ require_once 'Zend/Search/Lucene.php'; //create the index $index = new Zend_Search_Lucene('/tmp/feeds_index', true); // index each item $rss = Zend_Feed::import('http://feeds.feedburner.com/ZendDeveloperZone'); foreach ($rss->items as $item) { $doc = new Zend_Search_Lucene_Document(); if ($item->link && $item->title && $item->description) { $link = htmlentities(strip_tags($item->link())); $doc->addField(Zend_Search_Lucene_Field::UnIndexed('link', $link)); $title = htmlentities(strip_tags($item->title())); $doc->addField(Zend_Search_Lucene_Field::Text('title', $title)); $contents = htmlentities(strip_tags($item->description())); $doc->addField(Zend_Search_Lucene_Field::Text('contents', $contents)); echo "Adding {$item->title()}...\n"; $index->addDocument($doc); } } $index->commit();
function rebuild_search_indexes() { global $success_msg; global $error_msg; global $warning_msg; global $all_settings; global $indexable_folders; $index_folder = get_setting('search_indexes_folder', $all_settings); try { setlocale(LC_CTYPE, LOCALE); Zend_Search_Lucene_Analysis_Analyzer::setDefault(new Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8_CaseInsensitive()); $index = new Zend_Search_Lucene($index_folder, true); $files_to_index = get_website_files($indexable_folders); foreach ($files_to_index as $html_file => $page_url) { if (can_index_website_file($html_file)) { $f1 = strtolower($html_file); if (end_with($f1, 'html') || end_with($f1, 'htm')) { $file_content = file_get_contents($html_file); } elseif (end_with($f1, 'php')) { if (is_http_code_200($page_url)) { $file_content = get_url_content($page_url); } } if (isset($file_content)) { $file_content = '<html>' . strstr($file_content, '<head'); $doc = Zend_Search_Lucene_Document_Html::loadHTML($file_content, true, 'UTF-8'); $doc->addField(Zend_Search_Lucene_Field::Text('url', $page_url, 'UTF-8')); $index->addDocument($doc); flush(); } } } $broken_urls = array(); foreach (get_dynamic_urls(get_setting('search_dynamic_pages', $all_settings)) as $url) { if (is_http_code_200($url)) { $content = get_url_content($url); $content = '<html>' . strstr($content, '<head'); $doc = Zend_Search_Lucene_Document_Html::loadHTML($content, true, 'UTF-8'); $doc->addField(Zend_Search_Lucene_Field::Text('url', $url, 'UTF-8')); $index->addDocument($doc); flush(); } else { array_push($broken_urls, $url); } } if (file_exists($index_folder)) { if (count($broken_urls) > 0) { $warning_msg = '<p>The website was successfully indexed, but the following URL\'s were skipped because they are broken:</p>'; $warning_msg .= '<ul class="disc">'; foreach ($broken_urls as $broken_url) { $warning_msg .= '<li><a href="' . $broken_url . '">' . $broken_url . '</a></li>'; } $warning_msg .= '</ul>'; $warning_msg .= '<p>Please remove them from the "List of dynamic pages" field.</p>'; } else { $success_msg = 'The website was successfully indexed.'; } } else { $error_msg = 'An error occurred during the website indexing. The error message is: the folder that stores the website indexes couldn\'t be created'; } } catch (Exception $e) { $error_msg = 'An error occurred during the website indexing. The error message is: ' . $e->getMessage(); } }
//get the record, should only be one foreach ($doc as $thisdoc) { mtrace(" Delete: {$thisdoc->title} (database id = {$thisdoc->dbid}, index id = {$thisdoc->id}, moodle instance id = {$thisdoc->docid})"); $dbcontrol->delDocument($thisdoc); $index->delete($thisdoc->id); } //add new modified document back into index if (!($add = $get_document_function($update->id, $update->itemtype))) { // ignore on errors continue; } //object to insert into db $dbid = $dbcontrol->addDocument($add); //synchronise db with index $add->addField(Zend_Search_Lucene_Field::Keyword('dbid', $dbid)); mtrace(" Add: {$add->title} (database id = {$add->dbid}, moodle instance id = {$add->docid})"); $index->addDocument($add); } } else { mtrace("No types to update.\n"); } mtrace("Finished {$mod->name}.\n"); } } } } //commit changes $index->commit(); //update index date set_config("search_indexer_update_date", $startupdatedate); mtrace("Finished {$update_count} updates");
function rebuild_search_indexes() { global $success_msg; global $error_msg; global $warning_msg; global $all_settings; $index_folder = get_setting('search_indexes_folder', $all_settings); try { $index = new Zend_Search_Lucene($index_folder, true); setlocale(LC_CTYPE, 'en_US'); foreach (get_all_html_files(dirname(__FILE__)) as $html_file => $html_url) { if (can_index_html_file($html_file)) { $file_content = file_get_contents($html_file); $file_content = '<html>' . strstr($file_content, '<head'); $doc = Zend_Search_Lucene_Document_Html::loadHTML($file_content); $doc->addField(Zend_Search_Lucene_Field::Text('url', $html_url, 'UTF-8')); $index->addDocument($doc); flush(); } } $broken_urls = array(); foreach (get_dynamic_urls(get_setting('search_dynamic_pages', $all_settings)) as $url) { $headers = get_headers($url); if (strrpos($headers[0], '200')) { $content = file_get_contents($url); $content = '<html>' . strstr($content, '<head'); $doc = Zend_Search_Lucene_Document_Html::loadHTML($content); $doc->addField(Zend_Search_Lucene_Field::Text('url', $url, 'UTF-8')); $index->addDocument($doc); flush(); } else { array_push($broken_urls, $url); } } if (file_exists($index_folder)) { if (count($broken_urls) > 0) { $warning_msg = '<p>The website was successfully indexed, but the following URL\'s were skipped because they are broken:</p>'; $warning_msg .= '<ul class="disc">'; foreach ($broken_urls as $broken_url) { $warning_msg .= '<li><a href="' . $broken_url . '">' . $broken_url . '</a></li>'; } $warning_msg .= '</ul>'; $warning_msg .= '<p>Please remove them from the "List of dynamic pages" field.</p>'; } else { $success_msg = 'The website was successfully indexed.'; } } else { $error_msg = 'An error occurred during the website indexing. The error message is: the folder that stores the website indexes couldn\'t be created'; } } catch (Exception $e) { $error_msg = 'An error occurred during the website indexing. The error message is: ' . $e->getMessage(); } }