public function testAddFieldMethodChaining()
{
$document = new Document();
$this->assertTrue($document->addField(Document\Field::Text('title', 'Title')) instanceof Document);
$document = new Document();
$document->addField(Document\Field::Text('title', 'Title'))->addField(Document\Field::Text('annotation', 'Annotation'))->addField(Document\Field::Text('body', 'Document body, document body, document body...'));
}
public function createDocument(EntityInterface $entity) : Document
{
$document = new Document();
$context = $this->createContext($entity);
$fieldsCollection = $context->getFieldsCollection();
$document->addField(Field::unIndexed('identifier', $entity->getId()));
$fieldsCollection->map(function (SearchField $field) use($document) {
$document->addField(Field::text($field->getName(), $field->getValue()));
});
return $document;
}
private function appendToDocument(Lucene\Document $document, $name, $value)
{
if (is_string($value)) {
$document->addField(Lucene\Document\Field::keyword($name, utf8_decode($value)));
} elseif ($value instanceof \DateTime) {
$document->addField(Lucene\Document\Field::keyword($name, $value->getTimestamp()));
} elseif (is_array($value)) {
$document->addField(Lucene\Document\Field::text($name, implode(',', $value)));
} else {
$document->addField(Lucene\Document\Field::binary($name, $value));
}
}
/**
* @param $data
* @param SearchIndexInterface $index
*
* @return IndexInterface
*/
public function index($data, SearchIndexInterface $index)
{
$this->unindex($data, $index);
$indexDoc = new Document();
$indexDoc->addField(Field::Keyword('news_id', $data->id));
$indexDoc->addField(Field::UnIndexed('type', "news"));
$indexDoc->addField(Field::UnIndexed('identifier', $data->id));
$indexDoc->addField(Field::UnIndexed('date_time', $data->created_date->format('c')));
$indexDoc->addField(Field::UnIndexed('date', $data->created_date->format('j. M. Y')));
$indexDoc->addField(Field::Text('title', $data->title, 'utf-8'));
$indexDoc->addField(Field::Text('body', $data->body, 'utf-8'));
$index->addDocument($indexDoc);
return $this;
}
/**
* @dataProvider searchResultDataProvider
*/
function testSearchLuceneResultContent($fileId, $name, $path, $size, $score, $mimeType, $modified, $container)
{
require_once __DIR__ . '/util/dummyindex.php';
$index = new DummyIndex();
$doc = new Document();
$doc->addField(Document\Field::Keyword('fileId', $fileId));
$doc->addField(Document\Field::Text('path', '/test/files' . $path, 'UTF-8'));
$doc->addField(Document\Field::unIndexed('mtime', $modified));
$doc->addField(Document\Field::unIndexed('size', $size));
$doc->addField(Document\Field::unIndexed('mimetype', $mimeType));
$index->addDocument($doc);
$hit = new QueryHit($index);
$hit->score = $score;
$hit->id = 0;
$hit->document_id = 0;
$searchResult = new \OCA\Search_Lucene\Search\LuceneResult($hit);
$this->assertInstanceOf('OCA\\Search_Lucene\\Search\\LuceneResult', $searchResult);
$this->assertEquals($fileId, $searchResult->id);
$this->assertEquals('lucene', $searchResult->type);
$this->assertEquals($path, $searchResult->path);
$this->assertEquals($name, $searchResult->name);
$this->assertEquals($mimeType, $searchResult->mime_type);
$this->assertEquals($size, $searchResult->size);
$this->assertEquals($score, $searchResult->score);
$this->assertEquals($modified, $searchResult->modified);
}
/**
* @param $data
* @param SearchIndexInterface $index
*
* @return IndexInterface
*/
public function index($data, SearchIndexInterface $index)
{
$this->unindex($data, $index);
$indexDoc = new Document();
$indexDoc->addField(Field::Keyword('group_id', $data->id));
$indexDoc->addField(Field::UnIndexed('type', "group"));
$indexDoc->addField(Field::UnIndexed('identifier', $data->url));
$indexDoc->addField(Field::UnIndexed('date_time', date('c')));
$indexDoc->addField(Field::UnIndexed('date', date('j. M. Y')));
$indexDoc->addField(Field::Text('title', $data->name_short, 'utf-8'));
$indexDoc->addField(Field::Text('body', $data->description, 'utf-8'));
$index->addDocument($indexDoc);
return $this;
}
/**
* Create or update an indexed document
*
* @param object $object
*/
public function index($object)
{
// create property accessor
$accessor = PropertyAccess::createPropertyAccessor();
// delete existing documents with same id
foreach ($this->index->find('id:' . $accessor->getValue($object, 'id')) as $hit) {
$this->index->delete($hit->id);
}
// create new Lucene document
$doc = new Document();
// add primary key to identify it in the search results
$doc->addField(Field::keyword('id', $accessor->getValue($object, 'id')));
// add entity class reference to identify it in the search results
$doc->addField(Field::unIndexed('entityClass', get_class($object)));
// analyze each property's annotations to see which ones must be add to the document
$reflClass = new ReflectionClass($object);
foreach ($reflClass->getProperties() as $property) {
$reflProperty = new \ReflectionProperty($object, $property->name);
$annotation = $this->reader->getPropertyAnnotation($reflProperty, '\\Keratine\\Lucene\\Mapping\\Annotation\\DocumentField');
if ($annotation) {
$value = $accessor->getValue($object, $property->name);
$value = $this->ensureString($value);
// use the appropriate indexing strategy for the field
switch ($annotation->type) {
case 'keyword':
$doc->addField(Field::keyword($property->name, $value, 'UTF-8'));
break;
case 'unIndexed':
$doc->addField(Field::unIndexed($property->name, $value, 'UTF-8'));
break;
case 'binary':
$doc->addField(Field::binary($property->name, $value));
break;
case 'text':
$doc->addField(Field::text($property->name, $value, 'UTF-8'));
break;
case 'unStored':
default:
$doc->addField(Field::unStored($property->name, $value, 'UTF-8'));
break;
}
}
}
// add the document to the index and commit it
$this->index->addDocument($doc);
$this->index->commit();
}
function testUpdate()
{
// preparation
$app = new Application();
$container = $app->getContainer();
// get an index
/** @var Index $index */
$index = $container->query('Index');
// add a document
$doc = new Document();
$doc->addField(Document\Field::Keyword('fileId', '1'));
$doc->addField(Document\Field::Text('path', '/somewhere/deep/down/the/rabbit/hole', 'UTF-8'));
$doc->addField(Document\Field::Text('users', 'alice', 'UTF-8'));
$index->index->addDocument($doc);
$index->commit();
// search for it
$idTerm = new Term('1', 'fileId');
$idQuery = new Query\Term($idTerm);
$query = new Query\Boolean();
$query->addSubquery($idQuery);
/** @var QueryHit $hit */
$hits = $index->find($query);
// get the document from the query hit
$foundDoc = $hits[0]->getDocument();
$this->assertEquals('alice', $foundDoc->getFieldValue('users'));
// delete the document from the index
//$index->index->delete($hit);
// change the 'users' key of the document
$foundDoc->addField(Document\Field::Text('users', 'bob', 'UTF-8'));
$this->assertEquals('bob', $foundDoc->getFieldValue('users'));
// add the document back to the index
$index->updateFile($foundDoc, '1');
$idTerm2 = new Term('1', 'fileId');
$idQuery2 = new Query\Term($idTerm2);
$query2 = new Query\Boolean();
$query2->addSubquery($idQuery2);
/** @var QueryHit $hit */
$hits2 = $index->find($query2);
// get the document from the query hit
$foundDoc2 = $hits2[0]->getDocument();
$this->assertEquals('bob', $foundDoc2->getFieldValue('users'));
}
public function testUpdate()
{
$this->connection->shouldReceive('getIndex')->andReturn($luceneIndex = m::mock());
$luceneIndex->shouldReceive('addDocument')->with(m::on(function ($arg) {
$doc = new Document();
$doc->addField(Field::keyword('primary_key', 1));
$doc->addField(Field::Keyword('class_uid', '12345'));
$field = Field::unStored('name', 'test name');
$field->boost = 1;
$doc->addField($field);
$field = Field::unStored('optional_attribute1', 'optional value');
$field->boost = 1;
$doc->addField($field);
$this->assertEquals($doc, $arg);
return true;
}))->once();
$luceneIndex->shouldReceive('find')->with(m::on(function ($arg) {
$term = new MultiTerm();
$term->addTerm(new Term(1, 'primary_key'), true);
$term->addTerm(new Term('12345', 'class_uid'), true);
$this->assertEquals($term, $arg);
return true;
}))->andReturnUsing(function () {
$hitMock = m::mock();
$hitMock->id = 10;
return [$hitMock];
})->once();
$luceneIndex->shouldReceive('delete')->with(10)->once();
$index = $this->createIndex();
$index->update($this->model);
}
/**
* Lists all Post models.
* @return mixed
*/
public function actionIndex()
{
$searchModel = new PostSearch();
$dataProvider = $searchModel->search(Yii::$app->request->post());
//setlocale(LC_ALL, 'en_US.UTF-8');
setlocale(LC_CTYPE, 'ru_RU.UTF-8');
//Lucene\Lucene::setDefaultSearchField('contents');
Lucene\Search\QueryParser::setDefaultEncoding('UTF-8');
Lucene\Analysis\Analyzer\Analyzer::setDefault(new Lucene\Analysis\Analyzer\Common\Utf8\CaseInsensitive());
Lucene\Lucene::setResultSetLimit(10);
// create blog posts index located in /data/posts_index ,make sure the folder is writable
$index = Lucene\Lucene::create('data/posts_index');
$posts = Post::find()->all();
//var_dump($posts);die();
// iterate through posts and build the index
foreach ($posts as $p) {
$doc = new Lucene\Document();
$doc->addField(Lucene\Document\Field::UnIndexed('entry_id', $p->id));
$doc->addField(Lucene\Document\Field::Keyword('title', $p->title));
$doc->addField(Lucene\Document\Field::text('contents', $p->content));
$index->addDocument($doc);
}
// commit the index
$index->commit();
//Lucene\Analysis\Analyzer\Analyzer::setDefault(new Lucene\Analysis\Analyzer\Common\Utf8\CaseInsensitive());
// explode the search query to individual words
$words = explode(' ', urldecode(Yii::$app->getRequest()->getQueryParam('q')));
// start a search query and add a term for each word to it
$query = new Lucene\Search\Query\MultiTerm();
foreach ($words as $w) {
$query->addTerm(new Lucene\Index\Term($w));
}
// open and query the index
$index = Lucene\Lucene::open('data/posts_index');
$results = $index->find($query);
// the search results
//var_dump($results);
return $this->render('index', ['searchModel' => $searchModel, 'dataProvider' => $dataProvider, 'search' => $results, 'query' => $query]);
}
/**
* インデックスファイルを生成
*/
public static function updateIndex()
{
if (empty(self::$igo)) {
self::$igo = new Tagger(array('dict_dir' => LIB_DIR . 'ipadic', 'reduce_mode' => true));
}
Analyzer::setDefault(new Utf8());
// 索引の作成
$index = Lucene::create(CACHE_DIR . self::INDEX_NAME);
foreach (Listing::pages() as $page) {
if (empty($page)) {
continue;
}
$wiki = Factory::Wiki($page);
// 読む権限がない場合スキップ
if (!$wiki->isReadable() || $wiki->isHidden()) {
continue;
}
/*
// HTML出力
$html[] = '<html><head>';
$html[] = '<meta http-equiv="Content-type" content="text/html; charset=UTF-8"/>';
$html[] = '<title>' . $wiki->title() . '</title>';
$html[] = '</head>';
$html[] = '<body>' . $wiki->render() . '</body>';
$html[] = '</html>';
*/
$doc = new LuceneDoc();
$doc->addField(Field::Text('title', $wiki->title()));
// Store document URL to identify it in the search results
$doc->addField(Field::Text('url', $wiki->uri()));
// Index document contents
//$contents = join(" ", self::$igo->wakati(strip_tags($wiki->render())));
$contents = strip_tags($wiki->render());
$doc->addField(Field::UnStored('contents', $contents));
// 索引へ文書の登録
$index->addDocument($doc);
}
$index->optimize();
}
public function generateIndexAction()
{
$searchIndexLocation = $this->getIndexLocation();
$index = Lucene\Lucene::create($searchIndexLocation);
$userTable = $this->getServiceLocator()->get('UserTable');
$uploadTable = $this->getServiceLocator()->get('UploadTable');
$allUploads = $uploadTable->fetchAll();
foreach ($allUploads as $fileUpload) {
$uploadOwner = $userTable->getById($fileUpload->getUserId());
// создание полей lucene
$fileUploadId = Document\Field::unIndexed('upload_id', $fileUpload->getId());
$label = Document\Field::Text('label', $fileUpload->getLabel());
$owner = Document\Field::Text('owner', $uploadOwner->getName());
$uploadPath = $this->getFileUploadLocation();
$fileName = $fileUpload->getFilename();
$filePath = $uploadPath . DIRECTORY_SEPARATOR . $fileName;
if (substr_compare($fileName, ".xlsx", strlen($fileName) - strlen(".xlsx"), strlen(".xlsx")) === 0) {
// Индексирование таблицы excel
$indexDoc = Lucene\Document\Xlsx::loadXlsxFile($filePath);
} else {
if (substr_compare($fileName, ".docx", strlen($fileName) - strlen(".docx"), strlen(".docx")) === 0) {
// Индексирование документа Word
$indexDoc = Lucene\Document\Docx::loadDocxFile($filePath);
} else {
$indexDoc = new Lucene\Document();
}
}
// создание нового документа и добавление всех полей
$indexDoc = new Lucene\Document();
$indexDoc->addField($label);
$indexDoc->addField($owner);
$indexDoc->addField($fileUploadId);
$index->addDocument($indexDoc);
}
$index->commit();
$response = $this->getResponse();
$response->setContent("Index Ok");
return $response;
}
/**
* {@inheritdoc}
*/
public function addProduct(ProductInterface $product, $indexName = ProductIndexerInterface::DEFAULT_INDEX_NAME)
{
$index = $this->searchIndexManager->getIndex(ProductIndexerInterface::DEFAULT_INDEX_NAME);
$document = new Document();
$document->addField(Field::unIndexed('identifier', $product->getId()));
$document->addField(Field::text('name', $product->translate('en')->getName()));
$document->addField(Field::text('shortDescription', $product->translate()->getShortDescription()));
$document->addField(Field::text('description', $product->translate()->getDescription()));
$index->addDocument($document);
$index->commit();
}
/**
* Update document in index for model
*
* @param Model $model
*/
public function update(Model $model)
{
// Remove any existing documents for model.
$this->delete($model);
// Create new document for model.
$doc = new Document();
list($name, $value) = $this->config->primaryKeyPair($model);
// Add private key.
$doc->addField(Field::keyword($name, $value));
// Add model's class UID.
list($name, $value) = $this->config->classUidPair($model);
// Add class uid for identification of model's class.
$doc->addField(Field::Keyword($name, $value));
// Get base fields.
$fields = $this->config->fields($model);
// Add fields to document to be indexed (but not stored).
foreach ($fields as $fieldName => $options) {
$fieldValue = $model->{trim($fieldName)};
$field = Field::unStored(trim($fieldName), strip_tags(trim($fieldValue)));
$field->boost = array_get($options, 'boost');
$doc->addField($field);
}
// Get dynamic fields.
$optionalAttributes = $this->config->optionalAttributes($model);
// Add optional attributes to document to be indexed (but not stored).
foreach ($optionalAttributes as $fieldName => $options) {
$fieldValue = array_get($options, "value");
$field = Field::unStored(trim($fieldName), strip_tags(trim($fieldValue)));
$field->boost = array_get($options, "boost");
$doc->addField($field);
}
// Set boost for model.
$doc->boost = $this->config->boost($model);
// Add document to index.
$this->index()->addDocument($doc);
}
public function addAetCommunicationToSearchIndex($index, Communication $aetCommunication)
{
// Create a new document
$document = new Document();
$document->addField(Field::keyword('dbId', $aetCommunication->getId(), 'utf-8'));
$document->addField(Field::unStored('title', $aetCommunication->getTitle(), 'utf-8'));
$document->addField(Field::unStored('shortdesc', $aetCommunication->getShortDesc(), 'utf-8'));
$document->addField(Field::unStored('body', html_entity_decode(strip_tags($aetCommunication->getBody()), ENT_SUBSTITUTE, 'UTF-8'), 'utf-8'));
$document->addField(Field::unStored('author', $aetCommunication->getUser()->getFirstname() . " " . $aetCommunication->getUser()->getLastname(), 'utf-8'));
// Add your document to the index
$index->addDocument($document);
// Commit your change
$index->commit();
$index->optimize();
}
/**
* Adds a document to this segment.
*
* @param \ZendSearch\Lucene\Document $document
* @throws LuceneException\UnsupportedMethodCallException
*/
public function addDocument(Document $document)
{
$storedFields = array();
$docNorms = array();
$similarity = AbstractSimilarity::getDefault();
foreach ($document->getFieldNames() as $fieldName) {
$field = $document->getField($fieldName);
if ($field->storeTermVector) {
/**
* @todo term vector storing support
*/
throw new LuceneException\UnsupportedMethodCallException('Store term vector functionality is not supported yet.');
}
if ($field->isIndexed) {
if ($field->isTokenized) {
$analyzer = Analyzer\Analyzer::getDefault();
$analyzer->setInput($field->value, $field->encoding);
$position = 0;
$tokenCounter = 0;
while (($token = $analyzer->nextToken()) !== null) {
$tokenCounter++;
$term = new Index\Term($token->getTermText(), $field->name);
$termKey = $term->key();
if (!isset($this->_termDictionary[$termKey])) {
// New term
$this->_termDictionary[$termKey] = $term;
$this->_termDocs[$termKey] = array();
$this->_termDocs[$termKey][$this->_docCount] = array();
} elseif (!isset($this->_termDocs[$termKey][$this->_docCount])) {
// Existing term, but new term entry
$this->_termDocs[$termKey][$this->_docCount] = array();
}
$position += $token->getPositionIncrement();
$this->_termDocs[$termKey][$this->_docCount][] = $position;
}
if ($tokenCounter == 0) {
// Field contains empty value. Treat it as non-indexed and non-tokenized
$field = clone $field;
$field->isIndexed = $field->isTokenized = false;
} else {
$docNorms[$field->name] = chr($similarity->encodeNorm($similarity->lengthNorm($field->name, $tokenCounter) * $document->boost * $field->boost));
}
} elseif (($fieldUtf8Value = $field->getUtf8Value()) == '') {
// Field contains empty value. Treat it as non-indexed and non-tokenized
$field = clone $field;
$field->isIndexed = $field->isTokenized = false;
} else {
$term = new Index\Term($fieldUtf8Value, $field->name);
$termKey = $term->key();
if (!isset($this->_termDictionary[$termKey])) {
// New term
$this->_termDictionary[$termKey] = $term;
$this->_termDocs[$termKey] = array();
$this->_termDocs[$termKey][$this->_docCount] = array();
} elseif (!isset($this->_termDocs[$termKey][$this->_docCount])) {
// Existing term, but new term entry
$this->_termDocs[$termKey][$this->_docCount] = array();
}
$this->_termDocs[$termKey][$this->_docCount][] = 0;
// position
$docNorms[$field->name] = chr($similarity->encodeNorm($similarity->lengthNorm($field->name, 1) * $document->boost * $field->boost));
}
}
if ($field->isStored) {
$storedFields[] = $field;
}
$this->addField($field);
}
foreach ($this->_fields as $fieldName => $field) {
if (!$field->isIndexed) {
continue;
}
if (!isset($this->_norms[$fieldName])) {
$this->_norms[$fieldName] = str_repeat(chr($similarity->encodeNorm($similarity->lengthNorm($fieldName, 0))), $this->_docCount);
}
if (isset($docNorms[$fieldName])) {
$this->_norms[$fieldName] .= $docNorms[$fieldName];
} else {
$this->_norms[$fieldName] .= chr($similarity->encodeNorm($similarity->lengthNorm($fieldName, 0)));
}
}
$this->addStoredFields($storedFields);
}
public function generateIndexAction()
{
$searchIndexLocation = $this->getIndexLocation();
$index = Lucene\Lucene::create($searchIndexLocation);
$userTable = $this->getServiceLocator()->get('UserTable');
$uploadTable = $this->getServiceLocator()->get('UploadTable');
$allUploads = $uploadTable->fetchAll();
foreach ($allUploads as $fileUpload) {
//
$uploadOwner = $userTable->getUser($fileUpload->user_id);
// id field
$fileUploadId = Document\Field::unIndexed('upload_id', $fileUpload->id);
// label field
$label = Document\Field::Text('label', $fileUpload->label);
// owner field
$owner = Document\Field::Text('owner', $uploadOwner->name);
if (substr_compare($fileUpload->filename, ".xlsx", strlen($fileUpload->filename) - strlen(".xlsx"), strlen(".xlsx")) === 0) {
// index excel sheet
$uploadPath = $this->getFileUploadLocation();
$indexDoc = Lucene\Document\Xlsx::loadXlsxFile($uploadPath . "/" . $fileUpload->filename);
} else {
if (substr_compare($fileUpload->filename, ".docx", strlen($fileUpload->filename) - strlen(".docx"), strlen(".docx")) === 0) {
// index word doc
$uploadPath = $this->getFileUploadLocation();
$indexDoc = Lucene\Document\Docx::loadDocxFile($uploadPath . "/" . $fileUpload->filename);
} else {
$indexDoc = new Lucene\Document();
}
}
$indexDoc->addField($label);
$indexDoc->addField($owner);
$indexDoc->addField($fileUploadId);
$index->addDocument($indexDoc);
}
$index->commit();
}
/**
* Add index
* @param integer $id
* @param SearchCollection $index
*/
public function add($id, SearchCollection $index)
{
if ($this->config()->exists('zend_search', 'index')) {
$document = new Document();
$document->addField(Field::keyword('id', $id));
foreach ($index as $field) {
$document->addField($field);
}
$this->index()->addDocument($document);
$this->index()->commit();
}
}
/**
* @group ZF-9680
*/
public function testIsDeletedWithoutExplicitCommit()
{
$index = Lucene\Lucene::create(__DIR__ . '/_index/_files');
$document = new Document();
$document->addField(Document\Field::Keyword('_id', 'myId'));
$document->addField(Document\Field::Keyword('bla', 'blubb'));
$index->addDocument($document);
$this->assertFalse($index->isDeleted(0));
}
/**
* Обновить индекс для объекта ISearchable
*
* @param ISearchable $item
*/
public function updateIndex(ISearchable $item)
{
// получаем доступ собственно к индексу
$index = $this->connection;
// удаляем старый индекс
$this->deleteIndex($item);
// недоступные не индексируем
if (!$item->isAvailableForIndexing()) {
return;
}
$document = new Document();
// сохраняем первичный ключ модели для идентификации ее в результатах поиска
$document->addField(Field::Keyword('pk', $item->getId()));
// id моделей могут пересекаться (например, у продуктов и услуг),
// поэтому добавим второе поле для однозначной идентификации
$document->addField(Field::Keyword('model', get_class($item)));
// индексируем поля модели
foreach ($item->getAttributesForIndexing() as $attribute) {
$field = $attribute->getFieldName();
$value = $attribute->getValue();
$document->addField(Field::unStored($field, strip_tags($value)));
}
// добавляем запись в индекс
$index->addDocument($document);
$index->commit();
}
public function index($indexer, $commit = true, $optimize = true)
{
$document = new Document();
$document->addField(Field::keyword('pk', $this->getId()));
$document->addField(Field::Text('course', $this->getSubjectcode()));
$document->addField(Field::Text('cross-listed', str_replace(array(';', ',', '|'), ' ', $this->getCrossListed())));
$document->addField(Field::Text('instructor', $this->getLegalContentOwner()));
$document->addField(Field::Unstored('comments', $this->getComments()));
$indexer->addDocument($document);
if ($commit) {
$indexer->commit();
}
if ($optimize) {
$indexer->optimize();
}
}
/**
* @ORM\PostPersist
*/
public function updateLuceneIndex()
{
$index = self::getLuceneIndex();
// remove existing entries
foreach ($index->find('pk:' . $this->getId()) as $hit) {
$index->delete($hit->id);
}
// don't index expired and non-activated jobs
if ($this->isExpired() || !$this->getIsActivated()) {
return;
}
$doc = new Document();
// store job primary key to identify it in the search results
$doc->addField(Document\Field::Keyword('pk', $this->getId()));
// index job fields
$doc->addField(Document\Field::UnStored('position', $this->getPosition(), 'utf-8'));
$doc->addField(Document\Field::UnStored('company', $this->getCompany(), 'utf-8'));
$doc->addField(Document\Field::UnStored('location', $this->getLocation(), 'utf-8'));
$doc->addField(Document\Field::UnStored('description', $this->getDescription(), 'utf-8'));
// add job to the index
$index->addDocument($doc);
$index->commit();
}
public function generateSearchAction()
{
$searchIndexLocation = $this->getIndexLocation();
$index = Lucene\Lucene::create($searchIndexLocation);
$allUsers = $this->getUserTable()->fetchAll(false);
foreach ($allUsers as $user) {
$id = Document\Field::keyword('userId', $user->userId);
$firstName = Document\Field::text('firstName', $user->firstName);
$lastName = Document\Field::text('lastName', $user->lastName);
$email = Document\Field::text('email', $user->email);
$role = Document\Field::text('role', $user->role);
$activated = Document\Field::keyword('activated', $user->activated);
$indexDoc = new Lucene\Document();
$indexDoc->addField($id);
$indexDoc->addField($firstName);
$indexDoc->addField($lastName);
$indexDoc->addField($email);
$indexDoc->addField($role);
$indexDoc->addField($activated);
$index->addDocument($indexDoc);
}
$index->commit();
}
/**
* create document from configured fields within extracted data
* @param string $url
* @param array $page
* @return Document
*/
protected function createDocument($url, $page)
{
$document = new Document();
if (!isset($page['status_code'])) {
$page['status_code'] = 00;
//tmp
}
setlocale(LC_ALL, "cs_CZ.UTF-8");
$document->addField(Field::keyword('url', $url));
// ancestor URLs to search by URL
$urlParts = parse_url($url);
if (isset($urlParts['path']) && $urlParts['path'] && strlen($urlParts['path']) > 1) {
$uri = $urlParts['path'];
$uris = array($uri);
do {
$uri = substr($uri, 0, strrpos($uri, '/'));
$uris[] = $uri;
} while (strrpos($uri, '/') > 1);
$document->addField(Field::text(Page::URIS_KEY, implode(' ', $uris)));
}
foreach (array(Page::TITLE_KEY, Page::DESCRIPTION_KEY, Page::BODY_KEY, Page::IMAGE_KEY) as $fieldName) {
$fieldValue = isset($page[$fieldName]) ? $page[$fieldName] : '';
switch ($fieldName) {
case Page::TITLE_KEY:
case Page::DESCRIPTION_KEY:
case Page::BODY_KEY:
$field = Field::text($fieldName, $fieldValue);
// translit
$fieldTranslit = Field::text($fieldName . '_translit', str_replace("'", '', iconv('UTF-8', 'ASCII//TRANSLIT//IGNORE', $fieldValue)));
$fieldTranslit->boost = isset($this->parameters[self::BOOST_PARAM][$fieldName]) ? $this->parameters[self::BOOST_PARAM][$fieldName] : 1.25;
$document->addField($fieldTranslit);
break;
case Page::IMAGE_KEY:
$field = Field::unIndexed($fieldName, $fieldValue);
break;
default:
$translitValue = str_replace("'", '', iconv('UTF-8', 'ASCII//TRANSLIT//IGNORE', $fieldValue));
$field = Field::text($fieldName, $fieldValue . ($translitValue != $fieldValue ? ' ' . $translitValue : ''));
}
$field->boost = isset($this->parameters[self::BOOST_PARAM][$fieldName]) ? $this->parameters[self::BOOST_PARAM][$fieldName] : 1.25;
$document->addField($field);
}
// title tags as configured i.e. h1, h2, ...
foreach ($this->parameters[self::TITLE_TAGS_PARAM] as $fieldName) {
$fieldValue = Page::hasHeadlineType($page, $fieldName) ? Page::getHeadline($page, $fieldName) : '';
$field = Field::text($fieldName, $fieldValue);
$field->boost = isset($this->parameters[self::BOOST_PARAM][$fieldName]) ? $this->parameters[self::BOOST_PARAM][$fieldName] : 1;
$document->addField($field);
$fieldTranslit = Field::text($fieldName . '_translit', str_replace("'", '', iconv('UTF-8', 'ASCII//TRANSLIT//IGNORE', $fieldValue)));
$fieldTranslit->boost = isset($this->parameters[self::BOOST_PARAM][$fieldName]) ? $this->parameters[self::BOOST_PARAM][$fieldName] : 1.25;
$document->addField($fieldTranslit);
}
// page ID if selector defined
if ($this->parameters[self::PAGE_ID_PARAM]) {
$fieldValue = isset($page[Page::PAGE_ID_KEY]) ? $page[Page::PAGE_ID_KEY] : '';
$field = Field::unIndexed(Page::PAGE_ID_KEY, $fieldValue);
$document->addField($field);
}
// route name if selector defined
if ($this->parameters[self::ROUTE_NAME_PARAM]) {
$fieldValue = isset($page[Page::ROUTE_NAME_KEY]) ? $page[Page::ROUTE_NAME_KEY] : '';
$field = Field::unIndexed(Page::ROUTE_NAME_KEY, $fieldValue);
$document->addField($field);
}
return $document;
}
protected function indexProperty(Document $document, \core_kernel_classes_Property $property)
{
$indexes = $property->getPropertyValues(new \core_kernel_classes_Property('http://www.tao.lu/Ontologies/TAO.rdf#PropertyIndex'));
foreach ($indexes as $indexUri) {
$index = new Index($indexUri);
$id = $index->getIdentifier();
$strings = $index->tokenize($this->resource->getPropertyValues($property));
if (!empty($strings)) {
if ($index->isFuzzyMatching()) {
// cannot store multiple fuzzy strings
$string = implode(' ', $strings);
$field = Document\Field::Text($index->getIdentifier(), $string);
$field->isStored = $index->isStored();
$document->addField($field);
} else {
$value = count($strings) > 1 ? $strings : reset($strings);
$field = Document\Field::Keyword($index->getIdentifier(), $value);
$field->isStored = $index->isStored() && !is_array($value);
// storage of arrays not supported
$document->addField($field);
}
}
}
}
/**
* Update document in index for model
*
* @param Model $model
*/
public function update(Model $model)
{
// Remove any existing documents for model.
$this->delete($model);
// Create new document for model.
$doc = new Document();
list($name, $value) = $this->config->privateKeyPair($model);
// Add private key.
$doc->addField(Field::keyword($name, $value));
// Add model's class UID.
list($name, $value) = $this->config->classUidPair($model);
// Add class uid for identification of model's class.
$doc->addField(Field::Keyword($name, $value));
$fields = $this->config->fields($model);
// Add fields to document to be indexed (but not stored).
foreach ($fields as $field) {
$doc->addField(Field::unStored(trim($field), strip_tags(trim($model->{trim($field)}))));
}
$optionalAttributes = $this->config->optionalAttributes($model);
// Add optional attributes to document to be indexed (but not stored).
foreach ($optionalAttributes as $fieldName => $fieldValue) {
$doc->addField(Field::unStored(trim($fieldName), strip_tags(trim($fieldValue))));
}
// Add document to index.
$this->index()->addDocument($doc);
}
public function processAction()
{
$userEmail = $this->getAuthService()->getStorage()->read();
if (!$userEmail) {
$this->flashMessenger()->addErrorMessage("not authorized");
return $this->getResponse()->setContent("not authorized");
}
$request = $this->getRequest();
$form = new UploadForm();
$uploadFile = $this->params()->fromFiles('fileupload');
if ($request->isPost()) {
$form->setData($request->getPost());
if ($form->isValid()) {
// Получение конфигурации из конфигурационных данных модуля
$uploadPath = $this->getFileUploadLocation();
// Сохранение выгруженного файла
$adapter = new \Zend\File\Transfer\Adapter\Http();
$adapter->setDestination($uploadPath);
if ($adapter->receive($uploadFile['name'])) {
$userTable = $this->getServiceLocator()->get('UserTable');
$user = $userTable->getUserByEmail($userEmail);
$upload = new \Users\Model\Upload();
// Успешная выгрузка файла
$exchange_data = array();
$exchange_data['label'] = $request->getPost()->get('label');
$exchange_data['filename'] = $uploadFile['name'];
$exchange_data['user_id'] = $user->getId();
$upload->exchangeArray($exchange_data);
$uploadTable = $this->getServiceLocator()->get('UploadTable');
$uploadTable->save($upload);
$upload->setId($uploadTable->getLastInsertValue());
//добавить в Lucene
$searchIndexLocation = $this->getIndexLocation();
$index = Lucene\Lucene::create($searchIndexLocation);
// создание полей lucene
$fileUploadId = Document\Field::unIndexed('upload_id', $upload->getId());
$label = Document\Field::Text('label', $upload->getLabel());
$owner = Document\Field::Text('owner', $user->getName());
$uploadPath = $this->getFileUploadLocation();
$fileName = $upload->getFilename();
$filePath = $uploadPath . DIRECTORY_SEPARATOR . $fileName;
if (substr_compare($fileName, ".xlsx", strlen($fileName) - strlen(".xlsx"), strlen(".xlsx")) === 0) {
// Индексирование таблицы excel
$indexDoc = Lucene\Document\Xlsx::loadXlsxFile($filePath);
} else {
if (substr_compare($fileName, ".docx", strlen($fileName) - strlen(".docx"), strlen(".docx")) === 0) {
// Индексирование документа Word
$indexDoc = Lucene\Document\Docx::loadDocxFile($filePath);
} else {
$indexDoc = new Lucene\Document();
}
}
// создание нового документа и добавление всех полей
$indexDoc = new Lucene\Document();
$indexDoc->addField($label);
$indexDoc->addField($owner);
$indexDoc->addField($fileUploadId);
$index->addDocument($indexDoc);
$index->commit();
}
}
}
return $this->redirect()->toRoute('uploads', array('action' => 'index'));
}
protected function createDocument($title, $body, $url)
{
$document = new Document();
$document->addField(Field::text('title', $title));
$document->addField(Field::text('body', strip_tags($body)));
$document->addField(Field::keyword('url', $url));
return $document;
}
/**
* {@inheritDoc}
*/
public function index($workspace, $path, Node $node)
{
$index = $this->getIndex($workspace);
$document = new Document();
$nodeName = PathHelper::getNodeName($path);
$localNodeName = $nodeName;
// PathHelper::getLocalNodeName($path);
$parentPath = PathHelper::getParentPath($path);
$document->addField(Field::Keyword(self::IDX_PATH, $path));
$document->addField(Field::Keyword(self::IDX_NODENAME, $nodeName));
$document->addField(Field::Keyword(self::IDX_NODELOCALNAME, $localNodeName));
$document->addField(Field::Keyword(self::IDX_PARENTPATH, $parentPath));
foreach ($node->getProperties() as $propertyName => $property) {
$propertyValue = $property['value'];
$propertyType = $property['type'];
if ($propertyName === Storage::INTERNAL_UUID) {
$document->addField(Field::Keyword(Storage::INTERNAL_UUID, $propertyValue));
continue;
}
switch ($propertyType) {
case PropertyType::TYPENAME_STRING:
case PropertyType::TYPENAME_NAME:
case PropertyType::TYPENAME_PATH:
case PropertyType::TYPENAME_URI:
$value = (array) $propertyValue;
$value = join(self::MULTIVALUE_SEPARATOR, $value);
$document->addField(Field::Text($propertyName, $value));
break;
case PropertyType::TYPENAME_DATE:
$values = (array) $propertyValue;
foreach ($values as $i => $value) {
if ($value instanceof \DateTime) {
$values[$i] = $value->format('c');
}
}
$value = join(self::MULTIVALUE_SEPARATOR, $values);
$document->addField(Field::Text($propertyName, $value));
break;
case PropertyType::TYPENAME_DECIMAL:
case PropertyType::TYPENAME_LONG:
case PropertyType::TYPENAME_DOUBLE:
$values = (array) $propertyValue;
foreach ($values as &$value) {
$value = sprintf('%0' . strlen(PHP_INT_MAX) . 's', $value);
}
$value = join(self::MULTIVALUE_SEPARATOR, $values);
$document->addField(Field::Text($propertyName, $value));
break;
case PropertyType::TYPENAME_BOOLEAN:
$values = (array) $propertyValue;
foreach ($values as &$value) {
if ($propertyValue === 'false') {
$value = self::VALUE_BOOLEAN_FALSE;
} else {
$value = 1;
}
}
$value = join(self::MULTIVALUE_SEPARATOR, $values);
$document->addField(Field::Text($propertyName, $value));
break;
}
}
$index->addDocument($document);
}
/**
* Indexa dados nos arquivos de json
*/
public function index()
{
$dir = realpath(dirname(__FILE__)) . DIRECTORY_SEPARATOR . "data" . DIRECTORY_SEPARATOR;
$jsonDir = $dir . "json";
$indexDir = $dir . "index";
// ler aquivos json
$files = scandir($jsonDir);
foreach ($files as $file) {
if ($file == '.' || $file == '..') {
continue;
}
// Se arquivo existe
if (is_file($jsonDir . DIRECTORY_SEPARATOR . $file)) {
$json = json_decode(file_get_contents($jsonDir . DIRECTORY_SEPARATOR . $file));
$indexName = substr($file, 0, -5);
// Cria index
$index = Lucene\Lucene::create($indexDir . DIRECTORY_SEPARATOR . $indexName);
// Cria documento e define campos para indexar
foreach ($json as $entry) {
$doc = new Lucene\Document();
$doc->addField(Lucene\Document\Field::Text('url', $entry->title));
$doc->addField(Lucene\Document\Field::UnStored('contents', $entry->text));
$index->addDocument($doc);
}
}
}
}
