public function testAddFieldMethodChaining()
{
$document = new Document();
$this->assertTrue($document->addField(Document\Field::Text('title', 'Title')) instanceof Document);
$document = new Document();
$document->addField(Document\Field::Text('title', 'Title'))->addField(Document\Field::Text('annotation', 'Annotation'))->addField(Document\Field::Text('body', 'Document body, document body, document body...'));
}
/**
* @dataProvider searchResultDataProvider
*/
function testSearchLuceneResultContent($fileId, $name, $path, $size, $score, $mimeType, $modified, $container)
{
require_once __DIR__ . '/util/dummyindex.php';
$index = new DummyIndex();
$doc = new Document();
$doc->addField(Document\Field::Keyword('fileId', $fileId));
$doc->addField(Document\Field::Text('path', '/test/files' . $path, 'UTF-8'));
$doc->addField(Document\Field::unIndexed('mtime', $modified));
$doc->addField(Document\Field::unIndexed('size', $size));
$doc->addField(Document\Field::unIndexed('mimetype', $mimeType));
$index->addDocument($doc);
$hit = new QueryHit($index);
$hit->score = $score;
$hit->id = 0;
$hit->document_id = 0;
$searchResult = new \OCA\Search_Lucene\Search\LuceneResult($hit);
$this->assertInstanceOf('OCA\\Search_Lucene\\Search\\LuceneResult', $searchResult);
$this->assertEquals($fileId, $searchResult->id);
$this->assertEquals('lucene', $searchResult->type);
$this->assertEquals($path, $searchResult->path);
$this->assertEquals($name, $searchResult->name);
$this->assertEquals($mimeType, $searchResult->mime_type);
$this->assertEquals($size, $searchResult->size);
$this->assertEquals($score, $searchResult->score);
$this->assertEquals($modified, $searchResult->modified);
}
/**
* Object constructor
*
* @param string $data
* @param boolean $storeContent
* @throws NotIndexedException
*/
private function __construct($data, $storeContent)
{
//TODO check PDF >1.5 metadata extraction
//do the content extraction
$parser = new Parser();
try {
$pdf = $parser->parseContent($data);
$body = $pdf->getText();
// Store contents
if ($storeContent) {
$this->addField(Document\Field::Text('body', $body, 'UTF-8'));
} else {
$this->addField(Document\Field::UnStored('body', $body, 'UTF-8'));
}
$details = $pdf->getDetails();
// Store meta data properties
foreach ($details as $key => $value) {
$key = strtolower($key);
if ($key === 'author') {
$key = 'creator';
}
$this->addField(Document\Field::Text($key, $value, 'UTF-8'));
}
} catch (\Exception $ex) {
throw new NotIndexedException(null, null, $ex);
}
}
public function add(Searchable $obj)
{
// Get Primary Key
$attributes = $obj->getSearchAttributes();
$index = $this->getIndex();
$doc = new \ZendSearch\Lucene\Document();
// Add Meta Data fields
foreach ($this->getMetaInfoArray($obj) as $fieldName => $fieldValue) {
$doc->addField(\ZendSearch\Lucene\Document\Field::keyword($fieldName, $fieldValue));
}
// Add provided search infos
foreach ($attributes as $key => $val) {
$doc->addField(\ZendSearch\Lucene\Document\Field::Text($key, $val, 'UTF-8'));
}
// Add comments - if record is content
if ($obj instanceof ContentActiveRecord) {
$comments = "";
foreach (Comment::findAll(['object_id' => $obj->getPrimaryKey(), 'object_model' => $obj->className()]) as $comment) {
$comments .= " " . $comment->message;
}
$doc->addField(\ZendSearch\Lucene\Document\Field::Text('comments', $comments, 'UTF-8'));
}
if (\Yii::$app->request->isConsoleRequest) {
print ".";
}
$index->addDocument($doc);
$index->commit();
}
/**
* Object constructor
*
* @param string $fileName
* @param boolean $storeContent
* @throws ExtensionNotLoadedException
* @throws RuntimeException
*/
private function __construct($fileName, $storeContent)
{
if (!class_exists('ZipArchive', false)) {
throw new ExtensionNotLoadedException('Open Document Text processing functionality requires Zip extension to be loaded');
}
// Document data holders
$documentHeadlines = array();
$documentParagraphs = array();
// Open OpenXML package
$package = new \ZipArchive();
$package->open($fileName);
// Read relations and search for officeDocument
$content = $package->getFromName('content.xml');
if ($content === false) {
throw new RuntimeException('Invalid archive or corrupted .odt file.');
}
// Prevent php from loading remote resources
$loadEntities = libxml_disable_entity_loader(true);
$sxe = simplexml_load_string($content, 'SimpleXMLElement', LIBXML_NOBLANKS | LIBXML_COMPACT);
// Restore entity loader state
libxml_disable_entity_loader($loadEntities);
foreach ($sxe->xpath('//text:h') as $headline) {
$h = strip_tags($headline->asXML());
$documentHeadlines[] = $h;
}
foreach ($sxe->xpath('//text:p') as $paragraph) {
$p = strip_tags($paragraph->asXML());
$documentParagraphs[] = $p;
}
// Read core properties
$coreProperties = $this->extractMetaData($package);
// Close file
$package->close();
// Store contents
if ($storeContent) {
$this->addField(Field::Text('headlines', implode(' ', $documentHeadlines), 'UTF-8'));
$this->addField(Field::Text('body', implode('', $documentParagraphs), 'UTF-8'));
} else {
$this->addField(Field::UnStored('headlines', implode(' ', $documentHeadlines), 'UTF-8'));
$this->addField(Field::UnStored('body', implode('', $documentParagraphs), 'UTF-8'));
}
// Store meta data properties
foreach ($coreProperties as $key => $value) {
$this->addField(Field::Text($key, $value, 'UTF-8'));
}
// Store title (if not present in meta data)
if (!isset($coreProperties['title'])) {
$this->addField(Field::Text('title', $fileName, 'UTF-8'));
}
}
/**
* @param $data
* @param SearchIndexInterface $index
*
* @return IndexInterface
*/
public function index($data, SearchIndexInterface $index)
{
$this->unindex($data, $index);
$indexDoc = new Document();
$indexDoc->addField(Field::Keyword('group_id', $data->id));
$indexDoc->addField(Field::UnIndexed('type', "group"));
$indexDoc->addField(Field::UnIndexed('identifier', $data->url));
$indexDoc->addField(Field::UnIndexed('date_time', date('c')));
$indexDoc->addField(Field::UnIndexed('date', date('j. M. Y')));
$indexDoc->addField(Field::Text('title', $data->name_short, 'utf-8'));
$indexDoc->addField(Field::Text('body', $data->description, 'utf-8'));
$index->addDocument($indexDoc);
return $this;
}
/**
* @param $data
* @param SearchIndexInterface $index
*
* @return IndexInterface
*/
public function index($data, SearchIndexInterface $index)
{
$this->unindex($data, $index);
$indexDoc = new Document();
$indexDoc->addField(Field::Keyword('news_id', $data->id));
$indexDoc->addField(Field::UnIndexed('type', "news"));
$indexDoc->addField(Field::UnIndexed('identifier', $data->id));
$indexDoc->addField(Field::UnIndexed('date_time', $data->created_date->format('c')));
$indexDoc->addField(Field::UnIndexed('date', $data->created_date->format('j. M. Y')));
$indexDoc->addField(Field::Text('title', $data->title, 'utf-8'));
$indexDoc->addField(Field::Text('body', $data->body, 'utf-8'));
$index->addDocument($indexDoc);
return $this;
}
function testUpdate()
{
// preparation
$app = new Application();
$container = $app->getContainer();
// get an index
/** @var Index $index */
$index = $container->query('Index');
// add a document
$doc = new Document();
$doc->addField(Document\Field::Keyword('fileId', '1'));
$doc->addField(Document\Field::Text('path', '/somewhere/deep/down/the/rabbit/hole', 'UTF-8'));
$doc->addField(Document\Field::Text('users', 'alice', 'UTF-8'));
$index->index->addDocument($doc);
$index->commit();
// search for it
$idTerm = new Term('1', 'fileId');
$idQuery = new Query\Term($idTerm);
$query = new Query\Boolean();
$query->addSubquery($idQuery);
/** @var QueryHit $hit */
$hits = $index->find($query);
// get the document from the query hit
$foundDoc = $hits[0]->getDocument();
$this->assertEquals('alice', $foundDoc->getFieldValue('users'));
// delete the document from the index
//$index->index->delete($hit);
// change the 'users' key of the document
$foundDoc->addField(Document\Field::Text('users', 'bob', 'UTF-8'));
$this->assertEquals('bob', $foundDoc->getFieldValue('users'));
// add the document back to the index
$index->updateFile($foundDoc, '1');
$idTerm2 = new Term('1', 'fileId');
$idQuery2 = new Query\Term($idTerm2);
$query2 = new Query\Boolean();
$query2->addSubquery($idQuery2);
/** @var QueryHit $hit */
$hits2 = $index->find($query2);
// get the document from the query hit
$foundDoc2 = $hits2[0]->getDocument();
$this->assertEquals('bob', $foundDoc2->getFieldValue('users'));
}
public function generateIndexAction()
{
$searchIndexLocation = $this->getIndexLocation();
$index = Lucene\Lucene::create($searchIndexLocation);
$userTable = $this->getServiceLocator()->get('UserTable');
$uploadTable = $this->getServiceLocator()->get('UploadTable');
$allUploads = $uploadTable->fetchAll();
foreach ($allUploads as $fileUpload) {
$uploadOwner = $userTable->getById($fileUpload->getUserId());
// создание полей lucene
$fileUploadId = Document\Field::unIndexed('upload_id', $fileUpload->getId());
$label = Document\Field::Text('label', $fileUpload->getLabel());
$owner = Document\Field::Text('owner', $uploadOwner->getName());
$uploadPath = $this->getFileUploadLocation();
$fileName = $fileUpload->getFilename();
$filePath = $uploadPath . DIRECTORY_SEPARATOR . $fileName;
if (substr_compare($fileName, ".xlsx", strlen($fileName) - strlen(".xlsx"), strlen(".xlsx")) === 0) {
// Индексирование таблицы excel
$indexDoc = Lucene\Document\Xlsx::loadXlsxFile($filePath);
} else {
if (substr_compare($fileName, ".docx", strlen($fileName) - strlen(".docx"), strlen(".docx")) === 0) {
// Индексирование документа Word
$indexDoc = Lucene\Document\Docx::loadDocxFile($filePath);
} else {
$indexDoc = new Lucene\Document();
}
}
// создание нового документа и добавление всех полей
$indexDoc = new Lucene\Document();
$indexDoc->addField($label);
$indexDoc->addField($owner);
$indexDoc->addField($fileUploadId);
$index->addDocument($indexDoc);
}
$index->commit();
$response = $this->getResponse();
$response->setContent("Index Ok");
return $response;
}
/**
* インデックスファイルを生成
*/
public static function updateIndex()
{
if (empty(self::$igo)) {
self::$igo = new Tagger(array('dict_dir' => LIB_DIR . 'ipadic', 'reduce_mode' => true));
}
Analyzer::setDefault(new Utf8());
// 索引の作成
$index = Lucene::create(CACHE_DIR . self::INDEX_NAME);
foreach (Listing::pages() as $page) {
if (empty($page)) {
continue;
}
$wiki = Factory::Wiki($page);
// 読む権限がない場合スキップ
if (!$wiki->isReadable() || $wiki->isHidden()) {
continue;
}
/*
// HTML出力
$html[] = '<html><head>';
$html[] = '<meta http-equiv="Content-type" content="text/html; charset=UTF-8"/>';
$html[] = '<title>' . $wiki->title() . '</title>';
$html[] = '</head>';
$html[] = '<body>' . $wiki->render() . '</body>';
$html[] = '</html>';
*/
$doc = new LuceneDoc();
$doc->addField(Field::Text('title', $wiki->title()));
// Store document URL to identify it in the search results
$doc->addField(Field::Text('url', $wiki->uri()));
// Index document contents
//$contents = join(" ", self::$igo->wakati(strip_tags($wiki->render())));
$contents = strip_tags($wiki->render());
$doc->addField(Field::UnStored('contents', $contents));
// 索引へ文書の登録
$index->addDocument($doc);
}
$index->optimize();
}
/**
* Object constructor
*
* @param string $fileName
* @param boolean $storeContent
* @throws \ZendSearch\Lucene\Exception\ExtensionNotLoadedException
* @throws \ZendSearch\Lucene\Exception\RuntimeException
*/
private function __construct($fileName, $storeContent)
{
if (!class_exists('ZipArchive', false)) {
throw new ExtensionNotLoadedException('MS Office documents processing functionality requires Zip extension to be loaded');
}
// Document data holders
$slides = array();
$slideNotes = array();
$documentBody = array();
$coreProperties = array();
// Open AbstractOpenXML package
$package = new \ZipArchive();
$package->open($fileName);
// Read relations and search for officeDocument
$relationsXml = $package->getFromName('_rels/.rels');
if ($relationsXml === false) {
throw new RuntimeException('Invalid archive or corrupted .pptx file.');
}
$relations = simplexml_load_string($relationsXml);
foreach ($relations->Relationship as $rel) {
if ($rel["Type"] == AbstractOpenXML::SCHEMA_OFFICEDOCUMENT) {
// Found office document! Search for slides...
$slideRelations = simplexml_load_string($package->getFromName($this->absoluteZipPath(dirname($rel["Target"]) . "/_rels/" . basename($rel["Target"]) . ".rels")));
foreach ($slideRelations->Relationship as $slideRel) {
if ($slideRel["Type"] == self::SCHEMA_SLIDERELATION) {
// Found slide!
$slides[str_replace('rId', '', (string) $slideRel["Id"])] = simplexml_load_string($package->getFromName($this->absoluteZipPath(dirname($rel["Target"]) . "/" . dirname($slideRel["Target"]) . "/" . basename($slideRel["Target"]))));
// Search for slide notes
$slideNotesRelations = simplexml_load_string($package->getFromName($this->absoluteZipPath(dirname($rel["Target"]) . "/" . dirname($slideRel["Target"]) . "/_rels/" . basename($slideRel["Target"]) . ".rels")));
foreach ($slideNotesRelations->Relationship as $slideNoteRel) {
if ($slideNoteRel["Type"] == self::SCHEMA_SLIDENOTESRELATION) {
// Found slide notes!
$slideNotes[str_replace('rId', '', (string) $slideRel["Id"])] = simplexml_load_string($package->getFromName($this->absoluteZipPath(dirname($rel["Target"]) . "/" . dirname($slideRel["Target"]) . "/" . dirname($slideNoteRel["Target"]) . "/" . basename($slideNoteRel["Target"]))));
break;
}
}
}
}
break;
}
}
// Sort slides
ksort($slides);
ksort($slideNotes);
// Extract contents from slides
foreach ($slides as $slideKey => $slide) {
// Register namespaces
$slide->registerXPathNamespace("p", self::SCHEMA_PRESENTATIONML);
$slide->registerXPathNamespace("a", self::SCHEMA_DRAWINGML);
// Fetch all text
$textElements = $slide->xpath('//a:t');
foreach ($textElements as $textElement) {
$documentBody[] = (string) $textElement;
}
// Extract contents from slide notes
if (isset($slideNotes[$slideKey])) {
// Fetch slide note
$slideNote = $slideNotes[$slideKey];
// Register namespaces
$slideNote->registerXPathNamespace("p", self::SCHEMA_PRESENTATIONML);
$slideNote->registerXPathNamespace("a", self::SCHEMA_DRAWINGML);
// Fetch all text
$textElements = $slideNote->xpath('//a:t');
foreach ($textElements as $textElement) {
$documentBody[] = (string) $textElement;
}
}
}
// Read core properties
$coreProperties = $this->extractMetaData($package);
// Close file
$package->close();
// Store filename
$this->addField(Field::Text('filename', $fileName, 'UTF-8'));
// Store contents
if ($storeContent) {
$this->addField(Field::Text('body', implode(' ', $documentBody), 'UTF-8'));
} else {
$this->addField(Field::UnStored('body', implode(' ', $documentBody), 'UTF-8'));
}
// Store meta data properties
foreach ($coreProperties as $key => $value) {
$this->addField(Field::Text($key, $value, 'UTF-8'));
}
// Store title (if not present in meta data)
if (!isset($coreProperties['title'])) {
$this->addField(Field::Text('title', $fileName, 'UTF-8'));
}
}
/**
* Object constructor
*
* @param string $fileName
* @param boolean $storeContent
* @throws \ZendSearch\Lucene\Exception\ExtensionNotLoadedException
* @throws \ZendSearch\Lucene\Exception\RuntimeException
*/
private function __construct($fileName, $storeContent)
{
if (!class_exists('ZipArchive', false)) {
throw new ExtensionNotLoadedException('MS Office documents processing functionality requires Zip extension to be loaded');
}
// Document data holders
$sharedStrings = array();
$worksheets = array();
$documentBody = array();
$coreProperties = array();
// Open AbstractOpenXML package
$package = new \ZipArchive();
$package->open($fileName);
// Read relations and search for officeDocument
$relationsXml = $package->getFromName('_rels/.rels');
if ($relationsXml === false) {
throw new RuntimeException('Invalid archive or corrupted .xlsx file.');
}
$relations = XmlSecurity::scan($relationsXml);
foreach ($relations->Relationship as $rel) {
if ($rel["Type"] == AbstractOpenXML::SCHEMA_OFFICEDOCUMENT) {
// Found office document! Read relations for workbook...
$workbookRelations = XmlSecurity::scan($package->getFromName($this->absoluteZipPath(dirname($rel["Target"]) . "/_rels/" . basename($rel["Target"]) . ".rels")));
$workbookRelations->registerXPathNamespace("rel", AbstractOpenXML::SCHEMA_RELATIONSHIP);
// Read shared strings
$sharedStringsPath = $workbookRelations->xpath("rel:Relationship[@Type='" . self::SCHEMA_SHAREDSTRINGS . "']");
$sharedStringsPath = (string) $sharedStringsPath[0]['Target'];
$xmlStrings = XmlSecurity::scan($package->getFromName($this->absoluteZipPath(dirname($rel["Target"]) . "/" . $sharedStringsPath)));
if (isset($xmlStrings) && isset($xmlStrings->si)) {
foreach ($xmlStrings->si as $val) {
if (isset($val->t)) {
$sharedStrings[] = (string) $val->t;
} elseif (isset($val->r)) {
$sharedStrings[] = $this->_parseRichText($val);
}
}
}
// Loop relations for workbook and extract worksheets...
foreach ($workbookRelations->Relationship as $workbookRelation) {
if ($workbookRelation["Type"] == self::SCHEMA_WORKSHEETRELATION) {
$worksheets[str_replace('rId', '', (string) $workbookRelation["Id"])] = XmlSecurity::scan($package->getFromName($this->absoluteZipPath(dirname($rel["Target"]) . "/" . dirname($workbookRelation["Target"]) . "/" . basename($workbookRelation["Target"]))));
}
}
break;
}
}
// Sort worksheets
ksort($worksheets);
// Extract contents from worksheets
foreach ($worksheets as $sheetKey => $worksheet) {
foreach ($worksheet->sheetData->row as $row) {
foreach ($row->c as $c) {
// Determine data type
$dataType = (string) $c["t"];
switch ($dataType) {
case "s":
// Value is a shared string
if ((string) $c->v != '') {
$value = $sharedStrings[intval($c->v)];
} else {
$value = '';
}
break;
case "b":
// Value is boolean
$value = (string) $c->v;
if ($value == '0') {
$value = false;
} elseif ($value == '1') {
$value = true;
} else {
$value = (bool) $c->v;
}
break;
case "inlineStr":
// Value is rich text inline
$value = $this->_parseRichText($c->is);
break;
case "e":
// Value is an error message
if ((string) $c->v != '') {
$value = (string) $c->v;
} else {
$value = '';
}
break;
default:
// Value is a string
$value = (string) $c->v;
// Check for numeric values
if (is_numeric($value) && $dataType != 's') {
if ($value == (int) $value) {
$value = (int) $value;
} elseif ($value == (double) $value) {
$value = (double) $value;
} elseif ($value == (double) $value) {
$value = (double) $value;
}
}
}
$documentBody[] = $value;
}
}
}
// Read core properties
$coreProperties = $this->extractMetaData($package);
// Close file
$package->close();
// Store filename
$this->addField(Field::Text('filename', $fileName, 'UTF-8'));
// Store contents
if ($storeContent) {
$this->addField(Field::Text('body', implode(' ', $documentBody), 'UTF-8'));
} else {
$this->addField(Field::UnStored('body', implode(' ', $documentBody), 'UTF-8'));
}
// Store meta data properties
foreach ($coreProperties as $key => $value) {
$this->addField(Field::Text($key, $value, 'UTF-8'));
}
// Store title (if not present in meta data)
if (!isset($coreProperties['title'])) {
$this->addField(Field::Text('title', $fileName, 'UTF-8'));
}
}
/**
* Object constructor
*
* @param string $fileName
* @param boolean $storeContent
* @throws \ZendSearch\Lucene\Exception\ExtensionNotLoadedException
* @throws \ZendSearch\Lucene\Exception\RuntimeException
*/
private function __construct($fileName, $storeContent)
{
if (!class_exists('ZipArchive', false)) {
throw new ExtensionNotLoadedException('MS Office documents processing functionality requires Zip extension to be loaded');
}
// Document data holders
$documentBody = array();
$coreProperties = array();
// Open AbstractOpenXML package
$package = new \ZipArchive();
$package->open($fileName);
// Read relations and search for officeDocument
$relationsXml = $package->getFromName('_rels/.rels');
if ($relationsXml === false) {
throw new RuntimeException('Invalid archive or corrupted .docx file.');
}
$relations = XMLSecurity::scan($relationsXml);
foreach ($relations->Relationship as $rel) {
if ($rel["Type"] == AbstractOpenXML::SCHEMA_OFFICEDOCUMENT) {
// Found office document! Read in contents...
$contents = XMLSecurity::scan($package->getFromName($this->absoluteZipPath(dirname($rel['Target']) . '/' . basename($rel['Target']))));
$contents->registerXPathNamespace('w', self::SCHEMA_WORDPROCESSINGML);
$paragraphs = $contents->xpath('//w:body/w:p');
foreach ($paragraphs as $paragraph) {
$runs = $paragraph->xpath('.//w:r/*[name() = "w:t" or name() = "w:br"]');
if ($runs === false) {
// Paragraph doesn't contain any text or breaks
continue;
}
foreach ($runs as $run) {
if ($run->getName() == 'br') {
// Break element
$documentBody[] = ' ';
} else {
$documentBody[] = (string) $run;
}
}
// Add space after each paragraph. So they are not bound together.
$documentBody[] = ' ';
}
break;
}
}
// Read core properties
$coreProperties = $this->extractMetaData($package);
// Close file
$package->close();
// Store filename
$this->addField(Field::Text('filename', $fileName, 'UTF-8'));
// Store contents
if ($storeContent) {
$this->addField(Field::Text('body', implode('', $documentBody), 'UTF-8'));
} else {
$this->addField(Field::UnStored('body', implode('', $documentBody), 'UTF-8'));
}
// Store meta data properties
foreach ($coreProperties as $key => $value) {
$this->addField(Field::Text($key, $value, 'UTF-8'));
}
// Store title (if not present in meta data)
if (!isset($coreProperties['title'])) {
$this->addField(Field::Text('title', $fileName, 'UTF-8'));
}
}
/**
* index a file
*
* @param File $file the file to be indexed
* @param bool $commit
*
* @return bool true when something was stored in the index, false otherwise (eg, folders are not indexed)
* @throws NotIndexedException when an unsupported file type is encountered
*/
public function indexFile(File $file, $commit = true)
{
// we decide how to index on mime type or file extension
$mimeType = $file->getMimeType();
$fileExtension = strtolower(pathinfo($file->getName(), PATHINFO_EXTENSION));
// initialize plain lucene document
$doc = new Document();
// index content for local files only
$storage = $file->getStorage();
if ($storage->isLocal()) {
$path = $storage->getLocalFile($file->getInternalPath());
//try to use special lucene document types
if ('text/html' === $mimeType) {
//TODO could be indexed, even if not local
$doc = HTML::loadHTML($file->getContent());
} else {
if ('text/' === substr($mimeType, 0, 5) || 'application/x-tex' === $mimeType) {
$body = $file->getContent();
if ($body != '') {
$doc->addField(Document\Field::UnStored('body', $body));
}
} else {
if ('application/pdf' === $mimeType) {
$doc = Pdf::loadPdf($file->getContent());
// the zend classes only understand docx and not doc files
} else {
if ($fileExtension === 'docx') {
$doc = Document\Docx::loadDocxFile($path);
//} else if ('application/msexcel' === $mimeType) {
} else {
if ($fileExtension === 'xlsx') {
$doc = Document\Xlsx::loadXlsxFile($path);
//} else if ('application/mspowerpoint' === $mimeType) {
} else {
if ($fileExtension === 'pptx') {
$doc = Document\Pptx::loadPptxFile($path);
} else {
if ($fileExtension === 'odt') {
$doc = Odt::loadOdtFile($path);
} else {
if ($fileExtension === 'ods') {
$doc = Ods::loadOdsFile($path);
} else {
throw new NotIndexedException();
}
}
}
}
}
}
}
}
}
// Store filecache id as unique id to lookup by when deleting
$doc->addField(Document\Field::Keyword('fileId', $file->getId()));
// Store document path for the search results
$doc->addField(Document\Field::Text('path', $file->getPath(), 'UTF-8'));
$doc->addField(Document\Field::unIndexed('mtime', $file->getMTime()));
$doc->addField(Document\Field::unIndexed('size', $file->getSize()));
$doc->addField(Document\Field::unIndexed('mimetype', $mimeType));
$this->index->updateFile($doc, $file->getId(), $commit);
return true;
}
public function testEncoding()
{
// forcing filter to UTF-8
$utf8text = iconv('UTF-8', 'UTF-8', 'Words with umlauts: åãü...');
$iso8859_1 = iconv('UTF-8', 'ISO-8859-1', $utf8text);
$field = Document\Field::Text('field', $iso8859_1, 'ISO-8859-1');
$this->assertEquals($field->encoding, 'ISO-8859-1');
$this->assertEquals($field->name, 'field');
$this->assertEquals($field->value, $iso8859_1);
$this->assertEquals($field->getUtf8Value(), $utf8text);
}
protected function indexProperty(Document $document, \core_kernel_classes_Property $property)
{
$indexes = $property->getPropertyValues(new \core_kernel_classes_Property('http://www.tao.lu/Ontologies/TAO.rdf#PropertyIndex'));
foreach ($indexes as $indexUri) {
$index = new Index($indexUri);
$id = $index->getIdentifier();
$strings = $index->tokenize($this->resource->getPropertyValues($property));
if (!empty($strings)) {
if ($index->isFuzzyMatching()) {
// cannot store multiple fuzzy strings
$string = implode(' ', $strings);
$field = Document\Field::Text($index->getIdentifier(), $string);
$field->isStored = $index->isStored();
$document->addField($field);
} else {
$value = count($strings) > 1 ? $strings : reset($strings);
$field = Document\Field::Keyword($index->getIdentifier(), $value);
$field->isStored = $index->isStored() && !is_array($value);
// storage of arrays not supported
$document->addField($field);
}
}
}
}
/**
* Object constructor
*
* @param string $data HTML string (may be HTML fragment, )
* @param boolean $isFile
* @param boolean $storeContent
* @param string $defaultEncoding HTML encoding, is used if it's not specified using Content-type HTTP-EQUIV meta tag.
*/
private function __construct($data, $isFile, $storeContent, $defaultEncoding = '')
{
$this->_doc = new \DOMDocument();
$this->_doc->substituteEntities = true;
if ($isFile) {
$htmlData = file_get_contents($data);
} else {
$htmlData = $data;
}
ErrorHandler::start(E_WARNING);
$this->_doc->loadHTML($htmlData);
ErrorHandler::stop();
if ($this->_doc->encoding === null) {
// Document encoding is not recognized
/** @todo improve HTML vs HTML fragment recognition */
if (preg_match('/<html>/i', $htmlData, $matches, PREG_OFFSET_CAPTURE)) {
// It's an HTML document
// Add additional HEAD section and recognize document
$htmlTagOffset = $matches[0][1] + strlen($matches[0][0]);
ErrorHandler::start(E_WARNING);
$this->_doc->loadHTML(iconv($defaultEncoding, 'UTF-8//IGNORE', substr($htmlData, 0, $htmlTagOffset)) . '<head><META HTTP-EQUIV="Content-type" CONTENT="text/html; charset=UTF-8"/></head>' . iconv($defaultEncoding, 'UTF-8//IGNORE', substr($htmlData, $htmlTagOffset)));
ErrorHandler::stop();
// Remove additional HEAD section
$xpath = new \DOMXPath($this->_doc);
$head = $xpath->query('/html/head')->item(0);
$head->parentNode->removeChild($head);
} else {
// It's an HTML fragment
ErrorHandler::start(E_WARNING);
$this->_doc->loadHTML('<html><head><META HTTP-EQUIV="Content-type" CONTENT="text/html; charset=UTF-8"/></head><body>' . iconv($defaultEncoding, 'UTF-8//IGNORE', $htmlData) . '</body></html>');
ErrorHandler::stop();
}
}
/** @todo Add correction of wrong HTML encoding recognition processing
* The case is:
* Content-type HTTP-EQUIV meta tag is presented, but ISO-8859-5 encoding is actually used,
* even $this->_doc->encoding demonstrates another recognized encoding
*/
$xpath = new \DOMXPath($this->_doc);
$docTitle = '';
$titleNodes = $xpath->query('/html/head/title');
foreach ($titleNodes as $titleNode) {
// title should always have only one entry, but we process all nodeset entries
$docTitle .= $titleNode->nodeValue . ' ';
}
$this->addField(Field::Text('title', $docTitle, 'UTF-8'));
$metaNodes = $xpath->query('/html/head/meta[@name]');
foreach ($metaNodes as $metaNode) {
$this->addField(Field::Text($metaNode->getAttribute('name'), $metaNode->getAttribute('content'), 'UTF-8'));
}
$docBody = '';
$bodyNodes = $xpath->query('/html/body');
foreach ($bodyNodes as $bodyNode) {
// body should always have only one entry, but we process all nodeset entries
$this->_retrieveNodeText($bodyNode, $docBody);
}
if ($storeContent) {
$this->addField(Field::Text('body', $docBody, 'UTF-8'));
} else {
$this->addField(Field::UnStored('body', $docBody, 'UTF-8'));
}
$linkNodes = $this->_doc->getElementsByTagName('a');
foreach ($linkNodes as $linkNode) {
if (($href = $linkNode->getAttribute('href')) != '' && (!self::$_excludeNoFollowLinks || strtolower($linkNode->getAttribute('rel')) != 'nofollow')) {
$this->_links[] = $href;
}
}
$linkNodes = $this->_doc->getElementsByTagName('area');
foreach ($linkNodes as $linkNode) {
if (($href = $linkNode->getAttribute('href')) != '' && (!self::$_excludeNoFollowLinks || strtolower($linkNode->getAttribute('rel')) != 'nofollow')) {
$this->_links[] = $href;
}
}
$this->_links = array_unique($this->_links);
$linkNodes = $xpath->query('/html/head/link');
foreach ($linkNodes as $linkNode) {
if (($href = $linkNode->getAttribute('href')) != '') {
$this->_headerLinks[] = $href;
}
}
$this->_headerLinks = array_unique($this->_headerLinks);
}
public function generateIndexAction()
{
$searchIndexLocation = $this->getIndexLocation();
$index = Lucene\Lucene::create($searchIndexLocation);
$userTable = $this->getServiceLocator()->get('UserTable');
$uploadTable = $this->getServiceLocator()->get('UploadTable');
$allUploads = $uploadTable->fetchAll();
foreach ($allUploads as $fileUpload) {
//
$uploadOwner = $userTable->getUser($fileUpload->user_id);
// id field
$fileUploadId = Document\Field::unIndexed('upload_id', $fileUpload->id);
// label field
$label = Document\Field::Text('label', $fileUpload->label);
// owner field
$owner = Document\Field::Text('owner', $uploadOwner->name);
if (substr_compare($fileUpload->filename, ".xlsx", strlen($fileUpload->filename) - strlen(".xlsx"), strlen(".xlsx")) === 0) {
// index excel sheet
$uploadPath = $this->getFileUploadLocation();
$indexDoc = Lucene\Document\Xlsx::loadXlsxFile($uploadPath . "/" . $fileUpload->filename);
} else {
if (substr_compare($fileUpload->filename, ".docx", strlen($fileUpload->filename) - strlen(".docx"), strlen(".docx")) === 0) {
// index word doc
$uploadPath = $this->getFileUploadLocation();
$indexDoc = Lucene\Document\Docx::loadDocxFile($uploadPath . "/" . $fileUpload->filename);
} else {
$indexDoc = new Lucene\Document();
}
}
$indexDoc->addField($label);
$indexDoc->addField($owner);
$indexDoc->addField($fileUploadId);
$index->addDocument($indexDoc);
}
$index->commit();
}
/**
* {@inheritDoc}
*/
public function index($workspace, $path, Node $node)
{
$index = $this->getIndex($workspace);
$document = new Document();
$nodeName = PathHelper::getNodeName($path);
$localNodeName = $nodeName;
// PathHelper::getLocalNodeName($path);
$parentPath = PathHelper::getParentPath($path);
$document->addField(Field::Keyword(self::IDX_PATH, $path));
$document->addField(Field::Keyword(self::IDX_NODENAME, $nodeName));
$document->addField(Field::Keyword(self::IDX_NODELOCALNAME, $localNodeName));
$document->addField(Field::Keyword(self::IDX_PARENTPATH, $parentPath));
foreach ($node->getProperties() as $propertyName => $property) {
$propertyValue = $property['value'];
$propertyType = $property['type'];
if ($propertyName === Storage::INTERNAL_UUID) {
$document->addField(Field::Keyword(Storage::INTERNAL_UUID, $propertyValue));
continue;
}
switch ($propertyType) {
case PropertyType::TYPENAME_STRING:
case PropertyType::TYPENAME_NAME:
case PropertyType::TYPENAME_PATH:
case PropertyType::TYPENAME_URI:
$value = (array) $propertyValue;
$value = join(self::MULTIVALUE_SEPARATOR, $value);
$document->addField(Field::Text($propertyName, $value));
break;
case PropertyType::TYPENAME_DATE:
$values = (array) $propertyValue;
foreach ($values as $i => $value) {
if ($value instanceof \DateTime) {
$values[$i] = $value->format('c');
}
}
$value = join(self::MULTIVALUE_SEPARATOR, $values);
$document->addField(Field::Text($propertyName, $value));
break;
case PropertyType::TYPENAME_DECIMAL:
case PropertyType::TYPENAME_LONG:
case PropertyType::TYPENAME_DOUBLE:
$values = (array) $propertyValue;
foreach ($values as &$value) {
$value = sprintf('%0' . strlen(PHP_INT_MAX) . 's', $value);
}
$value = join(self::MULTIVALUE_SEPARATOR, $values);
$document->addField(Field::Text($propertyName, $value));
break;
case PropertyType::TYPENAME_BOOLEAN:
$values = (array) $propertyValue;
foreach ($values as &$value) {
if ($propertyValue === 'false') {
$value = self::VALUE_BOOLEAN_FALSE;
} else {
$value = 1;
}
}
$value = join(self::MULTIVALUE_SEPARATOR, $values);
$document->addField(Field::Text($propertyName, $value));
break;
}
}
$index->addDocument($document);
}
public function processAction()
{
$userEmail = $this->getAuthService()->getStorage()->read();
if (!$userEmail) {
$this->flashMessenger()->addErrorMessage("not authorized");
return $this->getResponse()->setContent("not authorized");
}
$request = $this->getRequest();
$form = new UploadForm();
$uploadFile = $this->params()->fromFiles('fileupload');
if ($request->isPost()) {
$form->setData($request->getPost());
if ($form->isValid()) {
// Получение конфигурации из конфигурационных данных модуля
$uploadPath = $this->getFileUploadLocation();
// Сохранение выгруженного файла
$adapter = new \Zend\File\Transfer\Adapter\Http();
$adapter->setDestination($uploadPath);
if ($adapter->receive($uploadFile['name'])) {
$userTable = $this->getServiceLocator()->get('UserTable');
$user = $userTable->getUserByEmail($userEmail);
$upload = new \Users\Model\Upload();
// Успешная выгрузка файла
$exchange_data = array();
$exchange_data['label'] = $request->getPost()->get('label');
$exchange_data['filename'] = $uploadFile['name'];
$exchange_data['user_id'] = $user->getId();
$upload->exchangeArray($exchange_data);
$uploadTable = $this->getServiceLocator()->get('UploadTable');
$uploadTable->save($upload);
$upload->setId($uploadTable->getLastInsertValue());
//добавить в Lucene
$searchIndexLocation = $this->getIndexLocation();
$index = Lucene\Lucene::create($searchIndexLocation);
// создание полей lucene
$fileUploadId = Document\Field::unIndexed('upload_id', $upload->getId());
$label = Document\Field::Text('label', $upload->getLabel());
$owner = Document\Field::Text('owner', $user->getName());
$uploadPath = $this->getFileUploadLocation();
$fileName = $upload->getFilename();
$filePath = $uploadPath . DIRECTORY_SEPARATOR . $fileName;
if (substr_compare($fileName, ".xlsx", strlen($fileName) - strlen(".xlsx"), strlen(".xlsx")) === 0) {
// Индексирование таблицы excel
$indexDoc = Lucene\Document\Xlsx::loadXlsxFile($filePath);
} else {
if (substr_compare($fileName, ".docx", strlen($fileName) - strlen(".docx"), strlen(".docx")) === 0) {
// Индексирование документа Word
$indexDoc = Lucene\Document\Docx::loadDocxFile($filePath);
} else {
$indexDoc = new Lucene\Document();
}
}
// создание нового документа и добавление всех полей
$indexDoc = new Lucene\Document();
$indexDoc->addField($label);
$indexDoc->addField($owner);
$indexDoc->addField($fileUploadId);
$index->addDocument($indexDoc);
$index->commit();
}
}
}
return $this->redirect()->toRoute('uploads', array('action' => 'index'));
}
/**
* Create document
*
* @param $model
* @param $attributes
* @return Document
*/
private function createDocument($model, $attributes)
{
$document = new Document();
foreach ($attributes as $name => $value) {
if (is_array($value)) {
list($attribute, $fieldType) = [key($value), current($value)];
$value = $this->getAttributeValue($model, $attribute);
} else {
$fieldType = self::FIELD_TEXT;
}
$document->addField(Field::$fieldType($name, strip_tags($value)));
}
if (!method_exists($model, 'getUrl')) {
throw new \yii\base\InvalidValueException('The identity object must implement PageLink.');
}
$document->addField(Field::Text('url', $model->getUrl()));
return $document;
}
public function index($indexer, $commit = true, $optimize = true)
{
$document = new Document();
$document->addField(Field::keyword('pk', $this->getId()));
$document->addField(Field::Text('course', $this->getSubjectcode()));
$document->addField(Field::Text('cross-listed', str_replace(array(';', ',', '|'), ' ', $this->getCrossListed())));
$document->addField(Field::Text('instructor', $this->getLegalContentOwner()));
$document->addField(Field::Unstored('comments', $this->getComments()));
$indexer->addDocument($document);
if ($commit) {
$indexer->commit();
}
if ($optimize) {
$indexer->optimize();
}
}
public function testTermsStreamInterfaceSkipToTermsRetrievingTwoTermsCase()
{
$index = Lucene\Lucene::create(__DIR__ . '/_index/_files');
// Zero terms
$doc = new Document();
$doc->addField(Document\Field::Text('contents', 'someterm word'));
$index->addDocument($doc);
unset($index);
$index = Lucene\Lucene::open(__DIR__ . '/_index/_files');
$index->resetTermsStream();
$index->skipTo(new Index\Term('term', 'contents'));
$this->assertTrue($index->currentTerm() == new Index\Term('word', 'contents'));
$index->closeTermsStream();
}
/**
* Indexa dados nos arquivos de json
*/
public function index()
{
$dir = realpath(dirname(__FILE__)) . DIRECTORY_SEPARATOR . "data" . DIRECTORY_SEPARATOR;
$jsonDir = $dir . "json";
$indexDir = $dir . "index";
// ler aquivos json
$files = scandir($jsonDir);
foreach ($files as $file) {
if ($file == '.' || $file == '..') {
continue;
}
// Se arquivo existe
if (is_file($jsonDir . DIRECTORY_SEPARATOR . $file)) {
$json = json_decode(file_get_contents($jsonDir . DIRECTORY_SEPARATOR . $file));
$indexName = substr($file, 0, -5);
// Cria index
$index = Lucene\Lucene::create($indexDir . DIRECTORY_SEPARATOR . $indexName);
// Cria documento e define campos para indexar
foreach ($json as $entry) {
$doc = new Lucene\Document();
$doc->addField(Lucene\Document\Field::Text('url', $entry->title));
$doc->addField(Lucene\Document\Field::UnStored('contents', $entry->text));
$index->addDocument($doc);
}
}
}
}