예제 #1
0
 public function testAddFieldMethodChaining()
 {
     $document = new Document();
     $this->assertTrue($document->addField(Document\Field::Text('title', 'Title')) instanceof Document);
     $document = new Document();
     $document->addField(Document\Field::Text('title', 'Title'))->addField(Document\Field::Text('annotation', 'Annotation'))->addField(Document\Field::Text('body', 'Document body, document body, document body...'));
 }
예제 #2
0
 /**
  * @dataProvider searchResultDataProvider
  */
 function testSearchLuceneResultContent($fileId, $name, $path, $size, $score, $mimeType, $modified, $container)
 {
     require_once __DIR__ . '/util/dummyindex.php';
     $index = new DummyIndex();
     $doc = new Document();
     $doc->addField(Document\Field::Keyword('fileId', $fileId));
     $doc->addField(Document\Field::Text('path', '/test/files' . $path, 'UTF-8'));
     $doc->addField(Document\Field::unIndexed('mtime', $modified));
     $doc->addField(Document\Field::unIndexed('size', $size));
     $doc->addField(Document\Field::unIndexed('mimetype', $mimeType));
     $index->addDocument($doc);
     $hit = new QueryHit($index);
     $hit->score = $score;
     $hit->id = 0;
     $hit->document_id = 0;
     $searchResult = new \OCA\Search_Lucene\Search\LuceneResult($hit);
     $this->assertInstanceOf('OCA\\Search_Lucene\\Search\\LuceneResult', $searchResult);
     $this->assertEquals($fileId, $searchResult->id);
     $this->assertEquals('lucene', $searchResult->type);
     $this->assertEquals($path, $searchResult->path);
     $this->assertEquals($name, $searchResult->name);
     $this->assertEquals($mimeType, $searchResult->mime_type);
     $this->assertEquals($size, $searchResult->size);
     $this->assertEquals($score, $searchResult->score);
     $this->assertEquals($modified, $searchResult->modified);
 }
예제 #3
0
파일: pdf.php 프로젝트: ntvis/search_lucene
 /**
  * Object constructor
  *
  * @param string  $data
  * @param boolean $storeContent
  * @throws NotIndexedException
  */
 private function __construct($data, $storeContent)
 {
     //TODO check PDF >1.5 metadata extraction
     //do the content extraction
     $parser = new Parser();
     try {
         $pdf = $parser->parseContent($data);
         $body = $pdf->getText();
         // Store contents
         if ($storeContent) {
             $this->addField(Document\Field::Text('body', $body, 'UTF-8'));
         } else {
             $this->addField(Document\Field::UnStored('body', $body, 'UTF-8'));
         }
         $details = $pdf->getDetails();
         // Store meta data properties
         foreach ($details as $key => $value) {
             $key = strtolower($key);
             if ($key === 'author') {
                 $key = 'creator';
             }
             $this->addField(Document\Field::Text($key, $value, 'UTF-8'));
         }
     } catch (\Exception $ex) {
         throw new NotIndexedException(null, null, $ex);
     }
 }
예제 #4
0
 public function add(Searchable $obj)
 {
     // Get Primary Key
     $attributes = $obj->getSearchAttributes();
     $index = $this->getIndex();
     $doc = new \ZendSearch\Lucene\Document();
     // Add Meta Data fields
     foreach ($this->getMetaInfoArray($obj) as $fieldName => $fieldValue) {
         $doc->addField(\ZendSearch\Lucene\Document\Field::keyword($fieldName, $fieldValue));
     }
     // Add provided search infos
     foreach ($attributes as $key => $val) {
         $doc->addField(\ZendSearch\Lucene\Document\Field::Text($key, $val, 'UTF-8'));
     }
     // Add comments - if record is content
     if ($obj instanceof ContentActiveRecord) {
         $comments = "";
         foreach (Comment::findAll(['object_id' => $obj->getPrimaryKey(), 'object_model' => $obj->className()]) as $comment) {
             $comments .= " " . $comment->message;
         }
         $doc->addField(\ZendSearch\Lucene\Document\Field::Text('comments', $comments, 'UTF-8'));
     }
     if (\Yii::$app->request->isConsoleRequest) {
         print ".";
     }
     $index->addDocument($doc);
     $index->commit();
 }
예제 #5
0
파일: odt.php 프로젝트: ntvis/search_lucene
 /**
  * Object constructor
  *
  * @param string $fileName
  * @param boolean $storeContent
  * @throws ExtensionNotLoadedException
  * @throws RuntimeException
  */
 private function __construct($fileName, $storeContent)
 {
     if (!class_exists('ZipArchive', false)) {
         throw new ExtensionNotLoadedException('Open Document Text processing functionality requires Zip extension to be loaded');
     }
     // Document data holders
     $documentHeadlines = array();
     $documentParagraphs = array();
     // Open OpenXML package
     $package = new \ZipArchive();
     $package->open($fileName);
     // Read relations and search for officeDocument
     $content = $package->getFromName('content.xml');
     if ($content === false) {
         throw new RuntimeException('Invalid archive or corrupted .odt file.');
     }
     // Prevent php from loading remote resources
     $loadEntities = libxml_disable_entity_loader(true);
     $sxe = simplexml_load_string($content, 'SimpleXMLElement', LIBXML_NOBLANKS | LIBXML_COMPACT);
     // Restore entity loader state
     libxml_disable_entity_loader($loadEntities);
     foreach ($sxe->xpath('//text:h') as $headline) {
         $h = strip_tags($headline->asXML());
         $documentHeadlines[] = $h;
     }
     foreach ($sxe->xpath('//text:p') as $paragraph) {
         $p = strip_tags($paragraph->asXML());
         $documentParagraphs[] = $p;
     }
     // Read core properties
     $coreProperties = $this->extractMetaData($package);
     // Close file
     $package->close();
     // Store contents
     if ($storeContent) {
         $this->addField(Field::Text('headlines', implode(' ', $documentHeadlines), 'UTF-8'));
         $this->addField(Field::Text('body', implode('', $documentParagraphs), 'UTF-8'));
     } else {
         $this->addField(Field::UnStored('headlines', implode(' ', $documentHeadlines), 'UTF-8'));
         $this->addField(Field::UnStored('body', implode('', $documentParagraphs), 'UTF-8'));
     }
     // Store meta data properties
     foreach ($coreProperties as $key => $value) {
         $this->addField(Field::Text($key, $value, 'UTF-8'));
     }
     // Store title (if not present in meta data)
     if (!isset($coreProperties['title'])) {
         $this->addField(Field::Text('title', $fileName, 'UTF-8'));
     }
 }
예제 #6
0
파일: Group.php 프로젝트: bix0r/Stjornvisi
 /**
  * @param $data
  * @param SearchIndexInterface $index
  *
  * @return IndexInterface
  */
 public function index($data, SearchIndexInterface $index)
 {
     $this->unindex($data, $index);
     $indexDoc = new Document();
     $indexDoc->addField(Field::Keyword('group_id', $data->id));
     $indexDoc->addField(Field::UnIndexed('type', "group"));
     $indexDoc->addField(Field::UnIndexed('identifier', $data->url));
     $indexDoc->addField(Field::UnIndexed('date_time', date('c')));
     $indexDoc->addField(Field::UnIndexed('date', date('j. M. Y')));
     $indexDoc->addField(Field::Text('title', $data->name_short, 'utf-8'));
     $indexDoc->addField(Field::Text('body', $data->description, 'utf-8'));
     $index->addDocument($indexDoc);
     return $this;
 }
예제 #7
0
파일: News.php 프로젝트: bix0r/Stjornvisi
 /**
  * @param $data
  * @param SearchIndexInterface $index
  *
  * @return IndexInterface
  */
 public function index($data, SearchIndexInterface $index)
 {
     $this->unindex($data, $index);
     $indexDoc = new Document();
     $indexDoc->addField(Field::Keyword('news_id', $data->id));
     $indexDoc->addField(Field::UnIndexed('type', "news"));
     $indexDoc->addField(Field::UnIndexed('identifier', $data->id));
     $indexDoc->addField(Field::UnIndexed('date_time', $data->created_date->format('c')));
     $indexDoc->addField(Field::UnIndexed('date', $data->created_date->format('j. M. Y')));
     $indexDoc->addField(Field::Text('title', $data->title, 'utf-8'));
     $indexDoc->addField(Field::Text('body', $data->body, 'utf-8'));
     $index->addDocument($indexDoc);
     return $this;
 }
예제 #8
0
 function testUpdate()
 {
     // preparation
     $app = new Application();
     $container = $app->getContainer();
     // get an index
     /** @var Index $index */
     $index = $container->query('Index');
     // add a document
     $doc = new Document();
     $doc->addField(Document\Field::Keyword('fileId', '1'));
     $doc->addField(Document\Field::Text('path', '/somewhere/deep/down/the/rabbit/hole', 'UTF-8'));
     $doc->addField(Document\Field::Text('users', 'alice', 'UTF-8'));
     $index->index->addDocument($doc);
     $index->commit();
     // search for it
     $idTerm = new Term('1', 'fileId');
     $idQuery = new Query\Term($idTerm);
     $query = new Query\Boolean();
     $query->addSubquery($idQuery);
     /** @var QueryHit $hit */
     $hits = $index->find($query);
     // get the document from the query hit
     $foundDoc = $hits[0]->getDocument();
     $this->assertEquals('alice', $foundDoc->getFieldValue('users'));
     // delete the document from the index
     //$index->index->delete($hit);
     // change the 'users' key of the document
     $foundDoc->addField(Document\Field::Text('users', 'bob', 'UTF-8'));
     $this->assertEquals('bob', $foundDoc->getFieldValue('users'));
     // add the document back to the index
     $index->updateFile($foundDoc, '1');
     $idTerm2 = new Term('1', 'fileId');
     $idQuery2 = new Query\Term($idTerm2);
     $query2 = new Query\Boolean();
     $query2->addSubquery($idQuery2);
     /** @var QueryHit $hit */
     $hits2 = $index->find($query2);
     // get the document from the query hit
     $foundDoc2 = $hits2[0]->getDocument();
     $this->assertEquals('bob', $foundDoc2->getFieldValue('users'));
 }
예제 #9
0
 public function generateIndexAction()
 {
     $searchIndexLocation = $this->getIndexLocation();
     $index = Lucene\Lucene::create($searchIndexLocation);
     $userTable = $this->getServiceLocator()->get('UserTable');
     $uploadTable = $this->getServiceLocator()->get('UploadTable');
     $allUploads = $uploadTable->fetchAll();
     foreach ($allUploads as $fileUpload) {
         $uploadOwner = $userTable->getById($fileUpload->getUserId());
         // создание полей lucene
         $fileUploadId = Document\Field::unIndexed('upload_id', $fileUpload->getId());
         $label = Document\Field::Text('label', $fileUpload->getLabel());
         $owner = Document\Field::Text('owner', $uploadOwner->getName());
         $uploadPath = $this->getFileUploadLocation();
         $fileName = $fileUpload->getFilename();
         $filePath = $uploadPath . DIRECTORY_SEPARATOR . $fileName;
         if (substr_compare($fileName, ".xlsx", strlen($fileName) - strlen(".xlsx"), strlen(".xlsx")) === 0) {
             // Индексирование таблицы excel
             $indexDoc = Lucene\Document\Xlsx::loadXlsxFile($filePath);
         } else {
             if (substr_compare($fileName, ".docx", strlen($fileName) - strlen(".docx"), strlen(".docx")) === 0) {
                 // Индексирование документа Word
                 $indexDoc = Lucene\Document\Docx::loadDocxFile($filePath);
             } else {
                 $indexDoc = new Lucene\Document();
             }
         }
         // создание нового документа и добавление всех полей
         $indexDoc = new Lucene\Document();
         $indexDoc->addField($label);
         $indexDoc->addField($owner);
         $indexDoc->addField($fileUploadId);
         $index->addDocument($indexDoc);
     }
     $index->commit();
     $response = $this->getResponse();
     $response->setContent("Index Ok");
     return $response;
 }
예제 #10
0
 /**
  * インデックスファイルを生成
  */
 public static function updateIndex()
 {
     if (empty(self::$igo)) {
         self::$igo = new Tagger(array('dict_dir' => LIB_DIR . 'ipadic', 'reduce_mode' => true));
     }
     Analyzer::setDefault(new Utf8());
     // 索引の作成
     $index = Lucene::create(CACHE_DIR . self::INDEX_NAME);
     foreach (Listing::pages() as $page) {
         if (empty($page)) {
             continue;
         }
         $wiki = Factory::Wiki($page);
         // 読む権限がない場合スキップ
         if (!$wiki->isReadable() || $wiki->isHidden()) {
             continue;
         }
         /*
         			// HTML出力
         			$html[] = '<html><head>';
         			$html[] = '<meta http-equiv="Content-type" content="text/html; charset=UTF-8"/>';
         			$html[] = '<title>' . $wiki->title() . '</title>';
         			$html[] = '</head>';
         			$html[] = '<body>' . $wiki->render() . '</body>';
         			$html[] = '</html>';
         */
         $doc = new LuceneDoc();
         $doc->addField(Field::Text('title', $wiki->title()));
         // Store document URL to identify it in the search results
         $doc->addField(Field::Text('url', $wiki->uri()));
         // Index document contents
         //$contents = join(" ", self::$igo->wakati(strip_tags($wiki->render())));
         $contents = strip_tags($wiki->render());
         $doc->addField(Field::UnStored('contents', $contents));
         // 索引へ文書の登録
         $index->addDocument($doc);
     }
     $index->optimize();
 }
예제 #11
0
 /**
  * Object constructor
  *
  * @param string  $fileName
  * @param boolean $storeContent
  * @throws \ZendSearch\Lucene\Exception\ExtensionNotLoadedException
  * @throws \ZendSearch\Lucene\Exception\RuntimeException
  */
 private function __construct($fileName, $storeContent)
 {
     if (!class_exists('ZipArchive', false)) {
         throw new ExtensionNotLoadedException('MS Office documents processing functionality requires Zip extension to be loaded');
     }
     // Document data holders
     $slides = array();
     $slideNotes = array();
     $documentBody = array();
     $coreProperties = array();
     // Open AbstractOpenXML package
     $package = new \ZipArchive();
     $package->open($fileName);
     // Read relations and search for officeDocument
     $relationsXml = $package->getFromName('_rels/.rels');
     if ($relationsXml === false) {
         throw new RuntimeException('Invalid archive or corrupted .pptx file.');
     }
     $relations = simplexml_load_string($relationsXml);
     foreach ($relations->Relationship as $rel) {
         if ($rel["Type"] == AbstractOpenXML::SCHEMA_OFFICEDOCUMENT) {
             // Found office document! Search for slides...
             $slideRelations = simplexml_load_string($package->getFromName($this->absoluteZipPath(dirname($rel["Target"]) . "/_rels/" . basename($rel["Target"]) . ".rels")));
             foreach ($slideRelations->Relationship as $slideRel) {
                 if ($slideRel["Type"] == self::SCHEMA_SLIDERELATION) {
                     // Found slide!
                     $slides[str_replace('rId', '', (string) $slideRel["Id"])] = simplexml_load_string($package->getFromName($this->absoluteZipPath(dirname($rel["Target"]) . "/" . dirname($slideRel["Target"]) . "/" . basename($slideRel["Target"]))));
                     // Search for slide notes
                     $slideNotesRelations = simplexml_load_string($package->getFromName($this->absoluteZipPath(dirname($rel["Target"]) . "/" . dirname($slideRel["Target"]) . "/_rels/" . basename($slideRel["Target"]) . ".rels")));
                     foreach ($slideNotesRelations->Relationship as $slideNoteRel) {
                         if ($slideNoteRel["Type"] == self::SCHEMA_SLIDENOTESRELATION) {
                             // Found slide notes!
                             $slideNotes[str_replace('rId', '', (string) $slideRel["Id"])] = simplexml_load_string($package->getFromName($this->absoluteZipPath(dirname($rel["Target"]) . "/" . dirname($slideRel["Target"]) . "/" . dirname($slideNoteRel["Target"]) . "/" . basename($slideNoteRel["Target"]))));
                             break;
                         }
                     }
                 }
             }
             break;
         }
     }
     // Sort slides
     ksort($slides);
     ksort($slideNotes);
     // Extract contents from slides
     foreach ($slides as $slideKey => $slide) {
         // Register namespaces
         $slide->registerXPathNamespace("p", self::SCHEMA_PRESENTATIONML);
         $slide->registerXPathNamespace("a", self::SCHEMA_DRAWINGML);
         // Fetch all text
         $textElements = $slide->xpath('//a:t');
         foreach ($textElements as $textElement) {
             $documentBody[] = (string) $textElement;
         }
         // Extract contents from slide notes
         if (isset($slideNotes[$slideKey])) {
             // Fetch slide note
             $slideNote = $slideNotes[$slideKey];
             // Register namespaces
             $slideNote->registerXPathNamespace("p", self::SCHEMA_PRESENTATIONML);
             $slideNote->registerXPathNamespace("a", self::SCHEMA_DRAWINGML);
             // Fetch all text
             $textElements = $slideNote->xpath('//a:t');
             foreach ($textElements as $textElement) {
                 $documentBody[] = (string) $textElement;
             }
         }
     }
     // Read core properties
     $coreProperties = $this->extractMetaData($package);
     // Close file
     $package->close();
     // Store filename
     $this->addField(Field::Text('filename', $fileName, 'UTF-8'));
     // Store contents
     if ($storeContent) {
         $this->addField(Field::Text('body', implode(' ', $documentBody), 'UTF-8'));
     } else {
         $this->addField(Field::UnStored('body', implode(' ', $documentBody), 'UTF-8'));
     }
     // Store meta data properties
     foreach ($coreProperties as $key => $value) {
         $this->addField(Field::Text($key, $value, 'UTF-8'));
     }
     // Store title (if not present in meta data)
     if (!isset($coreProperties['title'])) {
         $this->addField(Field::Text('title', $fileName, 'UTF-8'));
     }
 }
예제 #12
0
 /**
  * Object constructor
  *
  * @param string  $fileName
  * @param boolean $storeContent
  * @throws \ZendSearch\Lucene\Exception\ExtensionNotLoadedException
  * @throws \ZendSearch\Lucene\Exception\RuntimeException
  */
 private function __construct($fileName, $storeContent)
 {
     if (!class_exists('ZipArchive', false)) {
         throw new ExtensionNotLoadedException('MS Office documents processing functionality requires Zip extension to be loaded');
     }
     // Document data holders
     $sharedStrings = array();
     $worksheets = array();
     $documentBody = array();
     $coreProperties = array();
     // Open AbstractOpenXML package
     $package = new \ZipArchive();
     $package->open($fileName);
     // Read relations and search for officeDocument
     $relationsXml = $package->getFromName('_rels/.rels');
     if ($relationsXml === false) {
         throw new RuntimeException('Invalid archive or corrupted .xlsx file.');
     }
     $relations = XmlSecurity::scan($relationsXml);
     foreach ($relations->Relationship as $rel) {
         if ($rel["Type"] == AbstractOpenXML::SCHEMA_OFFICEDOCUMENT) {
             // Found office document! Read relations for workbook...
             $workbookRelations = XmlSecurity::scan($package->getFromName($this->absoluteZipPath(dirname($rel["Target"]) . "/_rels/" . basename($rel["Target"]) . ".rels")));
             $workbookRelations->registerXPathNamespace("rel", AbstractOpenXML::SCHEMA_RELATIONSHIP);
             // Read shared strings
             $sharedStringsPath = $workbookRelations->xpath("rel:Relationship[@Type='" . self::SCHEMA_SHAREDSTRINGS . "']");
             $sharedStringsPath = (string) $sharedStringsPath[0]['Target'];
             $xmlStrings = XmlSecurity::scan($package->getFromName($this->absoluteZipPath(dirname($rel["Target"]) . "/" . $sharedStringsPath)));
             if (isset($xmlStrings) && isset($xmlStrings->si)) {
                 foreach ($xmlStrings->si as $val) {
                     if (isset($val->t)) {
                         $sharedStrings[] = (string) $val->t;
                     } elseif (isset($val->r)) {
                         $sharedStrings[] = $this->_parseRichText($val);
                     }
                 }
             }
             // Loop relations for workbook and extract worksheets...
             foreach ($workbookRelations->Relationship as $workbookRelation) {
                 if ($workbookRelation["Type"] == self::SCHEMA_WORKSHEETRELATION) {
                     $worksheets[str_replace('rId', '', (string) $workbookRelation["Id"])] = XmlSecurity::scan($package->getFromName($this->absoluteZipPath(dirname($rel["Target"]) . "/" . dirname($workbookRelation["Target"]) . "/" . basename($workbookRelation["Target"]))));
                 }
             }
             break;
         }
     }
     // Sort worksheets
     ksort($worksheets);
     // Extract contents from worksheets
     foreach ($worksheets as $sheetKey => $worksheet) {
         foreach ($worksheet->sheetData->row as $row) {
             foreach ($row->c as $c) {
                 // Determine data type
                 $dataType = (string) $c["t"];
                 switch ($dataType) {
                     case "s":
                         // Value is a shared string
                         if ((string) $c->v != '') {
                             $value = $sharedStrings[intval($c->v)];
                         } else {
                             $value = '';
                         }
                         break;
                     case "b":
                         // Value is boolean
                         $value = (string) $c->v;
                         if ($value == '0') {
                             $value = false;
                         } elseif ($value == '1') {
                             $value = true;
                         } else {
                             $value = (bool) $c->v;
                         }
                         break;
                     case "inlineStr":
                         // Value is rich text inline
                         $value = $this->_parseRichText($c->is);
                         break;
                     case "e":
                         // Value is an error message
                         if ((string) $c->v != '') {
                             $value = (string) $c->v;
                         } else {
                             $value = '';
                         }
                         break;
                     default:
                         // Value is a string
                         $value = (string) $c->v;
                         // Check for numeric values
                         if (is_numeric($value) && $dataType != 's') {
                             if ($value == (int) $value) {
                                 $value = (int) $value;
                             } elseif ($value == (double) $value) {
                                 $value = (double) $value;
                             } elseif ($value == (double) $value) {
                                 $value = (double) $value;
                             }
                         }
                 }
                 $documentBody[] = $value;
             }
         }
     }
     // Read core properties
     $coreProperties = $this->extractMetaData($package);
     // Close file
     $package->close();
     // Store filename
     $this->addField(Field::Text('filename', $fileName, 'UTF-8'));
     // Store contents
     if ($storeContent) {
         $this->addField(Field::Text('body', implode(' ', $documentBody), 'UTF-8'));
     } else {
         $this->addField(Field::UnStored('body', implode(' ', $documentBody), 'UTF-8'));
     }
     // Store meta data properties
     foreach ($coreProperties as $key => $value) {
         $this->addField(Field::Text($key, $value, 'UTF-8'));
     }
     // Store title (if not present in meta data)
     if (!isset($coreProperties['title'])) {
         $this->addField(Field::Text('title', $fileName, 'UTF-8'));
     }
 }
예제 #13
0
 /**
  * Object constructor
  *
  * @param string  $fileName
  * @param boolean $storeContent
  * @throws \ZendSearch\Lucene\Exception\ExtensionNotLoadedException
  * @throws \ZendSearch\Lucene\Exception\RuntimeException
  */
 private function __construct($fileName, $storeContent)
 {
     if (!class_exists('ZipArchive', false)) {
         throw new ExtensionNotLoadedException('MS Office documents processing functionality requires Zip extension to be loaded');
     }
     // Document data holders
     $documentBody = array();
     $coreProperties = array();
     // Open AbstractOpenXML package
     $package = new \ZipArchive();
     $package->open($fileName);
     // Read relations and search for officeDocument
     $relationsXml = $package->getFromName('_rels/.rels');
     if ($relationsXml === false) {
         throw new RuntimeException('Invalid archive or corrupted .docx file.');
     }
     $relations = XMLSecurity::scan($relationsXml);
     foreach ($relations->Relationship as $rel) {
         if ($rel["Type"] == AbstractOpenXML::SCHEMA_OFFICEDOCUMENT) {
             // Found office document! Read in contents...
             $contents = XMLSecurity::scan($package->getFromName($this->absoluteZipPath(dirname($rel['Target']) . '/' . basename($rel['Target']))));
             $contents->registerXPathNamespace('w', self::SCHEMA_WORDPROCESSINGML);
             $paragraphs = $contents->xpath('//w:body/w:p');
             foreach ($paragraphs as $paragraph) {
                 $runs = $paragraph->xpath('.//w:r/*[name() = "w:t" or name() = "w:br"]');
                 if ($runs === false) {
                     // Paragraph doesn't contain any text or breaks
                     continue;
                 }
                 foreach ($runs as $run) {
                     if ($run->getName() == 'br') {
                         // Break element
                         $documentBody[] = ' ';
                     } else {
                         $documentBody[] = (string) $run;
                     }
                 }
                 // Add space after each paragraph. So they are not bound together.
                 $documentBody[] = ' ';
             }
             break;
         }
     }
     // Read core properties
     $coreProperties = $this->extractMetaData($package);
     // Close file
     $package->close();
     // Store filename
     $this->addField(Field::Text('filename', $fileName, 'UTF-8'));
     // Store contents
     if ($storeContent) {
         $this->addField(Field::Text('body', implode('', $documentBody), 'UTF-8'));
     } else {
         $this->addField(Field::UnStored('body', implode('', $documentBody), 'UTF-8'));
     }
     // Store meta data properties
     foreach ($coreProperties as $key => $value) {
         $this->addField(Field::Text($key, $value, 'UTF-8'));
     }
     // Store title (if not present in meta data)
     if (!isset($coreProperties['title'])) {
         $this->addField(Field::Text('title', $fileName, 'UTF-8'));
     }
 }
예제 #14
0
 /**
  * index a file
  *
  * @param File $file the file to be indexed
  * @param bool $commit
  *
  * @return bool true when something was stored in the index, false otherwise (eg, folders are not indexed)
  * @throws NotIndexedException when an unsupported file type is encountered
  */
 public function indexFile(File $file, $commit = true)
 {
     // we decide how to index on mime type or file extension
     $mimeType = $file->getMimeType();
     $fileExtension = strtolower(pathinfo($file->getName(), PATHINFO_EXTENSION));
     // initialize plain lucene document
     $doc = new Document();
     // index content for local files only
     $storage = $file->getStorage();
     if ($storage->isLocal()) {
         $path = $storage->getLocalFile($file->getInternalPath());
         //try to use special lucene document types
         if ('text/html' === $mimeType) {
             //TODO could be indexed, even if not local
             $doc = HTML::loadHTML($file->getContent());
         } else {
             if ('text/' === substr($mimeType, 0, 5) || 'application/x-tex' === $mimeType) {
                 $body = $file->getContent();
                 if ($body != '') {
                     $doc->addField(Document\Field::UnStored('body', $body));
                 }
             } else {
                 if ('application/pdf' === $mimeType) {
                     $doc = Pdf::loadPdf($file->getContent());
                     // the zend classes only understand docx and not doc files
                 } else {
                     if ($fileExtension === 'docx') {
                         $doc = Document\Docx::loadDocxFile($path);
                         //} else if ('application/msexcel' === $mimeType) {
                     } else {
                         if ($fileExtension === 'xlsx') {
                             $doc = Document\Xlsx::loadXlsxFile($path);
                             //} else if ('application/mspowerpoint' === $mimeType) {
                         } else {
                             if ($fileExtension === 'pptx') {
                                 $doc = Document\Pptx::loadPptxFile($path);
                             } else {
                                 if ($fileExtension === 'odt') {
                                     $doc = Odt::loadOdtFile($path);
                                 } else {
                                     if ($fileExtension === 'ods') {
                                         $doc = Ods::loadOdsFile($path);
                                     } else {
                                         throw new NotIndexedException();
                                     }
                                 }
                             }
                         }
                     }
                 }
             }
         }
     }
     // Store filecache id as unique id to lookup by when deleting
     $doc->addField(Document\Field::Keyword('fileId', $file->getId()));
     // Store document path for the search results
     $doc->addField(Document\Field::Text('path', $file->getPath(), 'UTF-8'));
     $doc->addField(Document\Field::unIndexed('mtime', $file->getMTime()));
     $doc->addField(Document\Field::unIndexed('size', $file->getSize()));
     $doc->addField(Document\Field::unIndexed('mimetype', $mimeType));
     $this->index->updateFile($doc, $file->getId(), $commit);
     return true;
 }
예제 #15
0
 public function testEncoding()
 {
     // forcing filter to UTF-8
     $utf8text = iconv('UTF-8', 'UTF-8', 'Words with umlauts: åãü...');
     $iso8859_1 = iconv('UTF-8', 'ISO-8859-1', $utf8text);
     $field = Document\Field::Text('field', $iso8859_1, 'ISO-8859-1');
     $this->assertEquals($field->encoding, 'ISO-8859-1');
     $this->assertEquals($field->name, 'field');
     $this->assertEquals($field->value, $iso8859_1);
     $this->assertEquals($field->getUtf8Value(), $utf8text);
 }
 protected function indexProperty(Document $document, \core_kernel_classes_Property $property)
 {
     $indexes = $property->getPropertyValues(new \core_kernel_classes_Property('http://www.tao.lu/Ontologies/TAO.rdf#PropertyIndex'));
     foreach ($indexes as $indexUri) {
         $index = new Index($indexUri);
         $id = $index->getIdentifier();
         $strings = $index->tokenize($this->resource->getPropertyValues($property));
         if (!empty($strings)) {
             if ($index->isFuzzyMatching()) {
                 // cannot store multiple fuzzy strings
                 $string = implode(' ', $strings);
                 $field = Document\Field::Text($index->getIdentifier(), $string);
                 $field->isStored = $index->isStored();
                 $document->addField($field);
             } else {
                 $value = count($strings) > 1 ? $strings : reset($strings);
                 $field = Document\Field::Keyword($index->getIdentifier(), $value);
                 $field->isStored = $index->isStored() && !is_array($value);
                 // storage of arrays not supported
                 $document->addField($field);
             }
         }
     }
 }
예제 #17
0
파일: HTML.php 프로젝트: tonylow/skillslink
 /**
  * Object constructor
  *
  * @param string  $data         HTML string (may be HTML fragment, )
  * @param boolean $isFile
  * @param boolean $storeContent
  * @param string  $defaultEncoding   HTML encoding, is used if it's not specified using Content-type HTTP-EQUIV meta tag.
  */
 private function __construct($data, $isFile, $storeContent, $defaultEncoding = '')
 {
     $this->_doc = new \DOMDocument();
     $this->_doc->substituteEntities = true;
     if ($isFile) {
         $htmlData = file_get_contents($data);
     } else {
         $htmlData = $data;
     }
     ErrorHandler::start(E_WARNING);
     $this->_doc->loadHTML($htmlData);
     ErrorHandler::stop();
     if ($this->_doc->encoding === null) {
         // Document encoding is not recognized
         /** @todo improve HTML vs HTML fragment recognition */
         if (preg_match('/<html>/i', $htmlData, $matches, PREG_OFFSET_CAPTURE)) {
             // It's an HTML document
             // Add additional HEAD section and recognize document
             $htmlTagOffset = $matches[0][1] + strlen($matches[0][0]);
             ErrorHandler::start(E_WARNING);
             $this->_doc->loadHTML(iconv($defaultEncoding, 'UTF-8//IGNORE', substr($htmlData, 0, $htmlTagOffset)) . '<head><META HTTP-EQUIV="Content-type" CONTENT="text/html; charset=UTF-8"/></head>' . iconv($defaultEncoding, 'UTF-8//IGNORE', substr($htmlData, $htmlTagOffset)));
             ErrorHandler::stop();
             // Remove additional HEAD section
             $xpath = new \DOMXPath($this->_doc);
             $head = $xpath->query('/html/head')->item(0);
             $head->parentNode->removeChild($head);
         } else {
             // It's an HTML fragment
             ErrorHandler::start(E_WARNING);
             $this->_doc->loadHTML('<html><head><META HTTP-EQUIV="Content-type" CONTENT="text/html; charset=UTF-8"/></head><body>' . iconv($defaultEncoding, 'UTF-8//IGNORE', $htmlData) . '</body></html>');
             ErrorHandler::stop();
         }
     }
     /** @todo Add correction of wrong HTML encoding recognition processing
      * The case is:
      * Content-type HTTP-EQUIV meta tag is presented, but ISO-8859-5 encoding is actually used,
      * even $this->_doc->encoding demonstrates another recognized encoding
      */
     $xpath = new \DOMXPath($this->_doc);
     $docTitle = '';
     $titleNodes = $xpath->query('/html/head/title');
     foreach ($titleNodes as $titleNode) {
         // title should always have only one entry, but we process all nodeset entries
         $docTitle .= $titleNode->nodeValue . ' ';
     }
     $this->addField(Field::Text('title', $docTitle, 'UTF-8'));
     $metaNodes = $xpath->query('/html/head/meta[@name]');
     foreach ($metaNodes as $metaNode) {
         $this->addField(Field::Text($metaNode->getAttribute('name'), $metaNode->getAttribute('content'), 'UTF-8'));
     }
     $docBody = '';
     $bodyNodes = $xpath->query('/html/body');
     foreach ($bodyNodes as $bodyNode) {
         // body should always have only one entry, but we process all nodeset entries
         $this->_retrieveNodeText($bodyNode, $docBody);
     }
     if ($storeContent) {
         $this->addField(Field::Text('body', $docBody, 'UTF-8'));
     } else {
         $this->addField(Field::UnStored('body', $docBody, 'UTF-8'));
     }
     $linkNodes = $this->_doc->getElementsByTagName('a');
     foreach ($linkNodes as $linkNode) {
         if (($href = $linkNode->getAttribute('href')) != '' && (!self::$_excludeNoFollowLinks || strtolower($linkNode->getAttribute('rel')) != 'nofollow')) {
             $this->_links[] = $href;
         }
     }
     $linkNodes = $this->_doc->getElementsByTagName('area');
     foreach ($linkNodes as $linkNode) {
         if (($href = $linkNode->getAttribute('href')) != '' && (!self::$_excludeNoFollowLinks || strtolower($linkNode->getAttribute('rel')) != 'nofollow')) {
             $this->_links[] = $href;
         }
     }
     $this->_links = array_unique($this->_links);
     $linkNodes = $xpath->query('/html/head/link');
     foreach ($linkNodes as $linkNode) {
         if (($href = $linkNode->getAttribute('href')) != '') {
             $this->_headerLinks[] = $href;
         }
     }
     $this->_headerLinks = array_unique($this->_headerLinks);
 }
 public function generateIndexAction()
 {
     $searchIndexLocation = $this->getIndexLocation();
     $index = Lucene\Lucene::create($searchIndexLocation);
     $userTable = $this->getServiceLocator()->get('UserTable');
     $uploadTable = $this->getServiceLocator()->get('UploadTable');
     $allUploads = $uploadTable->fetchAll();
     foreach ($allUploads as $fileUpload) {
         //
         $uploadOwner = $userTable->getUser($fileUpload->user_id);
         // id field
         $fileUploadId = Document\Field::unIndexed('upload_id', $fileUpload->id);
         // label field
         $label = Document\Field::Text('label', $fileUpload->label);
         // owner field
         $owner = Document\Field::Text('owner', $uploadOwner->name);
         if (substr_compare($fileUpload->filename, ".xlsx", strlen($fileUpload->filename) - strlen(".xlsx"), strlen(".xlsx")) === 0) {
             // index excel sheet
             $uploadPath = $this->getFileUploadLocation();
             $indexDoc = Lucene\Document\Xlsx::loadXlsxFile($uploadPath . "/" . $fileUpload->filename);
         } else {
             if (substr_compare($fileUpload->filename, ".docx", strlen($fileUpload->filename) - strlen(".docx"), strlen(".docx")) === 0) {
                 // index word doc
                 $uploadPath = $this->getFileUploadLocation();
                 $indexDoc = Lucene\Document\Docx::loadDocxFile($uploadPath . "/" . $fileUpload->filename);
             } else {
                 $indexDoc = new Lucene\Document();
             }
         }
         $indexDoc->addField($label);
         $indexDoc->addField($owner);
         $indexDoc->addField($fileUploadId);
         $index->addDocument($indexDoc);
     }
     $index->commit();
 }
예제 #19
0
 /**
  * {@inheritDoc}
  */
 public function index($workspace, $path, Node $node)
 {
     $index = $this->getIndex($workspace);
     $document = new Document();
     $nodeName = PathHelper::getNodeName($path);
     $localNodeName = $nodeName;
     // PathHelper::getLocalNodeName($path);
     $parentPath = PathHelper::getParentPath($path);
     $document->addField(Field::Keyword(self::IDX_PATH, $path));
     $document->addField(Field::Keyword(self::IDX_NODENAME, $nodeName));
     $document->addField(Field::Keyword(self::IDX_NODELOCALNAME, $localNodeName));
     $document->addField(Field::Keyword(self::IDX_PARENTPATH, $parentPath));
     foreach ($node->getProperties() as $propertyName => $property) {
         $propertyValue = $property['value'];
         $propertyType = $property['type'];
         if ($propertyName === Storage::INTERNAL_UUID) {
             $document->addField(Field::Keyword(Storage::INTERNAL_UUID, $propertyValue));
             continue;
         }
         switch ($propertyType) {
             case PropertyType::TYPENAME_STRING:
             case PropertyType::TYPENAME_NAME:
             case PropertyType::TYPENAME_PATH:
             case PropertyType::TYPENAME_URI:
                 $value = (array) $propertyValue;
                 $value = join(self::MULTIVALUE_SEPARATOR, $value);
                 $document->addField(Field::Text($propertyName, $value));
                 break;
             case PropertyType::TYPENAME_DATE:
                 $values = (array) $propertyValue;
                 foreach ($values as $i => $value) {
                     if ($value instanceof \DateTime) {
                         $values[$i] = $value->format('c');
                     }
                 }
                 $value = join(self::MULTIVALUE_SEPARATOR, $values);
                 $document->addField(Field::Text($propertyName, $value));
                 break;
             case PropertyType::TYPENAME_DECIMAL:
             case PropertyType::TYPENAME_LONG:
             case PropertyType::TYPENAME_DOUBLE:
                 $values = (array) $propertyValue;
                 foreach ($values as &$value) {
                     $value = sprintf('%0' . strlen(PHP_INT_MAX) . 's', $value);
                 }
                 $value = join(self::MULTIVALUE_SEPARATOR, $values);
                 $document->addField(Field::Text($propertyName, $value));
                 break;
             case PropertyType::TYPENAME_BOOLEAN:
                 $values = (array) $propertyValue;
                 foreach ($values as &$value) {
                     if ($propertyValue === 'false') {
                         $value = self::VALUE_BOOLEAN_FALSE;
                     } else {
                         $value = 1;
                     }
                 }
                 $value = join(self::MULTIVALUE_SEPARATOR, $values);
                 $document->addField(Field::Text($propertyName, $value));
                 break;
         }
     }
     $index->addDocument($document);
 }
 public function processAction()
 {
     $userEmail = $this->getAuthService()->getStorage()->read();
     if (!$userEmail) {
         $this->flashMessenger()->addErrorMessage("not authorized");
         return $this->getResponse()->setContent("not authorized");
     }
     $request = $this->getRequest();
     $form = new UploadForm();
     $uploadFile = $this->params()->fromFiles('fileupload');
     if ($request->isPost()) {
         $form->setData($request->getPost());
         if ($form->isValid()) {
             // Получение конфигурации из конфигурационных данных модуля
             $uploadPath = $this->getFileUploadLocation();
             // Сохранение выгруженного файла
             $adapter = new \Zend\File\Transfer\Adapter\Http();
             $adapter->setDestination($uploadPath);
             if ($adapter->receive($uploadFile['name'])) {
                 $userTable = $this->getServiceLocator()->get('UserTable');
                 $user = $userTable->getUserByEmail($userEmail);
                 $upload = new \Users\Model\Upload();
                 // Успешная выгрузка файла
                 $exchange_data = array();
                 $exchange_data['label'] = $request->getPost()->get('label');
                 $exchange_data['filename'] = $uploadFile['name'];
                 $exchange_data['user_id'] = $user->getId();
                 $upload->exchangeArray($exchange_data);
                 $uploadTable = $this->getServiceLocator()->get('UploadTable');
                 $uploadTable->save($upload);
                 $upload->setId($uploadTable->getLastInsertValue());
                 //добавить в Lucene
                 $searchIndexLocation = $this->getIndexLocation();
                 $index = Lucene\Lucene::create($searchIndexLocation);
                 // создание полей lucene
                 $fileUploadId = Document\Field::unIndexed('upload_id', $upload->getId());
                 $label = Document\Field::Text('label', $upload->getLabel());
                 $owner = Document\Field::Text('owner', $user->getName());
                 $uploadPath = $this->getFileUploadLocation();
                 $fileName = $upload->getFilename();
                 $filePath = $uploadPath . DIRECTORY_SEPARATOR . $fileName;
                 if (substr_compare($fileName, ".xlsx", strlen($fileName) - strlen(".xlsx"), strlen(".xlsx")) === 0) {
                     // Индексирование таблицы excel
                     $indexDoc = Lucene\Document\Xlsx::loadXlsxFile($filePath);
                 } else {
                     if (substr_compare($fileName, ".docx", strlen($fileName) - strlen(".docx"), strlen(".docx")) === 0) {
                         // Индексирование документа Word
                         $indexDoc = Lucene\Document\Docx::loadDocxFile($filePath);
                     } else {
                         $indexDoc = new Lucene\Document();
                     }
                 }
                 // создание нового документа и добавление всех полей
                 $indexDoc = new Lucene\Document();
                 $indexDoc->addField($label);
                 $indexDoc->addField($owner);
                 $indexDoc->addField($fileUploadId);
                 $index->addDocument($indexDoc);
                 $index->commit();
             }
         }
     }
     return $this->redirect()->toRoute('uploads', array('action' => 'index'));
 }
예제 #21
0
 /**
  * Create document
  *
  * @param $model
  * @param $attributes
  * @return Document
  */
 private function createDocument($model, $attributes)
 {
     $document = new Document();
     foreach ($attributes as $name => $value) {
         if (is_array($value)) {
             list($attribute, $fieldType) = [key($value), current($value)];
             $value = $this->getAttributeValue($model, $attribute);
         } else {
             $fieldType = self::FIELD_TEXT;
         }
         $document->addField(Field::$fieldType($name, strip_tags($value)));
     }
     if (!method_exists($model, 'getUrl')) {
         throw new \yii\base\InvalidValueException('The identity object must implement PageLink.');
     }
     $document->addField(Field::Text('url', $model->getUrl()));
     return $document;
 }
예제 #22
0
파일: Exam.php 프로젝트: ubc/examdb
 public function index($indexer, $commit = true, $optimize = true)
 {
     $document = new Document();
     $document->addField(Field::keyword('pk', $this->getId()));
     $document->addField(Field::Text('course', $this->getSubjectcode()));
     $document->addField(Field::Text('cross-listed', str_replace(array(';', ',', '|'), ' ', $this->getCrossListed())));
     $document->addField(Field::Text('instructor', $this->getLegalContentOwner()));
     $document->addField(Field::Unstored('comments', $this->getComments()));
     $indexer->addDocument($document);
     if ($commit) {
         $indexer->commit();
     }
     if ($optimize) {
         $indexer->optimize();
     }
 }
예제 #23
0
 public function testTermsStreamInterfaceSkipToTermsRetrievingTwoTermsCase()
 {
     $index = Lucene\Lucene::create(__DIR__ . '/_index/_files');
     // Zero terms
     $doc = new Document();
     $doc->addField(Document\Field::Text('contents', 'someterm word'));
     $index->addDocument($doc);
     unset($index);
     $index = Lucene\Lucene::open(__DIR__ . '/_index/_files');
     $index->resetTermsStream();
     $index->skipTo(new Index\Term('term', 'contents'));
     $this->assertTrue($index->currentTerm() == new Index\Term('word', 'contents'));
     $index->closeTermsStream();
 }
 /**
  * Indexa dados nos arquivos de json
  */
 public function index()
 {
     $dir = realpath(dirname(__FILE__)) . DIRECTORY_SEPARATOR . "data" . DIRECTORY_SEPARATOR;
     $jsonDir = $dir . "json";
     $indexDir = $dir . "index";
     // ler aquivos json
     $files = scandir($jsonDir);
     foreach ($files as $file) {
         if ($file == '.' || $file == '..') {
             continue;
         }
         // Se arquivo existe
         if (is_file($jsonDir . DIRECTORY_SEPARATOR . $file)) {
             $json = json_decode(file_get_contents($jsonDir . DIRECTORY_SEPARATOR . $file));
             $indexName = substr($file, 0, -5);
             // Cria index
             $index = Lucene\Lucene::create($indexDir . DIRECTORY_SEPARATOR . $indexName);
             // Cria documento e define campos para indexar
             foreach ($json as $entry) {
                 $doc = new Lucene\Document();
                 $doc->addField(Lucene\Document\Field::Text('url', $entry->title));
                 $doc->addField(Lucene\Document\Field::UnStored('contents', $entry->text));
                 $index->addDocument($doc);
             }
         }
     }
 }