/** * Generate compound index file */ protected function _generateCFS() { $cfsFile = $this->_directory->createFile($this->_name . '.cfs'); $cfsFile->writeVInt(count($this->_files)); $dataOffsetPointers = array(); foreach ($this->_files as $fileName) { $dataOffsetPointers[$fileName] = $cfsFile->tell(); $cfsFile->writeLong(0); // write dummy data $cfsFile->writeString($fileName); } foreach ($this->_files as $fileName) { // Get actual data offset $dataOffset = $cfsFile->tell(); // Seek to the data offset pointer $cfsFile->seek($dataOffsetPointers[$fileName]); // Write actual data offset value $cfsFile->writeLong($dataOffset); // Seek back to the end of file $cfsFile->seek($dataOffset); $dataFile = $this->_directory->getFileObject($fileName); $byteCount = $this->_directory->fileLength($fileName); while ($byteCount > 0) { $data = $dataFile->readBytes(min($byteCount, 131072)); $byteCount -= strlen($data); $cfsFile->writeBytes($data); } $this->_directory->deleteFile($fileName); } }
/** * Opens the index. * * IndexReader constructor needs Directory as a parameter. It should be * a string with a path to the index folder or a Directory object. * * @param mixed $directory * @throws Zend_Search_Lucene_Exception */ public function __construct($directory = null, $create = false) { if ($directory === null) { throw new Zend_Search_Exception('No index directory specified'); } if ($directory instanceof Zend_Search_Lucene_Storage_Directory_Filesystem) { $this->_directory = $directory; $this->_closeDirOnExit = false; } else { $this->_directory = new Zend_Search_Lucene_Storage_Directory_Filesystem($directory); $this->_closeDirOnExit = true; } $this->_segmentInfos = array(); // Mark index as "under processing" to prevent other processes from premature index cleaning Zend_Search_Lucene_LockManager::obtainReadLock($this->_directory); // Escalate read lock to prevent current generation index files to be deleted while opening process is not done Zend_Search_Lucene_LockManager::escalateReadLock($this->_directory); $this->_generation = self::getActualGeneration($this->_directory); if ($create) { try { Zend_Search_Lucene_LockManager::obtainWriteLock($this->_directory); } catch (Zend_Search_Lucene_Exception $e) { if (strpos($e->getMessage(), 'Can\'t obtain exclusive index lock') === false) { throw $e; } else { throw new Zend_Search_Lucene_Exception('Can\'t create index. It\'s under processing now'); } } if ($this->_generation == -1) { // Directory doesn't contain existing index, start from 1 $this->_generation = 1; $nameCounter = 0; } else { // Directory contains existing index $segmentsFile = $this->_directory->getFileObject(self::getSegmentFileName($this->_generation)); $segmentsFile->seek(12); // 12 = 4 (int, file format marker) + 8 (long, index version) $nameCounter = $segmentsFile->readInt(); $this->_generation++; } Zend_Search_Lucene_Index_Writer::createIndex($this->_directory, $this->_generation, $nameCounter); Zend_Search_Lucene_LockManager::releaseWriteLock($this->_directory); } if ($this->_generation == -1) { throw new Zend_Search_Lucene_Exception('Index doesn\'t exists in the specified directory.'); } else { if ($this->_generation == 0) { $this->_readPre21SegmentsFile(); } else { $this->_readSegmentsFile(); } } // De-escalate read lock to prevent current generation index files to be deleted while opening process is not done Zend_Search_Lucene_LockManager::deEscalateReadLock($this->_directory); }
/** * Opens the index. * * IndexReader constructor needs Directory as a parameter. It should be * a string with a path to the index folder or a Directory object. * * @param mixed $directory * @throws Zend_Search_Lucene_Exception */ public function __construct($directory = null, $create = false) { if ($directory === null) { throw new Zend_Search_Exception('No index directory specified'); } if ($directory instanceof Zend_Search_Lucene_Storage_Directory_Filesystem) { $this->_directory = $directory; $this->_closeDirOnExit = false; } else { $this->_directory = new Zend_Search_Lucene_Storage_Directory_Filesystem($directory); $this->_closeDirOnExit = true; } // Get a shared lock to the index $this->_lock = $this->_directory->createFile('index.lock'); $this->_segmentInfos = array(); if ($create) { // Throw an exception if index is under processing now if (!$this->_lock->lock(LOCK_EX, true)) { throw new Zend_Search_Lucene_Exception('Can\'t create index. It\'s under processing now'); } // Writer will create segments file for empty segments list $this->_writer = new Zend_Search_Lucene_Index_Writer($this->_directory, $this->_segmentInfos, true); if (!$this->_lock->lock(LOCK_SH)) { throw new Zend_Search_Lucene_Exception('Can\'t reduce lock level from Exclusive to Shared'); } } else { // Wait if index is under switching from one set of segments to another (Index_Writer::_updateSegments()) if (!$this->_lock->lock(LOCK_SH)) { throw new Zend_Search_Lucene_Exception('Can\'t obtain shared index lock'); } $this->_writer = null; } $segmentsFile = $this->_directory->getFileObject('segments'); $format = $segmentsFile->readInt(); if ($format != (int) 0xffffffff) { throw new Zend_Search_Lucene_Exception('Wrong segments file format'); } // read version // $segmentsFile->readLong(); $segmentsFile->readInt(); $segmentsFile->readInt(); // read segment name counter $segmentsFile->readInt(); $segments = $segmentsFile->readInt(); $this->_docCount = 0; // read segmentInfos for ($count = 0; $count < $segments; $count++) { $segName = $segmentsFile->readString(); $segSize = $segmentsFile->readInt(); $this->_docCount += $segSize; $this->_segmentInfos[] = new Zend_Search_Lucene_Index_SegmentInfo($segName, $segSize, $this->_directory); } }
public static function getActualGeneration(Zend_Search_Lucene_Storage_Directory $directory) { require_once 'Zend/Search/Lucene/Exception.php'; try { for ($count = 0; $count < self::GENERATION_RETRIEVE_COUNT; $count++) { // Try to get generation file $genFile = $directory->getFileObject('segments.gen', false); $format = $genFile->readInt(); if ($format != (int) 0xfffffffe) { throw new Zend_Search_Lucene_Exception('Wrong segments.gen file format'); } $gen1 = $genFile->readLong(); $gen2 = $genFile->readLong(); if ($gen1 == $gen2) { return $gen1; } usleep(self::GENERATION_RETRIEVE_PAUSE * 1000); } // All passes are failed throw new Zend_Search_Lucene_Exception('Index is under processing now'); } catch (Zend_Search_Lucene_Exception $e) { if (strpos($e->getMessage(), 'is not readable') !== false) { try { // Try to open old style segments file $segmentsFile = $directory->getFileObject('segments', false); // It's pre-2.1 index return 0; } catch (Zend_Search_Lucene_Exception $e) { if (strpos($e->getMessage(), 'is not readable') !== false) { return -1; } else { throw $e; } } } else { throw $e; } } return -1; }
/** * Update segments file by adding current segment to a list * @todo !!!!!Finish the implementation * * @throws Zend_Search_Lucene_Exception */ private function _updateSegments() { $segmentsFile = $this->_directory->getFileObject('segments'); $newSegmentFile = $this->_directory->createFile('segments.new'); $newSegmentFile->writeInt((int) 0xffffffff); $newSegmentFile->writeLong($this->_version); $newSegmentFile->writeInt($this->_segmentNameCounter); $newSegmentFile->writeInt($this->_segments + count($this->_newSegments)); $segmentsFile->seek(20); $newSegmentFile->writeBytes($segmentsFile->readBytes($this->_directory->fileLength('segments') - 20)); foreach ($this->_newSegments as $segmentName => $segmentInfo) { $newSegmentFile->writeString($segmentName); $newSegmentFile->writeInt($segmentInfo->count()); } $this->_directory->renameFile('segments.new', 'segments'); }
/** * Get name for new segment * * @return string */ private function _newSegmentName() { Zend_Search_Lucene_LockManager::obtainWriteLock($this->_directory); $generation = Zend_Search_Lucene::getActualGeneration($this->_directory); $segmentsFile = $this->_directory->getFileObject(Zend_Search_Lucene::getSegmentFileName($generation), false); $segmentsFile->seek(12); // 12 = 4 (int, file format marker) + 8 (long, index version) $segmentNameCounter = $segmentsFile->readInt(); $segmentsFile->seek(12); // 12 = 4 (int, file format marker) + 8 (long, index version) $segmentsFile->writeInt($segmentNameCounter + 1); // Flash output to guarantee that wrong value will not be loaded between unlock and // return (which calls $segmentsFile destructor) $segmentsFile->flush(); Zend_Search_Lucene_LockManager::releaseWriteLock($this->_directory); return '_' . base_convert($segmentNameCounter, 10, 36); }
/** * Opens the index. * * IndexReader constructor needs Directory as a parameter. It should be * a string with a path to the index folder or a Directory object. * * @param mixed $directory * @throws Zend_Search_Lucene_Exception */ public function __construct($directory = null, $create = false) { if ($directory === null) { throw new Zend_Search_Exception('No index directory specified'); } if ($directory instanceof Zend_Search_Lucene_Storage_Directory_Filesystem) { $this->_directory = $directory; $this->_closeDirOnExit = false; } else { $this->_directory = new Zend_Search_Lucene_Storage_Directory_Filesystem($directory); $this->_closeDirOnExit = true; } if ($create) { $this->_writer = new Zend_Search_Lucene_Index_Writer($this->_directory, true); } else { $this->_writer = null; } $this->_segmentInfos = array(); $segmentsFile = $this->_directory->getFileObject('segments'); $format = $segmentsFile->readInt(); if ($format != (int) 0xffffffff) { throw new Zend_Search_Lucene_Exception('Wrong segments file format'); } // read version $segmentsFile->readLong(); // read counter $segmentsFile->readInt(); $segments = $segmentsFile->readInt(); $this->_docCount = 0; // read segmentInfos for ($count = 0; $count < $segments; $count++) { $segName = $segmentsFile->readString(); $segSize = $segmentsFile->readInt(); $this->_docCount += $segSize; $this->_segmentInfos[$count] = new Zend_Search_Lucene_Index_SegmentInfo($segName, $segSize, $this->_directory); } }
/** * Release exclusive optimization lock * * @param Zend_Search_Lucene_Storage_Directory $lockDirectory */ public static function releaseOptimizationLock(Zend_Search_Lucene_Storage_Directory $lockDirectory) { $lock = $lockDirectory->getFileObject(self::OPTIMIZATION_LOCK_FILE); $lock->unlock(); }
/** * Opens the index. * * IndexReader constructor needs Directory as a parameter. It should be * a string with a path to the index folder or a Directory object. * * @param Zend_Search_Lucene_Storage_Directory_Filesystem|string $directory * @throws Zend_Search_Lucene_Exception */ public function __construct($directory = null, $create = false) { if ($directory === null) { require_once LIB_DIR . '/Zend/Search/Lucene/Exception.php'; throw new Zend_Search_Exception('No index directory specified'); } if (is_string($directory)) { require_once LIB_DIR . '/Zend/Search/Lucene/Storage/Directory/Filesystem.php'; $this->_directory = new Zend_Search_Lucene_Storage_Directory_Filesystem($directory); $this->_closeDirOnExit = true; } else { $this->_directory = $directory; $this->_closeDirOnExit = false; } $this->_segmentInfos = array(); // Mark index as "under processing" to prevent other processes from premature index cleaning Zend_Search_Lucene_LockManager::obtainReadLock($this->_directory); $this->_generation = self::getActualGeneration($this->_directory); if ($create) { require_once LIB_DIR . '/Zend/Search/Lucene/Exception.php'; try { Zend_Search_Lucene_LockManager::obtainWriteLock($this->_directory); } catch (Zend_Search_Lucene_Exception $e) { Zend_Search_Lucene_LockManager::releaseReadLock($this->_directory); if (strpos($e->getMessage(), 'Can\'t obtain exclusive index lock') === false) { throw new Zend_Search_Lucene_Exception($e->getMessage(), $e->getCode(), $e); } else { throw new Zend_Search_Lucene_Exception('Can\'t create index. It\'s under processing now', 0, $e); } } if ($this->_generation == -1) { // Directory doesn't contain existing index, start from 1 $this->_generation = 1; $nameCounter = 0; } else { // Directory contains existing index $segmentsFile = $this->_directory->getFileObject(self::getSegmentFileName($this->_generation)); $segmentsFile->seek(12); // 12 = 4 (int, file format marker) + 8 (long, index version) $nameCounter = $segmentsFile->readInt(); $this->_generation++; } require_once LIB_DIR . '/Zend/Search/Lucene/Index/Writer.php'; Zend_Search_Lucene_Index_Writer::createIndex($this->_directory, $this->_generation, $nameCounter); Zend_Search_Lucene_LockManager::releaseWriteLock($this->_directory); } if ($this->_generation == -1) { require_once LIB_DIR . '/Zend/Search/Lucene/Exception.php'; throw new Zend_Search_Lucene_Exception('Index doesn\'t exists in the specified directory.'); } else { if ($this->_generation == 0) { $this->_readPre21SegmentsFile(); } else { $this->_readSegmentsFile(); } } }
/** * Get name for new segment * * @return string */ private function _newSegmentName() { // Do not share file handler to get file updates from other sessions. $segmentsFile = $this->_directory->getFileObject('segments', false); // Get exclusive segments file lock // We have guarantee, that we will not intersect with _updateSegments() call // of other process, because it needs exclusive index lock and waits // until all other searchers won't stop if (!$segmentsFile->lock(LOCK_EX)) { throw new Zend_Search_Lucene_Exception('Can\'t obtain exclusive index lock'); } $segmentsFile->seek(12); // 12 = 4 (int, file format marker) + 8 (long, index version) $segmentNameCounter = $segmentsFile->readInt(); $segmentsFile->seek(12); // 12 = 4 (int, file format marker) + 8 (long, index version) $segmentsFile->writeInt($segmentNameCounter + 1); // Flash output to guarantee that wrong value will not be loaded between unlock and // return (which calls $segmentsFile destructor) $segmentsFile->flush(); $segmentsFile->unlock(); return '_' . base_convert($segmentNameCounter, 10, 36); }
/** * Get name for new segment * * @return string */ private function _newSegmentName() { $segmentsFile = $this->_directory->getFileObject('segments'); $segmentsFile->seek(12); // 12 = 4 (int, file format marker) + 8 (long, index version) $segmentNameCounter = $segmentsFile->readInt(); $segmentsFile->seek(12); // 12 = 4 (int, file format marker) + 8 (long, index version) $segmentsFile->writeInt($segmentNameCounter + 1); return '_' . base_convert($segmentNameCounter, 10, 36); }
/** * Release shared read lock * * @param Zend_Search_Lucene_Storage_Directory $lockDirectory */ public static function releaseReadLock(Zend_Search_Lucene_Storage_Directory $lockDirectory) { // get exclusive operation lock on read lock file. if ($lockDirectory->fileExists(self::EXCLUSIVE_READ_LOCK_FILE)) { throw new Zend_Search_Lucene_Exception('Can\'t obtain shared reading index lock : read lock operation in progress'); } else { $lock = $lockDirectory->createFile(self::EXCLUSIVE_READ_LOCK_FILE); } try { $lock = $lockDirectory->getFileObject(self::READ_LOCK_FILE, false); $accesscounter = $lock->readBytes($lock->size()); $lock->close(); $accesscounter--; $lockDirectory->deleteFile(self::READ_LOCK_FILE); if ($accesscounter > 1) { // we have'nt released all locks. Create the file again. $lock = $lockDirectory->createFile(self::READ_LOCK_FILE); $lock->writeBytes($accesscount); $lock->flush(); $lock->close(); } } catch (Zend_Search_Lucene_Exception $e) { $lockDirectory->deleteFile(self::EXCLUSIVE_READ_LOCK_FILE); // accept failover here // throw new Zend_Search_Lucene_Exception('Can\'t obtain shared reading index lock'); } $lockDirectory->deleteFile(self::EXCLUSIVE_READ_LOCK_FILE); }
public static function getActualGeneration(Zend_Search_Lucene_Storage_Directory $directory) { try { for ($count = 0; $count < self::GENERATION_RETRIEVE_COUNT; $count++) { $genFile = $directory->getFileObject('segments.gen', false); $format = $genFile->readInt(); if ($format != (int) 0xfffffffe) { throw new Zend_Search_Lucene_Exception('Wrong segments.gen file format'); } $gen1 = $genFile->readLong(); $gen2 = $genFile->readLong(); if ($gen1 == $gen2) { return $gen1; } usleep(self::GENERATION_RETRIEVE_PAUSE * 1000); } throw new Zend_Search_Lucene_Exception('Index is under processing now'); } catch (Zend_Search_Lucene_Exception $e) { if (strpos($e->getMessage(), 'is not readable') !== false) { try { $segmentsFile = $directory->getFileObject('segments', false); return 0; } catch (Zend_Search_Lucene_Exception $e) { if (strpos($e->getMessage(), 'is not readable') !== false) { return -1; } else { throw new Zend_Search_Lucene_Exception($e->getMessage(), $e->getCode(), $e); } } } else { throw new Zend_Search_Lucene_Exception($e->getMessage(), $e->getCode(), $e); } } return -1; }