/**
* Generate compound index file
*/
protected function _generateCFS()
{
$cfsFile = $this->_directory->createFile($this->_name . '.cfs');
$cfsFile->writeVInt(count($this->_files));
$dataOffsetPointers = array();
foreach ($this->_files as $fileName) {
$dataOffsetPointers[$fileName] = $cfsFile->tell();
$cfsFile->writeLong(0);
// write dummy data
$cfsFile->writeString($fileName);
}
foreach ($this->_files as $fileName) {
// Get actual data offset
$dataOffset = $cfsFile->tell();
// Seek to the data offset pointer
$cfsFile->seek($dataOffsetPointers[$fileName]);
// Write actual data offset value
$cfsFile->writeLong($dataOffset);
// Seek back to the end of file
$cfsFile->seek($dataOffset);
$dataFile = $this->_directory->getFileObject($fileName);
$byteCount = $this->_directory->fileLength($fileName);
while ($byteCount > 0) {
$data = $dataFile->readBytes(min($byteCount, 131072));
$byteCount -= strlen($data);
$cfsFile->writeBytes($data);
}
$this->_directory->deleteFile($fileName);
}
}
/**
* Opens the index.
*
* IndexReader constructor needs Directory as a parameter. It should be
* a string with a path to the index folder or a Directory object.
*
* @param mixed $directory
* @throws Zend_Search_Lucene_Exception
*/
public function __construct($directory = null, $create = false)
{
if ($directory === null) {
throw new Zend_Search_Exception('No index directory specified');
}
if ($directory instanceof Zend_Search_Lucene_Storage_Directory_Filesystem) {
$this->_directory = $directory;
$this->_closeDirOnExit = false;
} else {
$this->_directory = new Zend_Search_Lucene_Storage_Directory_Filesystem($directory);
$this->_closeDirOnExit = true;
}
$this->_segmentInfos = array();
// Mark index as "under processing" to prevent other processes from premature index cleaning
Zend_Search_Lucene_LockManager::obtainReadLock($this->_directory);
// Escalate read lock to prevent current generation index files to be deleted while opening process is not done
Zend_Search_Lucene_LockManager::escalateReadLock($this->_directory);
$this->_generation = self::getActualGeneration($this->_directory);
if ($create) {
try {
Zend_Search_Lucene_LockManager::obtainWriteLock($this->_directory);
} catch (Zend_Search_Lucene_Exception $e) {
if (strpos($e->getMessage(), 'Can\'t obtain exclusive index lock') === false) {
throw $e;
} else {
throw new Zend_Search_Lucene_Exception('Can\'t create index. It\'s under processing now');
}
}
if ($this->_generation == -1) {
// Directory doesn't contain existing index, start from 1
$this->_generation = 1;
$nameCounter = 0;
} else {
// Directory contains existing index
$segmentsFile = $this->_directory->getFileObject(self::getSegmentFileName($this->_generation));
$segmentsFile->seek(12);
// 12 = 4 (int, file format marker) + 8 (long, index version)
$nameCounter = $segmentsFile->readInt();
$this->_generation++;
}
Zend_Search_Lucene_Index_Writer::createIndex($this->_directory, $this->_generation, $nameCounter);
Zend_Search_Lucene_LockManager::releaseWriteLock($this->_directory);
}
if ($this->_generation == -1) {
throw new Zend_Search_Lucene_Exception('Index doesn\'t exists in the specified directory.');
} else {
if ($this->_generation == 0) {
$this->_readPre21SegmentsFile();
} else {
$this->_readSegmentsFile();
}
}
// De-escalate read lock to prevent current generation index files to be deleted while opening process is not done
Zend_Search_Lucene_LockManager::deEscalateReadLock($this->_directory);
}
/**
* Opens the index.
*
* IndexReader constructor needs Directory as a parameter. It should be
* a string with a path to the index folder or a Directory object.
*
* @param mixed $directory
* @throws Zend_Search_Lucene_Exception
*/
public function __construct($directory = null, $create = false)
{
if ($directory === null) {
throw new Zend_Search_Exception('No index directory specified');
}
if ($directory instanceof Zend_Search_Lucene_Storage_Directory_Filesystem) {
$this->_directory = $directory;
$this->_closeDirOnExit = false;
} else {
$this->_directory = new Zend_Search_Lucene_Storage_Directory_Filesystem($directory);
$this->_closeDirOnExit = true;
}
// Get a shared lock to the index
$this->_lock = $this->_directory->createFile('index.lock');
$this->_segmentInfos = array();
if ($create) {
// Throw an exception if index is under processing now
if (!$this->_lock->lock(LOCK_EX, true)) {
throw new Zend_Search_Lucene_Exception('Can\'t create index. It\'s under processing now');
}
// Writer will create segments file for empty segments list
$this->_writer = new Zend_Search_Lucene_Index_Writer($this->_directory, $this->_segmentInfos, true);
if (!$this->_lock->lock(LOCK_SH)) {
throw new Zend_Search_Lucene_Exception('Can\'t reduce lock level from Exclusive to Shared');
}
} else {
// Wait if index is under switching from one set of segments to another (Index_Writer::_updateSegments())
if (!$this->_lock->lock(LOCK_SH)) {
throw new Zend_Search_Lucene_Exception('Can\'t obtain shared index lock');
}
$this->_writer = null;
}
$segmentsFile = $this->_directory->getFileObject('segments');
$format = $segmentsFile->readInt();
if ($format != (int) 0xffffffff) {
throw new Zend_Search_Lucene_Exception('Wrong segments file format');
}
// read version
// $segmentsFile->readLong();
$segmentsFile->readInt();
$segmentsFile->readInt();
// read segment name counter
$segmentsFile->readInt();
$segments = $segmentsFile->readInt();
$this->_docCount = 0;
// read segmentInfos
for ($count = 0; $count < $segments; $count++) {
$segName = $segmentsFile->readString();
$segSize = $segmentsFile->readInt();
$this->_docCount += $segSize;
$this->_segmentInfos[] = new Zend_Search_Lucene_Index_SegmentInfo($segName, $segSize, $this->_directory);
}
}
public static function getActualGeneration(Zend_Search_Lucene_Storage_Directory $directory)
{
require_once 'Zend/Search/Lucene/Exception.php';
try {
for ($count = 0; $count < self::GENERATION_RETRIEVE_COUNT; $count++) {
// Try to get generation file
$genFile = $directory->getFileObject('segments.gen', false);
$format = $genFile->readInt();
if ($format != (int) 0xfffffffe) {
throw new Zend_Search_Lucene_Exception('Wrong segments.gen file format');
}
$gen1 = $genFile->readLong();
$gen2 = $genFile->readLong();
if ($gen1 == $gen2) {
return $gen1;
}
usleep(self::GENERATION_RETRIEVE_PAUSE * 1000);
}
// All passes are failed
throw new Zend_Search_Lucene_Exception('Index is under processing now');
} catch (Zend_Search_Lucene_Exception $e) {
if (strpos($e->getMessage(), 'is not readable') !== false) {
try {
// Try to open old style segments file
$segmentsFile = $directory->getFileObject('segments', false);
// It's pre-2.1 index
return 0;
} catch (Zend_Search_Lucene_Exception $e) {
if (strpos($e->getMessage(), 'is not readable') !== false) {
return -1;
} else {
throw $e;
}
}
} else {
throw $e;
}
}
return -1;
}
/**
* Update segments file by adding current segment to a list
* @todo !!!!!Finish the implementation
*
* @throws Zend_Search_Lucene_Exception
*/
private function _updateSegments()
{
$segmentsFile = $this->_directory->getFileObject('segments');
$newSegmentFile = $this->_directory->createFile('segments.new');
$newSegmentFile->writeInt((int) 0xffffffff);
$newSegmentFile->writeLong($this->_version);
$newSegmentFile->writeInt($this->_segmentNameCounter);
$newSegmentFile->writeInt($this->_segments + count($this->_newSegments));
$segmentsFile->seek(20);
$newSegmentFile->writeBytes($segmentsFile->readBytes($this->_directory->fileLength('segments') - 20));
foreach ($this->_newSegments as $segmentName => $segmentInfo) {
$newSegmentFile->writeString($segmentName);
$newSegmentFile->writeInt($segmentInfo->count());
}
$this->_directory->renameFile('segments.new', 'segments');
}
/**
* Get name for new segment
*
* @return string
*/
private function _newSegmentName()
{
Zend_Search_Lucene_LockManager::obtainWriteLock($this->_directory);
$generation = Zend_Search_Lucene::getActualGeneration($this->_directory);
$segmentsFile = $this->_directory->getFileObject(Zend_Search_Lucene::getSegmentFileName($generation), false);
$segmentsFile->seek(12);
// 12 = 4 (int, file format marker) + 8 (long, index version)
$segmentNameCounter = $segmentsFile->readInt();
$segmentsFile->seek(12);
// 12 = 4 (int, file format marker) + 8 (long, index version)
$segmentsFile->writeInt($segmentNameCounter + 1);
// Flash output to guarantee that wrong value will not be loaded between unlock and
// return (which calls $segmentsFile destructor)
$segmentsFile->flush();
Zend_Search_Lucene_LockManager::releaseWriteLock($this->_directory);
return '_' . base_convert($segmentNameCounter, 10, 36);
}
/**
* Opens the index.
*
* IndexReader constructor needs Directory as a parameter. It should be
* a string with a path to the index folder or a Directory object.
*
* @param mixed $directory
* @throws Zend_Search_Lucene_Exception
*/
public function __construct($directory = null, $create = false)
{
if ($directory === null) {
throw new Zend_Search_Exception('No index directory specified');
}
if ($directory instanceof Zend_Search_Lucene_Storage_Directory_Filesystem) {
$this->_directory = $directory;
$this->_closeDirOnExit = false;
} else {
$this->_directory = new Zend_Search_Lucene_Storage_Directory_Filesystem($directory);
$this->_closeDirOnExit = true;
}
if ($create) {
$this->_writer = new Zend_Search_Lucene_Index_Writer($this->_directory, true);
} else {
$this->_writer = null;
}
$this->_segmentInfos = array();
$segmentsFile = $this->_directory->getFileObject('segments');
$format = $segmentsFile->readInt();
if ($format != (int) 0xffffffff) {
throw new Zend_Search_Lucene_Exception('Wrong segments file format');
}
// read version
$segmentsFile->readLong();
// read counter
$segmentsFile->readInt();
$segments = $segmentsFile->readInt();
$this->_docCount = 0;
// read segmentInfos
for ($count = 0; $count < $segments; $count++) {
$segName = $segmentsFile->readString();
$segSize = $segmentsFile->readInt();
$this->_docCount += $segSize;
$this->_segmentInfos[$count] = new Zend_Search_Lucene_Index_SegmentInfo($segName, $segSize, $this->_directory);
}
}
/**
* Release exclusive optimization lock
*
* @param Zend_Search_Lucene_Storage_Directory $lockDirectory
*/
public static function releaseOptimizationLock(Zend_Search_Lucene_Storage_Directory $lockDirectory)
{
$lock = $lockDirectory->getFileObject(self::OPTIMIZATION_LOCK_FILE);
$lock->unlock();
}
/**
* Opens the index.
*
* IndexReader constructor needs Directory as a parameter. It should be
* a string with a path to the index folder or a Directory object.
*
* @param Zend_Search_Lucene_Storage_Directory_Filesystem|string $directory
* @throws Zend_Search_Lucene_Exception
*/
public function __construct($directory = null, $create = false)
{
if ($directory === null) {
require_once LIB_DIR . '/Zend/Search/Lucene/Exception.php';
throw new Zend_Search_Exception('No index directory specified');
}
if (is_string($directory)) {
require_once LIB_DIR . '/Zend/Search/Lucene/Storage/Directory/Filesystem.php';
$this->_directory = new Zend_Search_Lucene_Storage_Directory_Filesystem($directory);
$this->_closeDirOnExit = true;
} else {
$this->_directory = $directory;
$this->_closeDirOnExit = false;
}
$this->_segmentInfos = array();
// Mark index as "under processing" to prevent other processes from premature index cleaning
Zend_Search_Lucene_LockManager::obtainReadLock($this->_directory);
$this->_generation = self::getActualGeneration($this->_directory);
if ($create) {
require_once LIB_DIR . '/Zend/Search/Lucene/Exception.php';
try {
Zend_Search_Lucene_LockManager::obtainWriteLock($this->_directory);
} catch (Zend_Search_Lucene_Exception $e) {
Zend_Search_Lucene_LockManager::releaseReadLock($this->_directory);
if (strpos($e->getMessage(), 'Can\'t obtain exclusive index lock') === false) {
throw new Zend_Search_Lucene_Exception($e->getMessage(), $e->getCode(), $e);
} else {
throw new Zend_Search_Lucene_Exception('Can\'t create index. It\'s under processing now', 0, $e);
}
}
if ($this->_generation == -1) {
// Directory doesn't contain existing index, start from 1
$this->_generation = 1;
$nameCounter = 0;
} else {
// Directory contains existing index
$segmentsFile = $this->_directory->getFileObject(self::getSegmentFileName($this->_generation));
$segmentsFile->seek(12);
// 12 = 4 (int, file format marker) + 8 (long, index version)
$nameCounter = $segmentsFile->readInt();
$this->_generation++;
}
require_once LIB_DIR . '/Zend/Search/Lucene/Index/Writer.php';
Zend_Search_Lucene_Index_Writer::createIndex($this->_directory, $this->_generation, $nameCounter);
Zend_Search_Lucene_LockManager::releaseWriteLock($this->_directory);
}
if ($this->_generation == -1) {
require_once LIB_DIR . '/Zend/Search/Lucene/Exception.php';
throw new Zend_Search_Lucene_Exception('Index doesn\'t exists in the specified directory.');
} else {
if ($this->_generation == 0) {
$this->_readPre21SegmentsFile();
} else {
$this->_readSegmentsFile();
}
}
}
/**
* Get name for new segment
*
* @return string
*/
private function _newSegmentName()
{
// Do not share file handler to get file updates from other sessions.
$segmentsFile = $this->_directory->getFileObject('segments', false);
// Get exclusive segments file lock
// We have guarantee, that we will not intersect with _updateSegments() call
// of other process, because it needs exclusive index lock and waits
// until all other searchers won't stop
if (!$segmentsFile->lock(LOCK_EX)) {
throw new Zend_Search_Lucene_Exception('Can\'t obtain exclusive index lock');
}
$segmentsFile->seek(12);
// 12 = 4 (int, file format marker) + 8 (long, index version)
$segmentNameCounter = $segmentsFile->readInt();
$segmentsFile->seek(12);
// 12 = 4 (int, file format marker) + 8 (long, index version)
$segmentsFile->writeInt($segmentNameCounter + 1);
// Flash output to guarantee that wrong value will not be loaded between unlock and
// return (which calls $segmentsFile destructor)
$segmentsFile->flush();
$segmentsFile->unlock();
return '_' . base_convert($segmentNameCounter, 10, 36);
}
/**
* Get name for new segment
*
* @return string
*/
private function _newSegmentName()
{
$segmentsFile = $this->_directory->getFileObject('segments');
$segmentsFile->seek(12);
// 12 = 4 (int, file format marker) + 8 (long, index version)
$segmentNameCounter = $segmentsFile->readInt();
$segmentsFile->seek(12);
// 12 = 4 (int, file format marker) + 8 (long, index version)
$segmentsFile->writeInt($segmentNameCounter + 1);
return '_' . base_convert($segmentNameCounter, 10, 36);
}
/**
* Release shared read lock
*
* @param Zend_Search_Lucene_Storage_Directory $lockDirectory
*/
public static function releaseReadLock(Zend_Search_Lucene_Storage_Directory $lockDirectory)
{
// get exclusive operation lock on read lock file.
if ($lockDirectory->fileExists(self::EXCLUSIVE_READ_LOCK_FILE)) {
throw new Zend_Search_Lucene_Exception('Can\'t obtain shared reading index lock : read lock operation in progress');
} else {
$lock = $lockDirectory->createFile(self::EXCLUSIVE_READ_LOCK_FILE);
}
try {
$lock = $lockDirectory->getFileObject(self::READ_LOCK_FILE, false);
$accesscounter = $lock->readBytes($lock->size());
$lock->close();
$accesscounter--;
$lockDirectory->deleteFile(self::READ_LOCK_FILE);
if ($accesscounter > 1) {
// we have'nt released all locks. Create the file again.
$lock = $lockDirectory->createFile(self::READ_LOCK_FILE);
$lock->writeBytes($accesscount);
$lock->flush();
$lock->close();
}
} catch (Zend_Search_Lucene_Exception $e) {
$lockDirectory->deleteFile(self::EXCLUSIVE_READ_LOCK_FILE);
// accept failover here
// throw new Zend_Search_Lucene_Exception('Can\'t obtain shared reading index lock');
}
$lockDirectory->deleteFile(self::EXCLUSIVE_READ_LOCK_FILE);
}
public static function getActualGeneration(Zend_Search_Lucene_Storage_Directory $directory)
{
try {
for ($count = 0; $count < self::GENERATION_RETRIEVE_COUNT; $count++) {
$genFile = $directory->getFileObject('segments.gen', false);
$format = $genFile->readInt();
if ($format != (int) 0xfffffffe) {
throw new Zend_Search_Lucene_Exception('Wrong segments.gen file format');
}
$gen1 = $genFile->readLong();
$gen2 = $genFile->readLong();
if ($gen1 == $gen2) {
return $gen1;
}
usleep(self::GENERATION_RETRIEVE_PAUSE * 1000);
}
throw new Zend_Search_Lucene_Exception('Index is under processing now');
} catch (Zend_Search_Lucene_Exception $e) {
if (strpos($e->getMessage(), 'is not readable') !== false) {
try {
$segmentsFile = $directory->getFileObject('segments', false);
return 0;
} catch (Zend_Search_Lucene_Exception $e) {
if (strpos($e->getMessage(), 'is not readable') !== false) {
return -1;
} else {
throw new Zend_Search_Lucene_Exception($e->getMessage(), $e->getCode(), $e);
}
}
} else {
throw new Zend_Search_Lucene_Exception($e->getMessage(), $e->getCode(), $e);
}
}
return -1;
}
