protected function getDropFolderFilesFromPhysicalFolder()
{
if ($this->fileTransferMgr->fileExists($this->dropFolder->path)) {
$physicalFiles = $this->fileTransferMgr->listFileObjects($this->dropFolder->path);
if ($physicalFiles) {
KalturaLog::log('Found [' . count($physicalFiles) . '] in the folder');
} else {
KalturaLog::info('No physical files found for drop folder id [' . $this->dropFolder->id . '] with path [' . $this->dropFolder->path . ']');
$physicalFiles = array();
}
} else {
throw new kFileTransferMgrException('Drop folder path not valid [' . $this->dropFolder->path . ']', kFileTransferMgrException::remotePathNotValid);
}
KalturaLog::info("physical files: ");
foreach ($physicalFiles as &$currlFile) {
KalturaLog::info(print_r($currlFile, true));
}
return $physicalFiles;
}
/**
* Tests kFileTransferMgr->fileExists()
*/
public function testFileExists()
{
// TODO Auto-generated kFileTransferMgrTest->testFileExists()
$this->markTestIncomplete("fileExists test not implemented");
$this->kFileTransferMgr->fileExists();
}
/**
* Main logic function.
* Sync between the list of physical files and drop folder file object for the given drop folder ($folder).
* Add new files, update sizes and status and delete physical files when required.
* @param KalturaDropFolder $folder
*/
private function watchFolder(KalturaDropFolder $folder)
{
KalturaLog::debug('Watching folder [' . $folder->id . ']');
// if remote folder -> login to server and set fileTransferManager
try {
$this->fileTransferMgr = DropFolderBatchUtils::getFileTransferManager($folder);
} catch (Exception $e) {
$this->unimpersonate();
KalturaLog::err('Cannot initialize file transfer manager for folder [' . $folder->id . '] - ' . $e->getMessage());
return;
// skipping to next folder
}
// get list of DropFolderFile objects from the current $folder
$dropFolderFiles = null;
$deletedDropFolderFiles = null;
try {
$dropFolderFiles = $this->getDropFolderFileObjects($folder->id);
} catch (Exception $e) {
$this->unimpersonate();
KalturaLog::err('Cannot get drop folder file list from the server for drop folder id [' . $folder->id . '] - ' . $e->getMessage());
return;
// skipping to next folder
}
// get a list of physical files from the folder's path
$physicalFiles = null;
try {
$physicalFiles = $this->getPhysicalFileList($folder);
} catch (Exception $e) {
$this->unimpersonate();
$physicalFiles = null;
}
if (!$physicalFiles) {
KalturaLog::err('Cannot get physical file list for drop folder id [' . $folder->id . '] with path [' . $folder->path . ']');
return;
// skipping to next folder
}
// with local drop folder, file may have been moved (hence deleted) immidiately upon ingestion
$autoDeleteOriginalFile = $folder->fileDeletePolicy == KalturaDropFolderFileDeletePolicy::AUTO_DELETE && $folder->autoFileDeleteDays == 0;
$dropFolderFileMapByName = array();
foreach ($dropFolderFiles as $dropFolderFile) {
if ($dropFolderFile->status !== KalturaDropFolderFileStatus::PURGED) {
if (!in_array($dropFolderFile->fileName, $physicalFiles)) {
if ($autoDeleteOriginalFile) {
$this->setFileAsPurged($dropFolderFile);
} else {
if (!in_array($dropFolderFile->status, self::$dropFolderFileErrorStatuses)) {
$this->errorWithFile($dropFolderFile, KalturaDropFolderFileErrorCode::ERROR_READING_FILE, 'Cannot find file with name [' . $dropFolderFile->fileName . ']');
}
}
continue;
}
$dropFolderFileMapByName[$dropFolderFile->fileName] = $dropFolderFile;
}
}
// get defined file name patterns
$ignorePatterns = $folder->ignoreFileNamePatterns;
$ignorePatterns = array_map('trim', explode(',', $ignorePatterns));
// sync between physical file list and drop folder file objects
foreach ($physicalFiles as $physicalFileName) {
try {
if (empty($physicalFileName) || $physicalFileName === '.' || $physicalFileName === '..') {
continue;
}
KalturaLog::debug("Watch file [{$physicalFileName}]");
$shouldIgnore = false;
foreach ($ignorePatterns as $ignorePattern) {
if (!is_null($ignorePattern) && $ignorePattern != '') {
if (fnmatch($ignorePattern, $physicalFileName)) {
$shouldIgnore = true;
KalturaLog::err("Ignoring file [{$physicalFileName}] matching ignore pattern [{$ignorePattern}]");
break;
}
}
}
if ($shouldIgnore) {
continue;
}
// translate file name to path+name on the shared location
$fullPath = $folder->path . '/' . $physicalFileName;
// skip non-accessible files
if (!$fullPath || !$this->fileTransferMgr->fileExists($fullPath)) {
KalturaLog::err("Cannot access physical file in path [{$fullPath}]");
continue;
}
// skip directories
/*
if (is_dir($fullPath)) {
KalturaLog::log("Path [$fullPath] is a directory - skipped");
continue;
}
*/
// check if file is already in the list of drop folder files
if (!array_key_exists($physicalFileName, $dropFolderFileMapByName)) {
// new physical file found in folder - add new drop folder file object with status UPLOADING
$this->addNewDropFolderFile($folder->id, $physicalFileName, $fullPath);
} else {
$currentDropFolderFile = $dropFolderFileMapByName[$physicalFileName];
try {
$lastModificationTime = $this->getModificationTime($fullPath);
} catch (Exception $e) {
$this->unimpersonate();
KalturaLog::err('Cannot get modification time for file in path [' . $fullPath . '] - ' . $e->getMessage());
continue;
// skipping to next file
}
$knownLastModificationTime = $currentDropFolderFile->lastModificationTime;
if ($knownLastModificationTime && $lastModificationTime > $knownLastModificationTime && $currentDropFolderFile->status != KalturaDropFolderFileStatus::UPLOADING) {
// file has been replaced by a new file with the same name
$this->setFileAsPurged($currentDropFolderFile);
$this->addNewDropFolderFile($folder->id, $physicalFileName, $fullPath, $lastModificationTime);
continue;
// continue to next file
}
// update existing drop folder file object according to current physical file
if ($currentDropFolderFile->status == KalturaDropFolderFileStatus::UPLOADING) {
$this->updateDropFolderFile($folder, $currentDropFolderFile, $fullPath, $lastModificationTime);
} else {
if ($currentDropFolderFile->status == KalturaDropFolderFileStatus::HANDLED) {
$this->purgeHandledFileIfNeeded($folder, $currentDropFolderFile, $fullPath);
} else {
if ($currentDropFolderFile->status == KalturaDropFolderFileStatus::DELETED) {
// purge file marked as deleted
$this->purgeFile($dropFolderFileMapByName[$physicalFileName], $fullPath);
continue;
}
}
}
}
} catch (Exception $e) {
$this->unimpersonate();
KalturaLog::err("Error handling drop folder file [{$physicalFileName}] " . $e->getMessage());
}
}
}
