/**
* Main logic function.
* Sync between the list of physical files and drop folder file object for the given drop folder ($folder).
* Add new files, update sizes and status and delete physical files when required.
* @param KalturaDropFolder $folder
*/
private function watchFolder(KalturaDropFolder $folder)
{
KalturaLog::debug('Watching folder [' . $folder->id . ']');
// if remote folder -> login to server and set fileTransferManager
try {
$this->fileTransferMgr = DropFolderBatchUtils::getFileTransferManager($folder);
} catch (Exception $e) {
$this->unimpersonate();
KalturaLog::err('Cannot initialize file transfer manager for folder [' . $folder->id . '] - ' . $e->getMessage());
return;
// skipping to next folder
}
// get list of DropFolderFile objects from the current $folder
$dropFolderFiles = null;
$deletedDropFolderFiles = null;
try {
$dropFolderFiles = $this->getDropFolderFileObjects($folder->id);
} catch (Exception $e) {
$this->unimpersonate();
KalturaLog::err('Cannot get drop folder file list from the server for drop folder id [' . $folder->id . '] - ' . $e->getMessage());
return;
// skipping to next folder
}
// get a list of physical files from the folder's path
$physicalFiles = null;
try {
$physicalFiles = $this->getPhysicalFileList($folder);
} catch (Exception $e) {
$this->unimpersonate();
$physicalFiles = null;
}
if (!$physicalFiles) {
KalturaLog::err('Cannot get physical file list for drop folder id [' . $folder->id . '] with path [' . $folder->path . ']');
return;
// skipping to next folder
}
// with local drop folder, file may have been moved (hence deleted) immidiately upon ingestion
$autoDeleteOriginalFile = $folder->fileDeletePolicy == KalturaDropFolderFileDeletePolicy::AUTO_DELETE && $folder->autoFileDeleteDays == 0;
$dropFolderFileMapByName = array();
foreach ($dropFolderFiles as $dropFolderFile) {
if ($dropFolderFile->status !== KalturaDropFolderFileStatus::PURGED) {
if (!in_array($dropFolderFile->fileName, $physicalFiles)) {
if ($autoDeleteOriginalFile) {
$this->setFileAsPurged($dropFolderFile);
} else {
if (!in_array($dropFolderFile->status, self::$dropFolderFileErrorStatuses)) {
$this->errorWithFile($dropFolderFile, KalturaDropFolderFileErrorCode::ERROR_READING_FILE, 'Cannot find file with name [' . $dropFolderFile->fileName . ']');
}
}
continue;
}
$dropFolderFileMapByName[$dropFolderFile->fileName] = $dropFolderFile;
}
}
// get defined file name patterns
$ignorePatterns = $folder->ignoreFileNamePatterns;
$ignorePatterns = array_map('trim', explode(',', $ignorePatterns));
// sync between physical file list and drop folder file objects
foreach ($physicalFiles as $physicalFileName) {
try {
if (empty($physicalFileName) || $physicalFileName === '.' || $physicalFileName === '..') {
continue;
}
KalturaLog::debug("Watch file [{$physicalFileName}]");
$shouldIgnore = false;
foreach ($ignorePatterns as $ignorePattern) {
if (!is_null($ignorePattern) && $ignorePattern != '') {
if (fnmatch($ignorePattern, $physicalFileName)) {
$shouldIgnore = true;
KalturaLog::err("Ignoring file [{$physicalFileName}] matching ignore pattern [{$ignorePattern}]");
break;
}
}
}
if ($shouldIgnore) {
continue;
}
// translate file name to path+name on the shared location
$fullPath = $folder->path . '/' . $physicalFileName;
// skip non-accessible files
if (!$fullPath || !$this->fileTransferMgr->fileExists($fullPath)) {
KalturaLog::err("Cannot access physical file in path [{$fullPath}]");
continue;
}
// skip directories
/*
if (is_dir($fullPath)) {
KalturaLog::log("Path [$fullPath] is a directory - skipped");
continue;
}
*/
// check if file is already in the list of drop folder files
if (!array_key_exists($physicalFileName, $dropFolderFileMapByName)) {
// new physical file found in folder - add new drop folder file object with status UPLOADING
$this->addNewDropFolderFile($folder->id, $physicalFileName, $fullPath);
} else {
$currentDropFolderFile = $dropFolderFileMapByName[$physicalFileName];
try {
$lastModificationTime = $this->getModificationTime($fullPath);
} catch (Exception $e) {
$this->unimpersonate();
KalturaLog::err('Cannot get modification time for file in path [' . $fullPath . '] - ' . $e->getMessage());
continue;
// skipping to next file
}
$knownLastModificationTime = $currentDropFolderFile->lastModificationTime;
if ($knownLastModificationTime && $lastModificationTime > $knownLastModificationTime && $currentDropFolderFile->status != KalturaDropFolderFileStatus::UPLOADING) {
// file has been replaced by a new file with the same name
$this->setFileAsPurged($currentDropFolderFile);
$this->addNewDropFolderFile($folder->id, $physicalFileName, $fullPath, $lastModificationTime);
continue;
// continue to next file
}
// update existing drop folder file object according to current physical file
if ($currentDropFolderFile->status == KalturaDropFolderFileStatus::UPLOADING) {
$this->updateDropFolderFile($folder, $currentDropFolderFile, $fullPath, $lastModificationTime);
} else {
if ($currentDropFolderFile->status == KalturaDropFolderFileStatus::HANDLED) {
$this->purgeHandledFileIfNeeded($folder, $currentDropFolderFile, $fullPath);
} else {
if ($currentDropFolderFile->status == KalturaDropFolderFileStatus::DELETED) {
// purge file marked as deleted
$this->purgeFile($dropFolderFileMapByName[$physicalFileName], $fullPath);
continue;
}
}
}
}
} catch (Exception $e) {
$this->unimpersonate();
KalturaLog::err("Error handling drop folder file [{$physicalFileName}] " . $e->getMessage());
}
}
}
public function handle()
{
$this->tempDirectory = sys_get_temp_dir();
if (!is_dir($this->tempDirectory)) {
KalturaLog::err('Missing temporary directory');
return false;
}
// check prerequisites
$checkConfig = $this->checkConfig();
if (!$checkConfig) {
KalturaLog::err('Missing required configurations');
return false;
}
$this->fileTransferMgr = DropFolderBatchUtils::getFileTransferManager($this->dropFolder);
if (!$this->fileTransferMgr) {
$this->dropFolderFile->status = KalturaDropFolderFileStatus::ERROR_HANDLING;
$this->dropFolderFile->errorCode = KalturaDropFolderFileErrorCode::INTERNAL_ERROR;
$this->dropFolderFile->errorDescription = 'Internal server error - cannot initiate file transfer manager';
KalturaLog::err($this->dropFolderFile->errorDescription);
$this->updateDropFolderFile();
return false;
}
$xmlPath = $this->getLocalXmlFilePath();
if (!$xmlPath) {
$this->dropFolderFile->status = KalturaDropFolderFileStatus::ERROR_HANDLING;
$this->dropFolderFile->errorCode = KalturaDropFolderFileErrorCode::ERROR_READING_FILE;
$this->dropFolderFile->errorDescription = 'Cannot read file at path [' . $this->dropFolder->path . '/' . $this->dropFolderFile->fileName . ']';
KalturaLog::err($this->dropFolderFile->errorDescription);
$this->updateDropFolderFile();
return false;
}
$xmlDoc = new KDOMDocument();
$xmlDoc->load($xmlPath);
if (!$xmlDoc) {
$this->dropFolderFile->status = KalturaDropFolderFileStatus::ERROR_HANDLING;
$this->dropFolderFile->errorCode = KalturaDropFolderFileErrorCode::ERROR_READING_FILE;
$this->dropFolderFile->errorDescription = "Cannot parse XML file at [{$xmlPath}]";
KalturaLog::err($this->dropFolderFile->errorDescription);
$this->updateDropFolderFile();
return false;
}
$localResources = $xmlDoc->getElementsByTagName(self::DROP_FOLDER_RESOURCE_NODE_NAME);
if (!$localResources) {
$this->dropFolderFile->status = KalturaDropFolderFileStatus::ERROR_HANDLING;
$this->dropFolderFile->errorCode = KalturaDropFolderFileErrorCode::ERROR_READING_FILE;
$this->dropFolderFile->errorDescription = "Cannot parse XML file at [{$xmlPath}]";
KalturaLog::err($this->dropFolderFile->errorDescription);
$this->updateDropFolderFile();
return false;
}
$replaceResources = array();
$localResourcesLength = $localResources->length;
foreach ($localResources as $local) {
// already have drop folder file id
if (!is_null($this->getDropFolderFileId($local))) {
continue;
}
// replacement/modification of $local must not happen inside this foreach loop
$dropFolderFileId = $this->checkFileExists($local);
if (is_null($dropFolderFileId)) {
KalturaLog::debug('Some required files do not exist in the drop folder - changing status to WAITING');
$this->dropFolderFile->status = KalturaDropFolderFileStatus::WAITING;
KalturaLog::debug('Changing status to WAITING');
$this->updateDropFolderFile();
return false;
}
$localVerified = $this->verifyLocalResource($local);
if (!$localVerified) {
$this->dropFolderFile->status = KalturaDropFolderFileStatus::ERROR_HANDLING;
// error code and description already set
KalturaLog::err($this->dropFolderFile->errorDescription);
$this->updateDropFolderFile();
return false;
}
$replaceResources[] = array($local, $dropFolderFileId);
}
foreach ($replaceResources as $replace) {
$this->replaceResource($replace[0], $replace[1], $xmlDoc);
}
// create a temporary XML file from the modified $xmlDoc
$tempFile = $this->tempDirectory . DIRECTORY_SEPARATOR . uniqid() . '_' . $this->dropFolderFile->fileName;
$xmlDoc->save($tempFile);
$tempFileRealPath = realpath($tempFile);
if (!$tempFileRealPath || !is_file($tempFileRealPath)) {
$this->dropFolderFile->status = KalturaDropFolderFileStatus::ERROR_HANDLING;
$this->dropFolderFile->errorCode = KalturaDropFolderFileErrorCode::ERROR_WRITING_TEMP_FILE;
$this->dropFolderFile->errorDescription = "Error writing temporary file [{$tempFileRealPath}]";
KalturaLog::err($this->dropFolderFile->errorDescription);
$this->updateDropFolderFile();
return false;
}
$conversionProfile = $this->getConversionProfile();
// add bulk upload of type KalturaBulkUploadType::DROP_FOLDER_XML
try {
$this->impersonate($this->dropFolderFile->partnerId);
$this->kClient->bulkUpload->add($conversionProfile->id, $tempFileRealPath, KalturaBulkUploadType::DROP_FOLDER_XML, $this->uploadedBy, $this->dropFolderFile->fileName);
$this->unimpersonate();
} catch (Exception $e) {
$this->unimpersonate();
$this->dropFolderFile->status = KalturaDropFolderFileStatus::ERROR_HANDLING;
$this->dropFolderFile->errorCode = KalturaDropFolderFileErrorCode::ERROR_ADDING_BULK_UPLOAD;
$this->dropFolderFile->errorDescription = 'Error adding bulk upload - ' . $e->getMessage();
KalturaLog::err($this->dropFolderFile->errorDescription);
$this->updateDropFolderFile();
KalturaLog::err($this->dropFolderFile->errorDescription);
return false;
}
//delete the temporary file
@unlink($tempFileRealPath);
$dropFolderFilePlugin = KalturaDropFolderClientPlugin::get($this->kClient);
$dropFolderFilePlugin->dropFolderFile->updateStatus($this->dropFolderFile->id, KalturaDropFolderFileStatus::HANDLED);
KalturaLog::debug('Drop folder file [' . $this->dropFolderFile->id . '] handled successfully');
return true;
// file handled
}