public function handleBulkUpload()
{
KBatchBase::impersonate($this->currentPartnerId);
$dropFolderPlugin = KalturaDropFolderClientPlugin::get(KBatchBase::$kClient);
$this->setContentResourceFilesMap($dropFolderPlugin);
KBatchBase::unimpersonate();
parent::handleBulkUpload();
}
public function handleBulkUpload()
{
KalturaLog::debug("Starting BulkUpload for XML drop folder file with id [" . $this->job->jobObjectId . ']');
KBatchBase::impersonate($this->currentPartnerId);
$dropFolderPlugin = KalturaDropFolderClientPlugin::get(KBatchBase::$kClient);
$this->setContentResourceFilesMap($dropFolderPlugin);
KBatchBase::unimpersonate();
parent::handleBulkUpload();
}
/**
* Main logic function.
* @param KalturaDropFolder $folder
*/
private function handleFolder(KalturaDropFolder $folder)
{
KalturaLog::debug('Handling folder [' . $folder->id . ']');
$fileNamePatterns = explode(',', $folder->fileNamePatterns);
if (count($fileNamePatterns) > 1) {
$fileNamePatterns = null;
} elseif (count($fileNamePatterns) == 1) {
foreach ($fileNamePatterns as $index => $fileNamePattern) {
$fileNamePatterns[$index] = trim($fileNamePattern, ' *');
}
$fileNamePatterns = reset($fileNamePatterns);
}
$dropFolderFilePlugin = KalturaDropFolderClientPlugin::get($this->kClient);
$dropFolderFileFilter = new KalturaDropFolderFileFilter();
$dropFolderFileFilter->dropFolderIdEqual = $folder->id;
$dropFolderFileFilter->statusIn = KalturaDropFolderFileStatus::PENDING . ',' . KalturaDropFolderFileStatus::WAITING . ',' . KalturaDropFolderFileStatus::NO_MATCH;
$dropFolderFileFilter->orderBy = KalturaDropFolderFileOrderBy::CREATED_AT_DESC;
if ($fileNamePatterns) {
$dropFolderFileFilter->fileNameLike = $fileNamePatterns;
}
$pager = new KalturaFilterPager();
$pager->pageIndex = 1;
try {
$dropFolderFiles = $dropFolderFilePlugin->dropFolderFile->listAction($dropFolderFileFilter, $pager);
/* @var $dropFolderFiles KalturaDropFolderFileListResponse */
} catch (KalturaAPIException $e) {
KalturaLog::err('Cannot get list of files for drop folder id [' . $folder->id . '] pageIndex [' . $pager->pageIndex . '] - ' . $e->getMessage());
return false;
}
while (count($dropFolderFiles->objects)) {
foreach ($dropFolderFiles->objects as $file) {
$fileHandled = $this->handleFile($folder, $file);
if ($fileHandled) {
// break loop and go to next folder, because current folder files' status might have changed
return true;
}
}
$pager->pageIndex++;
try {
$dropFolderFiles = $dropFolderFilePlugin->dropFolderFile->listAction($dropFolderFileFilter, $pager);
} catch (KalturaAPIException $e) {
KalturaLog::err('Cannot get list of files for drop folder id [' . $folder->id . '] pageIndex [' . $pager->pageIndex . ']- ' . $e->getMessage());
return false;
}
}
if ($pager->pageIndex > 1) {
return true;
} else {
return false;
}
// no file was handled
}
public function run($jobs = null)
{
KalturaLog::info("Drop folder watcher batch is running");
$this->dropFolderPlugin = KalturaDropFolderClientPlugin::get($this->kClient);
if ($this->taskConfig->isInitOnly()) {
return $this->init();
}
//TODO: use getFilter instead of taskConfig->params
// get drop folder tags to work on from configuration
$folderTags = $this->taskConfig->params->tags;
$currentDc = $this->taskConfig->params->dc;
if (strlen($folderTags) == 0) {
KalturaLog::err('Tags configuration is empty - cannot continue');
return;
}
if (strlen($currentDc) == 0) {
KalturaLog::err('DC configuration is empty - cannot continue');
return;
}
// get list of drop folders according to configuration
$filter = new KalturaDropFolderFilter();
if ($folderTags != '*') {
$filter->tagsMultiLikeOr = $folderTags;
}
$filter->dcEqual = $currentDc;
$filter->statusEqual = KalturaDropFolderStatus::ENABLED;
try {
$dropFolders = $this->dropFolderPlugin->dropFolder->listAction($filter);
} catch (Exception $e) {
$this->unimpersonate();
KalturaLog::err('Cannot get drop folder list - ' . $e->getMessage());
return;
}
$dropFolders = $dropFolders->objects;
KalturaLog::log('[' . count($dropFolders) . '] folders to watch');
foreach ($dropFolders as $folder) {
try {
$this->watchFolder($folder);
} catch (Exception $e) {
$this->unimpersonate();
KalturaLog::err('Unknown error with folder id [' . $folder->id . '] - ' . $e->getMessage());
}
}
}
public function run($jobs = null)
{
KalturaLog::info("Drop folder watcher batch is running");
$this->dropFolderPlugin = KalturaDropFolderClientPlugin::get(self::$kClient);
if (self::$taskConfig->isInitOnly()) {
return $this->init();
}
$dropFolders = $this->getDropFoldersList();
if (isset($dropFolders)) {
$dropFolders = $dropFolders->objects;
KalturaLog::log('[' . count($dropFolders) . '] folders to watch');
foreach ($dropFolders as $folder) {
/* @var $folder KalturaDropFolder */
try {
$this->impersonate($folder->partnerId);
$engine = KDropFolderEngine::getInstance($folder->type);
$engine->watchFolder($folder);
$this->setDropFolderOK($folder);
$this->unimpersonate();
} catch (kFileTransferMgrException $e) {
if ($e->getCode() == kFileTransferMgrException::cantConnect) {
$this->setDropFolderError($folder, KalturaDropFolderErrorCode::ERROR_CONNECT, DropFolderPlugin::ERROR_CONNECT_MESSAGE, $e);
} else {
if ($e->getCode() == kFileTransferMgrException::cantAuthenticate) {
$this->setDropFolderError($folder, KalturaDropFolderErrorCode::ERROR_AUTENTICATE, DropFolderPlugin::ERROR_AUTENTICATE_MESSAGE, $e);
} else {
$this->setDropFolderError($folder, KalturaDropFolderErrorCode::ERROR_GET_PHISICAL_FILE_LIST, DropFolderPlugin::ERROR_GET_PHISICAL_FILE_LIST_MESSAGE, $e);
}
}
$this->unimpersonate();
} catch (KalturaException $e) {
$this->setDropFolderError($folder, KalturaDropFolderErrorCode::ERROR_GET_DB_FILE_LIST, DropFolderPlugin::ERROR_GET_DB_FILE_LIST_MESSAGE, $e);
$this->unimpersonate();
} catch (Exception $e) {
$this->setDropFolderError($folder, KalturaDropFolderErrorCode::DROP_FOLDER_APP_ERROR, DropFolderPlugin::DROP_FOLDER_APP_ERROR_MESSAGE . $e->getMessage(), $e);
$this->unimpersonate();
}
}
}
}
/**
* @return KalturaDropFolderClientPlugin
*/
public static function get(KalturaClient $client)
{
if (!self::$instance) {
self::$instance = new KalturaDropFolderClientPlugin($client);
}
return self::$instance;
}
writeLog($logPrefix, 'Service URL ' . $serviceUrl);
$sleepSec = $config['sleep_time'];
$fileName = basename($filePath);
$folderPath = dirname($filePath);
writeLog($logPrefix, '---------------------------- Start handling --------------------------');
writeLog($logPrefix, 'action:' . $action);
writeLog($logPrefix, 'file path:' . $filePath);
writeLog($logPrefix, 'folder path:' . $folderPath);
writeLog($logPrefix, 'file name:' . $fileName);
writeLog($logPrefix, 'file size:' . $fileSize);
$kClientConfig = new KalturaConfiguration(-1);
$kClientConfig->serviceUrl = $serviceUrl;
$kClientConfig->curlTimeout = 180;
$kClientConfig->setLogger(new SyncDropFolderWatcherLogger($logPrefix));
$kClient = new KalturaClient($kClientConfig);
$dropFolderPlugin = KalturaDropFolderClientPlugin::get($kClient);
try {
$folder = null;
$filter = new KalturaDropFolderFilter();
$filter->pathEqual = $folderPath;
$filter->typeEqual = KalturaDropFolderType::LOCAL;
$filter->statusIn = KalturaDropFolderStatus::ENABLED . ',' . KalturaDropFolderStatus::ERROR;
$dropFolders = $dropFolderPlugin->dropFolder->listAction($filter);
writeLog($logPrefix, 'found ' . $dropFolders->totalCount . ' folders');
if ($dropFolders->totalCount == 1) {
$folder = $dropFolders->objects[0];
writeLog($logPrefix, 'drop folder id ' . $folder->id);
$ignorePatterns = array_map('trim', explode(',', $folder->ignoreFileNamePatterns));
foreach ($ignorePatterns as $ignorePattern) {
if (!is_null($ignorePattern) && $ignorePattern != '' && fnmatch($ignorePattern, $fileName)) {
writeLog($logPrefix, 'Ignoring file matching ignore pattern [' . $ignorePattern . ']');
public function __construct()
{
$this->dropFolderPlugin = KalturaDropFolderClientPlugin::get(KBatchBase::$kClient);
$this->dropFolderFileService = $this->dropFolderPlugin->dropFolderFile;
}
/**
* Update the associated drop folder file object with its current state
* @return KalturaDropFolderFile
* @param $updateStatus bool update status or not
*/
protected function updateDropFolderFile($updateStatus = true)
{
$dropFolderFilePlugin = KalturaDropFolderClientPlugin::get($this->kClient);
$updateFile = new KalturaDropFolderFile();
$updateFile->parsedSlug = $this->dropFolderFile->parsedSlug;
$updateFile->parsedFlavor = $this->dropFolderFile->parsedFlavor;
$updateFile->errorCode = $this->dropFolderFile->errorCode;
$updateFile->errorDescription = $this->dropFolderFile->errorDescription;
$this->impersonate($this->dropFolderFile->partnerId);
$updatedFile = $dropFolderFilePlugin->dropFolderFile->update($this->dropFolderFile->id, $updateFile);
if ($updateStatus) {
$updatedFile = $dropFolderFilePlugin->dropFolderFile->updateStatus($this->dropFolderFile->id, $this->dropFolderFile->status);
}
$this->unimpersonate();
return $updatedFile;
}
public function handle()
{
$this->tempDirectory = sys_get_temp_dir();
if (!is_dir($this->tempDirectory)) {
KalturaLog::err('Missing temporary directory');
return false;
}
// check prerequisites
$checkConfig = $this->checkConfig();
if (!$checkConfig) {
KalturaLog::err('Missing required configurations');
return false;
}
$this->fileTransferMgr = DropFolderBatchUtils::getFileTransferManager($this->dropFolder);
if (!$this->fileTransferMgr) {
$this->dropFolderFile->status = KalturaDropFolderFileStatus::ERROR_HANDLING;
$this->dropFolderFile->errorCode = KalturaDropFolderFileErrorCode::INTERNAL_ERROR;
$this->dropFolderFile->errorDescription = 'Internal server error - cannot initiate file transfer manager';
KalturaLog::err($this->dropFolderFile->errorDescription);
$this->updateDropFolderFile();
return false;
}
$xmlPath = $this->getLocalXmlFilePath();
if (!$xmlPath) {
$this->dropFolderFile->status = KalturaDropFolderFileStatus::ERROR_HANDLING;
$this->dropFolderFile->errorCode = KalturaDropFolderFileErrorCode::ERROR_READING_FILE;
$this->dropFolderFile->errorDescription = 'Cannot read file at path [' . $this->dropFolder->path . '/' . $this->dropFolderFile->fileName . ']';
KalturaLog::err($this->dropFolderFile->errorDescription);
$this->updateDropFolderFile();
return false;
}
$xmlDoc = new KDOMDocument();
$xmlDoc->load($xmlPath);
if (!$xmlDoc) {
$this->dropFolderFile->status = KalturaDropFolderFileStatus::ERROR_HANDLING;
$this->dropFolderFile->errorCode = KalturaDropFolderFileErrorCode::ERROR_READING_FILE;
$this->dropFolderFile->errorDescription = "Cannot parse XML file at [{$xmlPath}]";
KalturaLog::err($this->dropFolderFile->errorDescription);
$this->updateDropFolderFile();
return false;
}
$localResources = $xmlDoc->getElementsByTagName(self::DROP_FOLDER_RESOURCE_NODE_NAME);
if (!$localResources) {
$this->dropFolderFile->status = KalturaDropFolderFileStatus::ERROR_HANDLING;
$this->dropFolderFile->errorCode = KalturaDropFolderFileErrorCode::ERROR_READING_FILE;
$this->dropFolderFile->errorDescription = "Cannot parse XML file at [{$xmlPath}]";
KalturaLog::err($this->dropFolderFile->errorDescription);
$this->updateDropFolderFile();
return false;
}
$replaceResources = array();
$localResourcesLength = $localResources->length;
foreach ($localResources as $local) {
// already have drop folder file id
if (!is_null($this->getDropFolderFileId($local))) {
continue;
}
// replacement/modification of $local must not happen inside this foreach loop
$dropFolderFileId = $this->checkFileExists($local);
if (is_null($dropFolderFileId)) {
KalturaLog::debug('Some required files do not exist in the drop folder - changing status to WAITING');
$this->dropFolderFile->status = KalturaDropFolderFileStatus::WAITING;
KalturaLog::debug('Changing status to WAITING');
$this->updateDropFolderFile();
return false;
}
$localVerified = $this->verifyLocalResource($local);
if (!$localVerified) {
$this->dropFolderFile->status = KalturaDropFolderFileStatus::ERROR_HANDLING;
// error code and description already set
KalturaLog::err($this->dropFolderFile->errorDescription);
$this->updateDropFolderFile();
return false;
}
$replaceResources[] = array($local, $dropFolderFileId);
}
foreach ($replaceResources as $replace) {
$this->replaceResource($replace[0], $replace[1], $xmlDoc);
}
// create a temporary XML file from the modified $xmlDoc
$tempFile = $this->tempDirectory . DIRECTORY_SEPARATOR . uniqid() . '_' . $this->dropFolderFile->fileName;
$xmlDoc->save($tempFile);
$tempFileRealPath = realpath($tempFile);
if (!$tempFileRealPath || !is_file($tempFileRealPath)) {
$this->dropFolderFile->status = KalturaDropFolderFileStatus::ERROR_HANDLING;
$this->dropFolderFile->errorCode = KalturaDropFolderFileErrorCode::ERROR_WRITING_TEMP_FILE;
$this->dropFolderFile->errorDescription = "Error writing temporary file [{$tempFileRealPath}]";
KalturaLog::err($this->dropFolderFile->errorDescription);
$this->updateDropFolderFile();
return false;
}
$conversionProfile = $this->getConversionProfile();
// add bulk upload of type KalturaBulkUploadType::DROP_FOLDER_XML
try {
$this->impersonate($this->dropFolderFile->partnerId);
$this->kClient->bulkUpload->add($conversionProfile->id, $tempFileRealPath, KalturaBulkUploadType::DROP_FOLDER_XML, $this->uploadedBy, $this->dropFolderFile->fileName);
$this->unimpersonate();
} catch (Exception $e) {
$this->unimpersonate();
$this->dropFolderFile->status = KalturaDropFolderFileStatus::ERROR_HANDLING;
$this->dropFolderFile->errorCode = KalturaDropFolderFileErrorCode::ERROR_ADDING_BULK_UPLOAD;
$this->dropFolderFile->errorDescription = 'Error adding bulk upload - ' . $e->getMessage();
KalturaLog::err($this->dropFolderFile->errorDescription);
$this->updateDropFolderFile();
KalturaLog::err($this->dropFolderFile->errorDescription);
return false;
}
//delete the temporary file
@unlink($tempFileRealPath);
$dropFolderFilePlugin = KalturaDropFolderClientPlugin::get($this->kClient);
$dropFolderFilePlugin->dropFolderFile->updateStatus($this->dropFolderFile->id, KalturaDropFolderFileStatus::HANDLED);
KalturaLog::debug('Drop folder file [' . $this->dropFolderFile->id . '] handled successfully');
return true;
// file handled
}