public function handleBulkUpload() { KBatchBase::impersonate($this->currentPartnerId); $dropFolderPlugin = KalturaDropFolderClientPlugin::get(KBatchBase::$kClient); $this->setContentResourceFilesMap($dropFolderPlugin); KBatchBase::unimpersonate(); parent::handleBulkUpload(); }
public function handleBulkUpload() { KalturaLog::debug("Starting BulkUpload for XML drop folder file with id [" . $this->job->jobObjectId . ']'); KBatchBase::impersonate($this->currentPartnerId); $dropFolderPlugin = KalturaDropFolderClientPlugin::get(KBatchBase::$kClient); $this->setContentResourceFilesMap($dropFolderPlugin); KBatchBase::unimpersonate(); parent::handleBulkUpload(); }
/** * Main logic function. * @param KalturaDropFolder $folder */ private function handleFolder(KalturaDropFolder $folder) { KalturaLog::debug('Handling folder [' . $folder->id . ']'); $fileNamePatterns = explode(',', $folder->fileNamePatterns); if (count($fileNamePatterns) > 1) { $fileNamePatterns = null; } elseif (count($fileNamePatterns) == 1) { foreach ($fileNamePatterns as $index => $fileNamePattern) { $fileNamePatterns[$index] = trim($fileNamePattern, ' *'); } $fileNamePatterns = reset($fileNamePatterns); } $dropFolderFilePlugin = KalturaDropFolderClientPlugin::get($this->kClient); $dropFolderFileFilter = new KalturaDropFolderFileFilter(); $dropFolderFileFilter->dropFolderIdEqual = $folder->id; $dropFolderFileFilter->statusIn = KalturaDropFolderFileStatus::PENDING . ',' . KalturaDropFolderFileStatus::WAITING . ',' . KalturaDropFolderFileStatus::NO_MATCH; $dropFolderFileFilter->orderBy = KalturaDropFolderFileOrderBy::CREATED_AT_DESC; if ($fileNamePatterns) { $dropFolderFileFilter->fileNameLike = $fileNamePatterns; } $pager = new KalturaFilterPager(); $pager->pageIndex = 1; try { $dropFolderFiles = $dropFolderFilePlugin->dropFolderFile->listAction($dropFolderFileFilter, $pager); /* @var $dropFolderFiles KalturaDropFolderFileListResponse */ } catch (KalturaAPIException $e) { KalturaLog::err('Cannot get list of files for drop folder id [' . $folder->id . '] pageIndex [' . $pager->pageIndex . '] - ' . $e->getMessage()); return false; } while (count($dropFolderFiles->objects)) { foreach ($dropFolderFiles->objects as $file) { $fileHandled = $this->handleFile($folder, $file); if ($fileHandled) { // break loop and go to next folder, because current folder files' status might have changed return true; } } $pager->pageIndex++; try { $dropFolderFiles = $dropFolderFilePlugin->dropFolderFile->listAction($dropFolderFileFilter, $pager); } catch (KalturaAPIException $e) { KalturaLog::err('Cannot get list of files for drop folder id [' . $folder->id . '] pageIndex [' . $pager->pageIndex . ']- ' . $e->getMessage()); return false; } } if ($pager->pageIndex > 1) { return true; } else { return false; } // no file was handled }
public function run($jobs = null) { KalturaLog::info("Drop folder watcher batch is running"); $this->dropFolderPlugin = KalturaDropFolderClientPlugin::get($this->kClient); if ($this->taskConfig->isInitOnly()) { return $this->init(); } //TODO: use getFilter instead of taskConfig->params // get drop folder tags to work on from configuration $folderTags = $this->taskConfig->params->tags; $currentDc = $this->taskConfig->params->dc; if (strlen($folderTags) == 0) { KalturaLog::err('Tags configuration is empty - cannot continue'); return; } if (strlen($currentDc) == 0) { KalturaLog::err('DC configuration is empty - cannot continue'); return; } // get list of drop folders according to configuration $filter = new KalturaDropFolderFilter(); if ($folderTags != '*') { $filter->tagsMultiLikeOr = $folderTags; } $filter->dcEqual = $currentDc; $filter->statusEqual = KalturaDropFolderStatus::ENABLED; try { $dropFolders = $this->dropFolderPlugin->dropFolder->listAction($filter); } catch (Exception $e) { $this->unimpersonate(); KalturaLog::err('Cannot get drop folder list - ' . $e->getMessage()); return; } $dropFolders = $dropFolders->objects; KalturaLog::log('[' . count($dropFolders) . '] folders to watch'); foreach ($dropFolders as $folder) { try { $this->watchFolder($folder); } catch (Exception $e) { $this->unimpersonate(); KalturaLog::err('Unknown error with folder id [' . $folder->id . '] - ' . $e->getMessage()); } } }
public function run($jobs = null) { KalturaLog::info("Drop folder watcher batch is running"); $this->dropFolderPlugin = KalturaDropFolderClientPlugin::get(self::$kClient); if (self::$taskConfig->isInitOnly()) { return $this->init(); } $dropFolders = $this->getDropFoldersList(); if (isset($dropFolders)) { $dropFolders = $dropFolders->objects; KalturaLog::log('[' . count($dropFolders) . '] folders to watch'); foreach ($dropFolders as $folder) { /* @var $folder KalturaDropFolder */ try { $this->impersonate($folder->partnerId); $engine = KDropFolderEngine::getInstance($folder->type); $engine->watchFolder($folder); $this->setDropFolderOK($folder); $this->unimpersonate(); } catch (kFileTransferMgrException $e) { if ($e->getCode() == kFileTransferMgrException::cantConnect) { $this->setDropFolderError($folder, KalturaDropFolderErrorCode::ERROR_CONNECT, DropFolderPlugin::ERROR_CONNECT_MESSAGE, $e); } else { if ($e->getCode() == kFileTransferMgrException::cantAuthenticate) { $this->setDropFolderError($folder, KalturaDropFolderErrorCode::ERROR_AUTENTICATE, DropFolderPlugin::ERROR_AUTENTICATE_MESSAGE, $e); } else { $this->setDropFolderError($folder, KalturaDropFolderErrorCode::ERROR_GET_PHISICAL_FILE_LIST, DropFolderPlugin::ERROR_GET_PHISICAL_FILE_LIST_MESSAGE, $e); } } $this->unimpersonate(); } catch (KalturaException $e) { $this->setDropFolderError($folder, KalturaDropFolderErrorCode::ERROR_GET_DB_FILE_LIST, DropFolderPlugin::ERROR_GET_DB_FILE_LIST_MESSAGE, $e); $this->unimpersonate(); } catch (Exception $e) { $this->setDropFolderError($folder, KalturaDropFolderErrorCode::DROP_FOLDER_APP_ERROR, DropFolderPlugin::DROP_FOLDER_APP_ERROR_MESSAGE . $e->getMessage(), $e); $this->unimpersonate(); } } } }
/** * @return KalturaDropFolderClientPlugin */ public static function get(KalturaClient $client) { if (!self::$instance) { self::$instance = new KalturaDropFolderClientPlugin($client); } return self::$instance; }
writeLog($logPrefix, 'Service URL ' . $serviceUrl); $sleepSec = $config['sleep_time']; $fileName = basename($filePath); $folderPath = dirname($filePath); writeLog($logPrefix, '---------------------------- Start handling --------------------------'); writeLog($logPrefix, 'action:' . $action); writeLog($logPrefix, 'file path:' . $filePath); writeLog($logPrefix, 'folder path:' . $folderPath); writeLog($logPrefix, 'file name:' . $fileName); writeLog($logPrefix, 'file size:' . $fileSize); $kClientConfig = new KalturaConfiguration(-1); $kClientConfig->serviceUrl = $serviceUrl; $kClientConfig->curlTimeout = 180; $kClientConfig->setLogger(new SyncDropFolderWatcherLogger($logPrefix)); $kClient = new KalturaClient($kClientConfig); $dropFolderPlugin = KalturaDropFolderClientPlugin::get($kClient); try { $folder = null; $filter = new KalturaDropFolderFilter(); $filter->pathEqual = $folderPath; $filter->typeEqual = KalturaDropFolderType::LOCAL; $filter->statusIn = KalturaDropFolderStatus::ENABLED . ',' . KalturaDropFolderStatus::ERROR; $dropFolders = $dropFolderPlugin->dropFolder->listAction($filter); writeLog($logPrefix, 'found ' . $dropFolders->totalCount . ' folders'); if ($dropFolders->totalCount == 1) { $folder = $dropFolders->objects[0]; writeLog($logPrefix, 'drop folder id ' . $folder->id); $ignorePatterns = array_map('trim', explode(',', $folder->ignoreFileNamePatterns)); foreach ($ignorePatterns as $ignorePattern) { if (!is_null($ignorePattern) && $ignorePattern != '' && fnmatch($ignorePattern, $fileName)) { writeLog($logPrefix, 'Ignoring file matching ignore pattern [' . $ignorePattern . ']');
public function __construct() { $this->dropFolderPlugin = KalturaDropFolderClientPlugin::get(KBatchBase::$kClient); $this->dropFolderFileService = $this->dropFolderPlugin->dropFolderFile; }
/** * Update the associated drop folder file object with its current state * @return KalturaDropFolderFile * @param $updateStatus bool update status or not */ protected function updateDropFolderFile($updateStatus = true) { $dropFolderFilePlugin = KalturaDropFolderClientPlugin::get($this->kClient); $updateFile = new KalturaDropFolderFile(); $updateFile->parsedSlug = $this->dropFolderFile->parsedSlug; $updateFile->parsedFlavor = $this->dropFolderFile->parsedFlavor; $updateFile->errorCode = $this->dropFolderFile->errorCode; $updateFile->errorDescription = $this->dropFolderFile->errorDescription; $this->impersonate($this->dropFolderFile->partnerId); $updatedFile = $dropFolderFilePlugin->dropFolderFile->update($this->dropFolderFile->id, $updateFile); if ($updateStatus) { $updatedFile = $dropFolderFilePlugin->dropFolderFile->updateStatus($this->dropFolderFile->id, $this->dropFolderFile->status); } $this->unimpersonate(); return $updatedFile; }
public function handle() { $this->tempDirectory = sys_get_temp_dir(); if (!is_dir($this->tempDirectory)) { KalturaLog::err('Missing temporary directory'); return false; } // check prerequisites $checkConfig = $this->checkConfig(); if (!$checkConfig) { KalturaLog::err('Missing required configurations'); return false; } $this->fileTransferMgr = DropFolderBatchUtils::getFileTransferManager($this->dropFolder); if (!$this->fileTransferMgr) { $this->dropFolderFile->status = KalturaDropFolderFileStatus::ERROR_HANDLING; $this->dropFolderFile->errorCode = KalturaDropFolderFileErrorCode::INTERNAL_ERROR; $this->dropFolderFile->errorDescription = 'Internal server error - cannot initiate file transfer manager'; KalturaLog::err($this->dropFolderFile->errorDescription); $this->updateDropFolderFile(); return false; } $xmlPath = $this->getLocalXmlFilePath(); if (!$xmlPath) { $this->dropFolderFile->status = KalturaDropFolderFileStatus::ERROR_HANDLING; $this->dropFolderFile->errorCode = KalturaDropFolderFileErrorCode::ERROR_READING_FILE; $this->dropFolderFile->errorDescription = 'Cannot read file at path [' . $this->dropFolder->path . '/' . $this->dropFolderFile->fileName . ']'; KalturaLog::err($this->dropFolderFile->errorDescription); $this->updateDropFolderFile(); return false; } $xmlDoc = new KDOMDocument(); $xmlDoc->load($xmlPath); if (!$xmlDoc) { $this->dropFolderFile->status = KalturaDropFolderFileStatus::ERROR_HANDLING; $this->dropFolderFile->errorCode = KalturaDropFolderFileErrorCode::ERROR_READING_FILE; $this->dropFolderFile->errorDescription = "Cannot parse XML file at [{$xmlPath}]"; KalturaLog::err($this->dropFolderFile->errorDescription); $this->updateDropFolderFile(); return false; } $localResources = $xmlDoc->getElementsByTagName(self::DROP_FOLDER_RESOURCE_NODE_NAME); if (!$localResources) { $this->dropFolderFile->status = KalturaDropFolderFileStatus::ERROR_HANDLING; $this->dropFolderFile->errorCode = KalturaDropFolderFileErrorCode::ERROR_READING_FILE; $this->dropFolderFile->errorDescription = "Cannot parse XML file at [{$xmlPath}]"; KalturaLog::err($this->dropFolderFile->errorDescription); $this->updateDropFolderFile(); return false; } $replaceResources = array(); $localResourcesLength = $localResources->length; foreach ($localResources as $local) { // already have drop folder file id if (!is_null($this->getDropFolderFileId($local))) { continue; } // replacement/modification of $local must not happen inside this foreach loop $dropFolderFileId = $this->checkFileExists($local); if (is_null($dropFolderFileId)) { KalturaLog::debug('Some required files do not exist in the drop folder - changing status to WAITING'); $this->dropFolderFile->status = KalturaDropFolderFileStatus::WAITING; KalturaLog::debug('Changing status to WAITING'); $this->updateDropFolderFile(); return false; } $localVerified = $this->verifyLocalResource($local); if (!$localVerified) { $this->dropFolderFile->status = KalturaDropFolderFileStatus::ERROR_HANDLING; // error code and description already set KalturaLog::err($this->dropFolderFile->errorDescription); $this->updateDropFolderFile(); return false; } $replaceResources[] = array($local, $dropFolderFileId); } foreach ($replaceResources as $replace) { $this->replaceResource($replace[0], $replace[1], $xmlDoc); } // create a temporary XML file from the modified $xmlDoc $tempFile = $this->tempDirectory . DIRECTORY_SEPARATOR . uniqid() . '_' . $this->dropFolderFile->fileName; $xmlDoc->save($tempFile); $tempFileRealPath = realpath($tempFile); if (!$tempFileRealPath || !is_file($tempFileRealPath)) { $this->dropFolderFile->status = KalturaDropFolderFileStatus::ERROR_HANDLING; $this->dropFolderFile->errorCode = KalturaDropFolderFileErrorCode::ERROR_WRITING_TEMP_FILE; $this->dropFolderFile->errorDescription = "Error writing temporary file [{$tempFileRealPath}]"; KalturaLog::err($this->dropFolderFile->errorDescription); $this->updateDropFolderFile(); return false; } $conversionProfile = $this->getConversionProfile(); // add bulk upload of type KalturaBulkUploadType::DROP_FOLDER_XML try { $this->impersonate($this->dropFolderFile->partnerId); $this->kClient->bulkUpload->add($conversionProfile->id, $tempFileRealPath, KalturaBulkUploadType::DROP_FOLDER_XML, $this->uploadedBy, $this->dropFolderFile->fileName); $this->unimpersonate(); } catch (Exception $e) { $this->unimpersonate(); $this->dropFolderFile->status = KalturaDropFolderFileStatus::ERROR_HANDLING; $this->dropFolderFile->errorCode = KalturaDropFolderFileErrorCode::ERROR_ADDING_BULK_UPLOAD; $this->dropFolderFile->errorDescription = 'Error adding bulk upload - ' . $e->getMessage(); KalturaLog::err($this->dropFolderFile->errorDescription); $this->updateDropFolderFile(); KalturaLog::err($this->dropFolderFile->errorDescription); return false; } //delete the temporary file @unlink($tempFileRealPath); $dropFolderFilePlugin = KalturaDropFolderClientPlugin::get($this->kClient); $dropFolderFilePlugin->dropFolderFile->updateStatus($this->dropFolderFile->id, KalturaDropFolderFileStatus::HANDLED); KalturaLog::debug('Drop folder file [' . $this->dropFolderFile->id . '] handled successfully'); return true; // file handled }