public static function abortDbBatchJob(BatchJob $dbBatchJob, $force = false) { // No need to abort finished job if (in_array($dbBatchJob->getStatus(), BatchJobPeer::getClosedStatusList())) { if ($force) { $dbBatchJob->setExecutionStatus(BatchJobExecutionStatus::ABORTED); $dbBatchJob->save(); } return $dbBatchJob; } $lockObject = $dbBatchJob->getBatchJobLock(); if (is_null($lockObject)) { KalturaLog::err("Batch job [" . $dbBatchJob->getId() . "] doesn't have a lock object and can't be deleted. Status (" . $dbBatchJob->getStatus() . ")"); return $dbBatchJob; } // Update status $con = Propel::getConnection(); $update = new Criteria(); $update->add(BatchJobLockPeer::STATUS, BatchJob::BATCHJOB_STATUS_ABORTED); $update->add(BatchJobLockPeer::VERSION, $lockObject->getVersion() + 1); $updateCondition = new Criteria(); $updateCondition->add(BatchJobLockPeer::ID, $lockObject->getId(), Criteria::EQUAL); $updateCondition->add(BatchJobLockPeer::VERSION, $lockObject->getVersion(), Criteria::EQUAL); $updateCondition->add(BatchJobLockPeer::SCHEDULER_ID, null, Criteria::ISNULL); $affectedRows = BasePeer::doUpdate($updateCondition, $update, $con); if ($affectedRows) { $dbBatchJob->setExecutionStatus(BatchJobExecutionStatus::ABORTED); $dbBatchJob = self::updateBatchJob($dbBatchJob, BatchJob::BATCHJOB_STATUS_ABORTED); } else { $dbBatchJob->setExecutionStatus(BatchJobExecutionStatus::ABORTED); $dbBatchJob->save(); } self::abortChildJobs($dbBatchJob); return $dbBatchJob; }
/** * @param BatchJob $job * @param $errorDescription * @return BatchJob */ protected function finishJobWithError(BatchJob $job, $errorDescription) { $job->setStatus(BatchJob::BATCHJOB_STATUS_FAILED); $job->setDescription($job->getDescription() . '\\n' . $errorDescription); $job->save(); return $job; }
/** * @param string $puser_id * @param string $entry * @param string $version * @param string $file_format * @return BatchJob */ public static function addJob($puser_id, $entry, $version, $file_format) { $entryId = $entry->getId(); $entryIntId = $entry->getIntId(); $entryVersion = $version ? $version : $entry->getVersion(); if ($entry) { $partner = $entry->getPartner(); $email = $partner->getAdminEmail(); } $data = json_encode(array('puserId' => $puser_id, 'entryId' => $entryId, 'entryIntId' => $entryIntId, 'entryVersion' => $entryVersion, 'fileFormat' => $file_format, 'email' => $email)); $job = new BatchJob(); $job->setJobType(BatchJobType::FLATTEN); $job->setData($data, true); $job->setStatus(BatchJob::BATCHJOB_STATUS_PENDING); $job->setCheckAgainTimeout(time() + 10); $job->setProgress(0); $job->setMessage('Queued'); $job->setDescription('Queued, waiting to run'); $job->setUpdatesCount(0); $job->setEntryId($entryId); $job->setPartnerId($entry->getPartnerId()); $job->setSubpId($entry->getSubpId()); $job->save(); return $job; }
private function reconvertEntry($entry_id, $conversion_profile_id, $job_priority) { $entry = entryPeer::retrieveByPK($entry_id); $this->error = ""; if (!$entry) { $error = "Cannot reconvert entry [{$entry_id}]. Might be a deleted entry"; return array($entry_id, null, null, $error); } $flavorAsset = assetPeer::retrieveOriginalByEntryId($entry_id); if (!$flavorAsset) { $flavorAsset = assetPeer::retrieveReadyWebByEntryId($entry_id); if (!$flavorAsset) { $flavorAssets = assetPeer::retrieveFlavorsByEntryId($entry_id); if (!$flavorAssets) { $error = "Cannot find good enough flavor asset to re-convert from"; return array($entry_id, $entry, null, $error); } $flavorAsset = $flavorAssets[0]; // choose the first one } } $syncKey = $flavorAsset->getSyncKey(flavorAsset::FILE_SYNC_FLAVOR_ASSET_SUB_TYPE_ASSET); $filePath = kFileSyncUtils::getReadyLocalFilePathForKey($syncKey); if (!$filePath) { $error = "Cannot find a fileSync for the flavorAsset [" . $flavorAsset->getId() . "]"; return array($entry_id, $entry, null, $error); } $dbBatchJob = new BatchJob(); $dbBatchJob->setEntryId($entry_id); $dbBatchJob->setPartnerId($entry->getPartnerId()); $dbBatchJob->setStatus(BatchJob::BATCHJOB_STATUS_PENDING); $dbBatchJob->setDc(kDataCenterMgr::getCurrentDcId()); //$dbBatchJob->setPriority ( $job_priority ); Not supported anymore $dbBatchJob->setObjectId($entry_id); $dbBatchJob->setObjectType(BatchJobObjectType::ENTRY); $dbBatchJob->setJobType(BatchJobType::CONVERT_PROFILE); $dbBatchJob->save(); // creates a convert profile job $convertProfileData = new kConvertProfileJobData(); $convertProfileData->setFlavorAssetId($flavorAsset->getId()); $convertProfileData->setInputFileSyncLocalPath($filePath); kJobsManager::addJob($dbBatchJob, $convertProfileData, BatchJobType::CONVERT_PROFILE); // save again afget the addJob $dbBatchJob->save(); return array($entry_id, $entry, $dbBatchJob, $error); }
/** * Add new bulk upload batch job * Conversion profile id can be specified in the API or in the CSV file, the one in the CSV file will be stronger. * If no conversion profile was specified, partner's default will be used * * @action add * @param int $conversionProfileId Convertion profile id to use for converting the current bulk (-1 to use partner's default) * @param file $csvFileData CSV File * @return KalturaBulkUpload */ function addAction($conversionProfileId, $csvFileData) { // first we copy the file to "content/batchfiles/[partner_id]/" $origFilename = $csvFileData["name"]; $fileInfo = pathinfo($origFilename); $extension = strtolower($fileInfo["extension"]); if ($extension != "csv") { throw new KalturaAPIException(KalturaErrors::INVALID_FILE_EXTENSION); } $job = new BatchJob(); $job->setPartnerId($this->getPartnerId()); $job->save(); $syncKey = $job->getSyncKey(BatchJob::FILE_SYNC_BATCHJOB_SUB_TYPE_BULKUPLOADCSV); // kFileSyncUtils::file_put_contents($syncKey, file_get_contents($csvFileData["tmp_name"])); try { kFileSyncUtils::moveFromFile($csvFileData["tmp_name"], $syncKey, true); } catch (Exception $e) { throw new KalturaAPIException(KalturaErrors::BULK_UPLOAD_CREATE_CSV_FILE_SYNC_ERROR); } $csvPath = kFileSyncUtils::getLocalFilePathForKey($syncKey); $data = new KalturaBulkUploadJobData(); $data->csvFilePath = $csvPath; $data->userId = $this->getKuser()->getPuserId(); $data->uploadedBy = $this->getKuser()->getScreenName(); if ($conversionProfileId === -1) { $conversionProfileId = $this->getPartner()->getDefaultConversionProfileId(); } $kmcVersion = $this->getPartner()->getKmcVersion(); $check = null; if ($kmcVersion < 2) { $check = ConversionProfilePeer::retrieveByPK($conversionProfileId); } else { $check = conversionProfile2Peer::retrieveByPK($conversionProfileId); } if (!$check) { throw new KalturaAPIException(KalturaErrors::CONVERSION_PROFILE_ID_NOT_FOUND, $conversionProfileId); } $data->conversionProfileId = $conversionProfileId; $dbJob = kJobsManager::addJob($job, $data->toObject(), KalturaBatchJobType::BULKUPLOAD); $bulkUpload = new KalturaBulkUpload(); $bulkUpload->fromObject($dbJob); return $bulkUpload; }
public function executeImpl($partner_id, $subp_id, $puser_id, $partner_prefix, $puser_kuser) { $entry_id = $this->getPM("entry_id"); $entry = entryPeer::retrieveByPK($entry_id); if (!$entry) { $this->addError(APIErrors::INVALID_ENTRY_ID, "entry", $entry_id); } else { $job = new BatchJob(); $job->setJobType(BatchJobType::DVDCREATOR); $job->setStatus(BatchJob::BATCHJOB_STATUS_PENDING); //$job->setCheckAgainTimeout(time() + 10); $job->setEntryId($entry_id); $job->setPartnerId($entry->getPartnerId()); $job->setSubpId($entry->getSubpId()); $job->save(); $wrapper = objectWrapperBase::getWrapperClass($job, objectWrapperBase::DETAIL_LEVEL_DETAILED); // TODO - remove this code when cache works properly when saving objects (in their save method) $wrapper->removeFromCache("batch_job", $job->getId()); $this->addMsg("batchjob", $wrapper); } }
public function executeImpl($partner_id, $subp_id, $puser_id, $partner_prefix, $puser_kuser) { $fileField = "csv_file"; $profileId = $this->getP("profile_id"); if (count($_FILES) == 0) { $this->addError(APIErrors::NO_FILES_RECEIVED); return; } if (!@$_FILES[$fileField]) { $this->addError(APIErrors::INVALID_FILE_FIELD, $fileField); return; } // first we copy the file to "content/batchfiles/[partner_id]/" $origFilename = $_FILES[$fileField]['name']; $fileInfo = pathinfo($origFilename); $extension = strtolower($fileInfo['extension']); if ($extension != "csv") { $this->addError(APIErrors::INVALID_FILE_EXTENSION); return; } $job = new BatchJob(); $job->setPartnerId($partner_id); $job->save(); $syncKey = $job->getSyncKey(BatchJob::FILE_SYNC_BATCHJOB_SUB_TYPE_BULKUPLOADCSV); // kFileSyncUtils::file_put_contents($syncKey, file_get_contents($csvFileData["tmp_name"])); try { kFileSyncUtils::moveFromFile($_FILES[$fileField]['tmp_name'], $syncKey, true); } catch (Exception $e) { throw new KalturaAPIException(KalturaErrors::BULK_UPLOAD_CREATE_CSV_FILE_SYNC_ERROR); } $csvPath = kFileSyncUtils::getLocalFilePathForKey($syncKey); $data = new kBulkUploadJobData(); $data->setCsvFilePath($csvPath); $data->setUserId($puser_kuser->getPuserId()); $data->setUploadedBy($puser_kuser->getPuserName()); $data->setConversionProfileId($profileId); kJobsManager::addJob($job, $data, BatchJobType::BULKUPLOAD); $this->addMsg("status", "ok"); }
public static function handleConvertFailed(BatchJob $dbBatchJob, $engineType, $flavorAssetId, $flavorParamsOutputId, $mediaInfoId) { $flavorAsset = assetPeer::retrieveById($flavorAssetId); // verifies that flavor asset exists if (!$flavorAsset) { throw new APIException(APIErrors::INVALID_FLAVOR_ASSET_ID, $flavorAssetId); } /* * On Webex error, roll back the inter-src asset version in order to allow the retry to get ARF as a source, * rather than the invlaid WMV file (product of bad nbrplayer session) */ if ($dbBatchJob->getErrNumber() == BatchJobAppErrors::BLACK_OR_SILENT_CONTENT) { $prevVer = $flavorAsset->getPreviousVersion(); $currVer = $flavorAsset->getVersion(); KalturaLog::log("Webex conversion - Garbled Audio or Black frame or Silence. Rolling back asset/file-sync version - curr({$currVer}), prev({$prevVer})"); if (isset($prevVer)) { $syncKey = $flavorAsset->getSyncKey(asset::FILE_SYNC_ASSET_SUB_TYPE_ASSET, $currVer); if (isset($syncKey)) { kFileSyncUtils::deleteSyncFileForKey($syncKey, false, true); $flavorAsset->setVersion($prevVer); $flavorAsset->setPreviousVersion(null); KalturaLog::log("Webex conversion - Rolled back"); } } } $flavorAsset->setStatus(flavorAsset::FLAVOR_ASSET_STATUS_ERROR); $flavorAsset->save(); // try to create a convert job with the next engine if (!is_null($engineType)) { $data = $dbBatchJob->getData(); if ($data instanceof kConvartableJobData) { $data->incrementOperationSet(); $dbBatchJob->setData($data); $dbBatchJob->save(); } $newDbBatchJob = kBusinessPreConvertDL::redecideFlavorConvert($flavorAssetId, $flavorParamsOutputId, $mediaInfoId, $dbBatchJob, $engineType); if ($newDbBatchJob) { return true; } } // find the root job $rootBatchJob = $dbBatchJob->getRootJob(); if (!$rootBatchJob) { return false; } // the root is already failed if ($rootBatchJob->getStatus() == BatchJob::BATCHJOB_STATUS_FAILED || $rootBatchJob->getStatus() == BatchJob::BATCHJOB_STATUS_FATAL) { return false; } // bulk download root job no need to handle if ($rootBatchJob->getJobType() == BatchJobType::BULKDOWNLOAD) { kJobsManager::failBatchJob($rootBatchJob, "Convert job " . $dbBatchJob->getId() . " failed"); return false; } if (is_null($flavorParamsOutputId)) { kJobsManager::failBatchJob($rootBatchJob, "Job " . $dbBatchJob->getId() . " failed"); kBatchManager::updateEntry($dbBatchJob->getEntryId(), entryStatus::ERROR_CONVERTING); return false; } $readyBehavior = $dbBatchJob->getData()->getReadyBehavior(); if ($readyBehavior == flavorParamsConversionProfile::READY_BEHAVIOR_REQUIRED) { kJobsManager::failBatchJob($rootBatchJob, "Job " . $dbBatchJob->getId() . " failed"); kBatchManager::updateEntry($dbBatchJob->getEntryId(), entryStatus::ERROR_CONVERTING); return false; } // failing the root profile job if all child jobs failed if ($rootBatchJob->getJobType() != BatchJobType::CONVERT_PROFILE) { return false; } $siblingJobs = $rootBatchJob->getChildJobs(); foreach ($siblingJobs as $siblingJob) { /* @var $siblingJob BatchJob */ // not conversion job and should be ignored if ($siblingJob->getJobType() != BatchJobType::CONVERT && $siblingJob->getJobType() != BatchJobType::POSTCONVERT) { continue; } $jobData = $siblingJob->getData(); if (!$jobData || !$jobData instanceof kConvertJobData && !$jobData instanceof kPostConvertJobData) { KalturaLog::err("Job id [" . $siblingJob->getId() . "] has no valid job data"); continue; } // found child flavor asset that hasn't failed, no need to fail the root job $siblingFlavorAssetId = $jobData->getFlavorAssetId(); $siblingFlavorAsset = assetPeer::retrieveById($siblingFlavorAssetId); if ($siblingFlavorAsset && $siblingFlavorAsset->getStatus() != flavorAsset::FLAVOR_ASSET_STATUS_ERROR && $siblingFlavorAsset->getStatus() != flavorAsset::FLAVOR_ASSET_STATUS_NOT_APPLICABLE && $siblingFlavorAsset->getStatus() != flavorAsset::FLAVOR_ASSET_STATUS_DELETED) { return false; } } // all conversions failed, should fail the root job kJobsManager::failBatchJob($rootBatchJob, "All conversions failed"); kBatchManager::updateEntry($dbBatchJob->getEntryId(), entryStatus::ERROR_CONVERTING); return false; }
/** * @param BatchJob $dbBatchJob * @param kDistributionDeleteJobData $data * @param BatchJob $twinJob * @return BatchJob */ public static function onDistributionDeleteJobUpdated(BatchJob $dbBatchJob, kDistributionDeleteJobData $data, BatchJob $twinJob = null) { if ($data->getResults() || $data->getSentData()) { $entryDistribution = EntryDistributionPeer::retrieveByPK($data->getEntryDistributionId()); if (!$entryDistribution) { KalturaLog::err("Entry distribution [" . $data->getEntryDistributionId() . "] not found"); return $dbBatchJob; } if ($data->getResults()) { $entryDistribution->incrementDeleteResultsVersion(); } if ($data->getSentData()) { $entryDistribution->incrementDeleteDataVersion(); } $entryDistribution->save(); if ($data->getResults()) { $key = $entryDistribution->getSyncKey(EntryDistribution::FILE_SYNC_ENTRY_DISTRIBUTION_DELETE_RESULTS); kFileSyncUtils::file_put_contents($key, $data->getResults()); $data->setResults(null); } if ($data->getSentData()) { $key = $entryDistribution->getSyncKey(EntryDistribution::FILE_SYNC_ENTRY_DISTRIBUTION_DELETE_DATA); kFileSyncUtils::file_put_contents($key, $data->getSentData()); $data->setSentData(null); } $dbBatchJob->setData($data); $dbBatchJob->save(); } switch ($dbBatchJob->getStatus()) { case BatchJob::BATCHJOB_STATUS_PENDING: return self::onDistributionDeleteJobPending($dbBatchJob, $data, $twinJob); case BatchJob::BATCHJOB_STATUS_FINISHED: return self::onDistributionDeleteJobFinished($dbBatchJob, $data, $twinJob); case BatchJob::BATCHJOB_STATUS_FAILED: case BatchJob::BATCHJOB_STATUS_FATAL: return self::onDistributionDeleteJobFailed($dbBatchJob, $data, $twinJob); default: return $dbBatchJob; } }
public static function handleConvertFailed(BatchJob $dbBatchJob, $engineType, $flavorAssetId, $flavorParamsOutputId, $mediaInfoId) { $flavorAsset = assetPeer::retrieveById($flavorAssetId); // verifies that flavor asset exists if (!$flavorAsset) { throw new APIException(APIErrors::INVALID_FLAVOR_ASSET_ID, $flavorAssetId); } $flavorAsset->setStatus(flavorAsset::FLAVOR_ASSET_STATUS_ERROR); $flavorAsset->save(); // try to create a convert job with the next engine if (!is_null($engineType)) { $data = $dbBatchJob->getData(); if ($data instanceof kConvartableJobData) { $data->incrementOperationSet(); $dbBatchJob->setData($data); $dbBatchJob->save(); } $newDbBatchJob = kBusinessPreConvertDL::redecideFlavorConvert($flavorAssetId, $flavorParamsOutputId, $mediaInfoId, $dbBatchJob, $engineType); if ($newDbBatchJob) { return true; } } // find the root job $rootBatchJob = $dbBatchJob->getRootJob(); if (!$rootBatchJob) { return false; } // the root is already failed if ($rootBatchJob->getStatus() == BatchJob::BATCHJOB_STATUS_FAILED || $rootBatchJob->getStatus() == BatchJob::BATCHJOB_STATUS_FATAL) { return false; } // bulk download root job no need to handle if ($rootBatchJob->getJobType() == BatchJobType::BULKDOWNLOAD) { kJobsManager::failBatchJob($rootBatchJob, "Convert job " . $dbBatchJob->getId() . " failed"); return false; } if (is_null($flavorParamsOutputId)) { kJobsManager::failBatchJob($rootBatchJob, "Job " . $dbBatchJob->getId() . " failed"); kBatchManager::updateEntry($dbBatchJob->getEntryId(), entryStatus::ERROR_CONVERTING); return false; } $readyBehavior = $dbBatchJob->getData()->getReadyBehavior(); if ($readyBehavior == flavorParamsConversionProfile::READY_BEHAVIOR_REQUIRED) { kJobsManager::failBatchJob($rootBatchJob, "Job " . $dbBatchJob->getId() . " failed"); kBatchManager::updateEntry($dbBatchJob->getEntryId(), entryStatus::ERROR_CONVERTING); return false; } // failing the root profile job if all child jobs failed if ($rootBatchJob->getJobType() != BatchJobType::CONVERT_PROFILE) { return false; } $siblingJobs = $rootBatchJob->getChildJobs(); foreach ($siblingJobs as $siblingJob) { /* @var $siblingJob BatchJob */ // not conversion job and should be ignored if ($siblingJob->getJobType() != BatchJobType::CONVERT && $siblingJob->getJobType() != BatchJobType::POSTCONVERT) { continue; } $jobData = $siblingJob->getData(); if (!$jobData || !$jobData instanceof kConvertJobData && !$jobData instanceof kPostConvertJobData) { KalturaLog::err("Job id [" . $siblingJob->getId() . "] has no valid job data"); continue; } // found child flavor asset that hasn't failed, no need to fail the root job $siblingFlavorAssetId = $jobData->getFlavorAssetId(); $siblingFlavorAsset = assetPeer::retrieveById($siblingFlavorAssetId); if ($siblingFlavorAsset->getStatus() != flavorAsset::FLAVOR_ASSET_STATUS_ERROR && $siblingFlavorAsset->getStatus() != flavorAsset::FLAVOR_ASSET_STATUS_NOT_APPLICABLE && $siblingFlavorAsset->getStatus() != flavorAsset::FLAVOR_ASSET_STATUS_DELETED) { return false; } } // all conversions failed, should fail the root job kJobsManager::failBatchJob($rootBatchJob, "All conversions failed"); kBatchManager::updateEntry($dbBatchJob->getEntryId(), entryStatus::ERROR_CONVERTING); return false; }
private function onBulkUploadJobStatusUpdated(BatchJob $dbBatchJob) { $xmlDropFolderFile = DropFolderFilePeer::retrieveByPK($dbBatchJob->getObjectId()); if (!$xmlDropFolderFile) { return; } KalturaLog::debug('object id ' . $dbBatchJob->getObjectId()); switch ($dbBatchJob->getStatus()) { case BatchJob::BATCHJOB_STATUS_QUEUED: $jobData = $dbBatchJob->getData(); if (!is_null($jobData->getFilePath())) { $syncKey = $dbBatchJob->getSyncKey(BatchJob::FILE_SYNC_BATCHJOB_SUB_TYPE_BULKUPLOAD); try { kFileSyncUtils::moveFromFile($jobData->getFilePath(), $syncKey, true); } catch (Exception $e) { KalturaLog::err($e); throw new APIException(APIErrors::BULK_UPLOAD_CREATE_CSV_FILE_SYNC_ERROR); } $filePath = kFileSyncUtils::getLocalFilePathForKey($syncKey); $jobData->setFilePath($filePath); //save new info on the batch job $dbBatchJob->setData($jobData); $dbBatchJob->save(); } break; case BatchJob::BATCHJOB_STATUS_FINISHED: case BatchJob::BATCHJOB_STATUS_FINISHED_PARTIALLY: KalturaLog::debug("Handling Bulk Upload finished"); $xmlDropFolderFile->setStatus(DropFolderFileStatus::HANDLED); $xmlDropFolderFile->save(); break; case BatchJob::BATCHJOB_STATUS_FAILED: case BatchJob::BATCHJOB_STATUS_FATAL: KalturaLog::debug("Handling Bulk Upload failed"); $relatedFiles = DropFolderFilePeer::retrieveByLeadIdAndStatuses($xmlDropFolderFile->getId(), array(DropFolderFileStatus::PROCESSING)); foreach ($relatedFiles as $relatedFile) { $this->setFileError($relatedFile, DropFolderFileStatus::ERROR_HANDLING, DropFolderXmlBulkUploadPlugin::getErrorCodeCoreValue(DropFolderXmlBulkUploadErrorCode::ERROR_IN_BULK_UPLOAD), DropFolderXmlBulkUploadPlugin::ERROR_IN_BULK_UPLOAD_MESSAGE); } break; } }
/** * @return BatchJob */ public function createChild($same_root = true) { $child = new BatchJob(); $child->setStatus(self::BATCHJOB_STATUS_PENDING); $child->setParentJobId($this->id); $child->setPartnerId($this->partner_id); $child->setEntryId($this->entry_id); $child->setPriority($this->priority); $child->setSubpId($this->subp_id); $child->setBulkJobId($this->bulk_job_id); $child->setDc($this->dc); if ($same_root && $this->root_job_id) { $child->setRootJobId($this->root_job_id); } else { $child->setRootJobId($this->id); } $child->save(); return $child; }
/** * @param int $metadataProfileId * @param int $srcVersion * @param int $destVersion * @param string $xsl * * @return BatchJob */ private static function addTransformMetadataJob($partnerId, $metadataProfileId, $srcVersion, $destVersion, $xsl = null) { // check if any metadata objects require the transform $c = new Criteria(); $c->add(MetadataPeer::METADATA_PROFILE_ID, $metadataProfileId); $c->add(MetadataPeer::METADATA_PROFILE_VERSION, $destVersion, Criteria::LESS_THAN); $c->add(MetadataPeer::STATUS, Metadata::STATUS_VALID); $metadataCount = MetadataPeer::doCount($c); if (!$metadataCount) { return null; } $job = new BatchJob(); $job->setJobType(BatchJobType::METADATA_TRANSFORM); $job->setPartnerId($partnerId); $job->setObjectId($metadataProfileId); $job->setObjectType(kPluginableEnumsManager::apiToCore('BatchJobObjectType', MetadataBatchJobObjectType::METADATA_PROFILE)); $data = new kTransformMetadataJobData(); if ($xsl) { $job->save(); $key = $job->getSyncKey(BatchJob::FILE_SYNC_BATCHJOB_SUB_TYPE_CONFIG); kFileSyncUtils::file_put_contents($key, $xsl); $xslPath = kFileSyncUtils::getLocalFilePathForKey($key); $data->setSrcXslPath($xslPath); } $data->setMetadataProfileId($metadataProfileId); $data->setSrcVersion($srcVersion); $data->setDestVersion($destVersion); return kJobsManager::addJob($job, $data, BatchJobType::METADATA_TRANSFORM); }
/** * addFlavorConvertJob adds a single flavor conversion * * @param FileSyncKey $srcSyncKey * @param flavorParamsOutput $flavor * @param int $flavorAssetId * @param int $mediaInfoId * @param BatchJob $parentJob * @param int $lastEngineType * @param BatchJob $dbConvertFlavorJob * @return BatchJob */ public static function addFlavorConvertJob(FileSyncKey $srcSyncKey, flavorParamsOutput $flavor, $flavorAssetId, $mediaInfoId = null, BatchJob $parentJob = null, $lastEngineType = null, BatchJob $dbConvertFlavorJob = null) { list($fileSync, $local) = kFileSyncUtils::getReadyFileSyncForKey($srcSyncKey, true, false); $localPath = null; $remoteUrl = null; if ($fileSync) { if ($fileSync->getFileType() != FileSync::FILE_SYNC_FILE_TYPE_URL) { $localPath = $fileSync->getFullPath(); } $remoteUrl = $fileSync->getExternalUrl(); } // creates convert data $convertData = new kConvertJobData(); $convertData->setSrcFileSyncLocalPath($localPath); $convertData->setSrcFileSyncRemoteUrl($remoteUrl); $convertData->setMediaInfoId($mediaInfoId); $convertData->setFlavorParamsOutputId($flavor->getId()); $convertData->setFlavorAssetId($flavorAssetId); KalturaLog::log("Conversion engines string: '" . $flavor->getConversionEngines() . "'"); $currentConversionEngine = null; // TODO remove after all old version flavors migrated // parse supported engine types $conversionEngines = array(); if (!$flavor->getEngineVersion()) { $conversionEngines = explode(',', $flavor->getConversionEngines()); KalturaLog::log(count($conversionEngines) . " conversion engines found for the flavor"); $currentConversionEngine = reset($conversionEngines); // gets the first engine type } // remove until here if (is_null($lastEngineType)) { KalturaLog::log("Last Engine Type is null, engine version [" . $flavor->getEngineVersion() . "]"); if ($flavor->getEngineVersion()) { $operatorSet = new kOperatorSets(); $operatorSet->setSerialized($flavor->getOperators()); $nextOperator = $operatorSet->getOperator(); if (!$nextOperator) { KalturaLog::log("First operator is invalid"); return null; } KalturaLog::log("Set first operator in first set"); $currentConversionEngine = $nextOperator->id; } } else { if ($parentJob && $flavor->getEngineVersion() && ($parentJob->getJobType() == BatchJobType::CONVERT || $parentJob->getJobType() == BatchJobType::POSTCONVERT)) { // using next oprator KalturaLog::log("Adding next conversion operator"); $parentData = $parentJob->getData(); if (!$parentData || !$parentData instanceof kConvartableJobData) { KalturaLog::log("Parent job data is invalid"); return null; } $operatorSet = new kOperatorSets(); $operatorSet->setSerialized($flavor->getOperators()); $nextOperatorSet = $parentData->getCurrentOperationSet(); $nextOperatorIndex = $parentData->getCurrentOperationIndex() + 1; $nextOperator = $operatorSet->getOperator($nextOperatorSet, $nextOperatorIndex); if (!$nextOperator) { KalturaLog::log("Next operator is invalid"); return null; } KalturaLog::log("Moving to next operator [{$nextOperatorIndex}] in set [{$nextOperatorSet}]"); $convertData->setCurrentOperationSet($nextOperatorSet); $convertData->setCurrentOperationIndex($nextOperatorIndex); $currentConversionEngine = $nextOperator->id; } else { // TODO remove after all old version flavors migrated KalturaLog::log("Last used conversion engine is [{$lastEngineType}]"); // searching for $lastEngineType in the list while ($lastEngineType != $currentConversionEngine && next($conversionEngines)) { $currentConversionEngine = current($conversionEngines); } // takes the next engine $currentConversionEngine = next($conversionEngines); if (!$currentConversionEngine) { KalturaLog::log("There is no other conversion engine to use"); return null; } } } KalturaLog::log("Using conversion engine [{$currentConversionEngine}]"); // creats a child convert job if (is_null($dbConvertFlavorJob)) { if ($parentJob) { $dbConvertFlavorJob = $parentJob->createChild(); KalturaLog::log("Created from parent convert job with entry id [" . $dbConvertFlavorJob->getEntryId() . "]"); } else { $dbConvertFlavorJob = new BatchJob(); $dbConvertFlavorJob->setEntryId($flavor->getEntryId()); $dbConvertFlavorJob->setPartnerId($flavor->getPartnerId()); $dbConvertFlavorJob->save(); KalturaLog::log("Created from flavor convert job with entry id [" . $dbConvertFlavorJob->getEntryId() . "]"); } } $dbConvertFlavorJob->setFileSize(filesize($convertData->getSrcFileSyncLocalPath())); // TODO remove after all old version flavors migrated if (in_array(conversionEngineType::ENCODING_COM, $conversionEngines)) { $dbConvertFlavorJob->setOnStressDivertTo(conversionEngineType::ENCODING_COM); } // remove until here /* // Remarked by Dor until Tantan's return. // Code is supposed to get a configuration file from the engine and attach it to the batch job. // Was added for document conversion and is not used for now because of a bug of PDFCreator. KalturaLog::log("Calling CDLProceessFlavor with flavor params output[" . $flavor->getId() . "]"); $config = KDLWrap::CDLProceessFlavor($flavor); if($config) { $syncKey = $dbConvertFlavorJob->getSyncKey(BatchJob::FILE_SYNC_BATCHJOB_SUB_TYPE_CONFIG); kFileSyncUtils::file_put_contents($syncKey, $config); $fileSync = kFileSyncUtils::getLocalFileSyncForKey($syncKey); $remoteUrl = $fileSync->getExternalUrl(); $localPath = kFileSyncUtils::getLocalFilePathForKey($syncKey); $convertData->setConfigLocalPath($localPath); $convertData->setConfigRemoteUrl($remoteUrl); } */ $dbCurrentConversionEngine = kPluginableEnumsManager::apiToCore('conversionEngineType', $currentConversionEngine); return kJobsManager::addJob($dbConvertFlavorJob, $convertData, BatchJobType::CONVERT, $dbCurrentConversionEngine); }
/** * @param BatchJob $dbBatchJob * @param kCaptureThumbJobData $data * @param BatchJob $twinJob * @return BatchJob */ public static function handleCaptureThumbFinished(BatchJob $dbBatchJob, kCaptureThumbJobData $data, BatchJob $twinJob = null) { KalturaLog::debug("Captire thumbnail finished with destination file: " . $data->getThumbPath()); if ($dbBatchJob->getAbort()) { return $dbBatchJob; } // verifies that thumb asset created if (!$data->getThumbAssetId()) { throw new APIException(APIErrors::INVALID_THUMB_ASSET_ID, $data->getThumbAssetId()); } $thumbAsset = thumbAssetPeer::retrieveById($data->getThumbAssetId()); // verifies that thumb asset exists if (!$thumbAsset) { throw new APIException(APIErrors::INVALID_THUMB_ASSET_ID, $data->getThumbAssetId()); } $thumbAsset->incrementVersion(); $thumbAsset->setStatus(thumbAsset::FLAVOR_ASSET_STATUS_READY); if (file_exists($data->getThumbPath())) { list($width, $height, $type, $attr) = getimagesize($data->getThumbPath()); $thumbAsset->setWidth($width); $thumbAsset->setHeight($height); $thumbAsset->setSize(filesize($data->getThumbPath())); } $logPath = $data->getThumbPath() . '.log'; if (file_exists($logPath)) { $thumbAsset->incLogFileVersion(); $thumbAsset->save(); // creats the file sync $logSyncKey = $thumbAsset->getSyncKey(flavorAsset::FILE_SYNC_FLAVOR_ASSET_SUB_TYPE_CONVERT_LOG); try { kFileSyncUtils::moveFromFile($logPath, $logSyncKey); } catch (Exception $e) { $err = 'Saving conversion log: ' . $e->getMessage(); KalturaLog::err($err); $desc = $dbBatchJob->getDescription() . "\n" . $err; $dbBatchJob->getDescription($desc); } } else { $thumbAsset->save(); } $syncKey = $thumbAsset->getSyncKey(thumbAsset::FILE_SYNC_FLAVOR_ASSET_SUB_TYPE_ASSET); kFileSyncUtils::moveFromFile($data->getThumbPath(), $syncKey); $data->setThumbPath(kFileSyncUtils::getLocalFilePathForKey($syncKey)); KalturaLog::debug("Thumbnail archived file to: " . $data->getThumbPath()); // save the data changes to the db $dbBatchJob->setData($data); $dbBatchJob->save(); if ($thumbAsset->hasTag(thumbParams::TAG_DEFAULT_THUMB)) { $entry = $dbBatchJob->getEntry(false, false); if (!$entry) { throw new APIException(APIErrors::INVALID_ENTRY, $dbBatchJob, $dbBatchJob->getEntryId()); } // increment thumbnail version $entry->setThumbnail(".jpg"); $entry->save(); $entrySyncKey = $entry->getSyncKey(entry::FILE_SYNC_ENTRY_SUB_TYPE_THUMB); $syncFile = kFileSyncUtils::createSyncFileLinkForKey($entrySyncKey, $syncKey, false); if ($syncFile) { // removes the DEFAULT_THUMB tag from all other thumb assets $entryThumbAssets = thumbAssetPeer::retrieveByEntryId($thumbAsset->getEntryId()); foreach ($entryThumbAssets as $entryThumbAsset) { if ($entryThumbAsset->getId() == $thumbAsset->getId()) { continue; } if (!$entryThumbAsset->hasTag(thumbParams::TAG_DEFAULT_THUMB)) { continue; } $entryThumbAsset->removeTags(array(thumbParams::TAG_DEFAULT_THUMB)); $entryThumbAsset->save(); } } } if (!is_null($thumbAsset->getFlavorParamsId())) { kFlowHelper::generateThumbnailsFromFlavor($dbBatchJob->getEntryId(), $dbBatchJob, $thumbAsset->getFlavorParamsId()); } return $dbBatchJob; }
/** * @return BatchJob */ public function createChild($same_root = true, $dc = null) { $child = new BatchJob(); $child->setStatus(self::BATCHJOB_STATUS_PENDING); $child->setParentJobId($this->id); $child->setPartnerId($this->partner_id); $child->setEntryId($this->entry_id); $child->setPriority($this->priority); $child->setSubpId($this->subp_id); $child->setBulkJobId($this->bulk_job_id); // the condition is required in the special case of file_sync import jobs which are created on one dc but run from the other $child->setDc($dc === null ? $this->dc : $dc); if ($same_root && $this->root_job_id) { $child->setRootJobId($this->root_job_id); } else { $child->setRootJobId($this->id); } $child->save(); return $child; }
/** * Function adds bulk upload job to the queue * @param Partner $partner * @param kBulkUploadJobData $jobData * @param string $bulkUploadType * @throws APIException * @return BatchJob */ public static function addBulkUploadJob(Partner $partner, kBulkUploadJobData $jobData, $bulkUploadType = null) { $job = new BatchJob(); $job->setPartnerId($partner->getId()); $job->setJobType(BatchJobType::BULKUPLOAD); $job->setJobSubType($bulkUploadType); $job->save(); $syncKey = $job->getSyncKey(BatchJob::FILE_SYNC_BATCHJOB_SUB_TYPE_BULKUPLOAD); // kFileSyncUtils::file_put_contents($syncKey, file_get_contents($csvFileData["tmp_name"])); try { kFileSyncUtils::moveFromFile($jobData->getFilePath(), $syncKey, true); } catch (Exception $e) { throw new APIException(APIErrors::BULK_UPLOAD_CREATE_CSV_FILE_SYNC_ERROR); } $filePath = kFileSyncUtils::getLocalFilePathForKey($syncKey); if (is_null($jobData)) { throw new APIException(APIErrors::BULK_UPLOAD_BULK_UPLOAD_TYPE_NOT_VALID, $bulkUploadType); } if (!$jobData->getBulkUploadObjectType()) { $jobData->setBulkUploadObjectType(BulkUploadObjectType::ENTRY); } $jobData->setFilePath($filePath); if ($jobData->getBulkUploadObjectType() == BulkUploadObjectType::ENTRY && !$jobData->getObjectData()->getConversionProfileId()) { $jobData->setConversionProfileId($partner->getDefaultConversionProfileId()); $kmcVersion = $partner->getKmcVersion(); $check = null; if ($kmcVersion < 2) { $check = ConversionProfilePeer::retrieveByPK($jobData->getConversionProfileId()); } else { $check = conversionProfile2Peer::retrieveByPK($jobData->getConversionProfileId()); } if (!$check) { throw new APIException(APIErrors::CONVERSION_PROFILE_ID_NOT_FOUND, $jobData->getConversionProfileId()); } } return kJobsManager::addJob($job, $jobData, BatchJobType::BULKUPLOAD, kPluginableEnumsManager::apiToCore("BulkUploadType", $bulkUploadType)); }
/** * Performs the work of inserting or updating the row in the database. * * If the object is new, it inserts it; otherwise an update is performed. * All related objects are also updated in this method. * * @param PropelPDO $con * @return int The number of rows affected by this insert/update and any referring fk objects' save() operations. * @throws PropelException * @see save() */ protected function doSave(PropelPDO $con) { $affectedRows = 0; // initialize var to track total num of affected rows if (!$this->alreadyInSave) { $this->alreadyInSave = true; // We call the save method on the following object(s) if they // were passed to this object by their coresponding set // method. This object relates to these object(s) by a // foreign key reference. if ($this->aBatchJob !== null) { if ($this->aBatchJob->isModified() || $this->aBatchJob->isNew()) { $affectedRows += $this->aBatchJob->save($con); } $this->setBatchJob($this->aBatchJob); } // If this object has been modified, then save it to the database. $this->objectSaved = false; if ($this->isModified()) { if ($this->isNew()) { $pk = BatchJobLockSuspendPeer::doInsert($this, $con); $affectedRows += 1; // we are assuming that there is only 1 row per doInsert() which // should always be true here (even though technically // BasePeer::doInsert() can insert multiple rows). $this->setNew(false); $this->objectSaved = true; } else { $affectedObjects = BatchJobLockSuspendPeer::doUpdate($this, $con); if ($affectedObjects) { $this->objectSaved = true; } $affectedRows += $affectedObjects; } $this->resetModified(); // [HL] After being saved an object is no longer 'modified' } $this->alreadyInSave = false; } return $affectedRows; }
/** * @param BatchJob $dbBatchJob * @param BatchJob $twinJob * @return bool true if should continue to the next consumer */ public function updatedJob(BatchJob $dbBatchJob, BatchJob $twinJob = null) { try { $jobType = $dbBatchJob->getJobType(); if (is_null($dbBatchJob->getQueueTime()) && $dbBatchJob->getStatus() != BatchJob::BATCHJOB_STATUS_PENDING) { $dbBatchJob->setQueueTime(time()); $dbBatchJob->save(); } if ($dbBatchJob->getStatus() == BatchJob::BATCHJOB_STATUS_FINISHED) { $dbBatchJob->setFinishTime(time()); $dbBatchJob->save(); } if ($dbBatchJob->getStatus() == BatchJob::BATCHJOB_STATUS_RETRY) { $dbBatchJob->setCheckAgainTimeout(time() + BatchJobPeer::getCheckAgainTimeout($jobType)); $dbBatchJob->setQueueTime(null); $dbBatchJob->save(); } if ($dbBatchJob->getStatus() == BatchJob::BATCHJOB_STATUS_ALMOST_DONE) { $dbBatchJob->setCheckAgainTimeout(time() + BatchJobPeer::getCheckAgainTimeout($jobType)); $dbBatchJob->save(); } if ($dbBatchJob->getStatus() == BatchJob::BATCHJOB_STATUS_FAILED || $dbBatchJob->getStatus() == BatchJob::BATCHJOB_STATUS_FATAL) { $dbBatchJob->setFinishTime(time()); $dbBatchJob->save(); // TODO - don't abort if it's bulk upload kJobsManager::abortChildJobs($dbBatchJob); } switch ($jobType) { case BatchJobType::IMPORT: $dbBatchJob = $this->updatedImport($dbBatchJob, $dbBatchJob->getData(), $twinJob); break; case BatchJobType::EXTRACT_MEDIA: $dbBatchJob = $this->updatedExtractMedia($dbBatchJob, $dbBatchJob->getData(), $twinJob); break; case BatchJobType::CONVERT: $dbBatchJob = $this->updatedConvert($dbBatchJob, $dbBatchJob->getData(), $twinJob); break; case BatchJobType::POSTCONVERT: $dbBatchJob = $this->updatedPostConvert($dbBatchJob, $dbBatchJob->getData(), $twinJob); break; case BatchJobType::BULKUPLOAD: $dbBatchJob = $this->updatedBulkUpload($dbBatchJob, $dbBatchJob->getData(), $twinJob); break; case BatchJobType::CONVERT_PROFILE: $dbBatchJob = $this->updatedConvertProfile($dbBatchJob, $dbBatchJob->getData(), $twinJob); break; case BatchJobType::BULKDOWNLOAD: $dbBatchJob = $this->updatedBulkDownload($dbBatchJob, $dbBatchJob->getData(), $twinJob); break; case BatchJobType::PROVISION_PROVIDE: $dbBatchJob = $this->updatedProvisionProvide($dbBatchJob, $dbBatchJob->getData(), $twinJob); break; case BatchJobType::PROVISION_DELETE: $dbBatchJob = $this->updatedProvisionDelete($dbBatchJob, $dbBatchJob->getData(), $twinJob); break; case BatchJobType::CONVERT_COLLECTION: $dbBatchJob = $this->updatedConvertCollection($dbBatchJob, $dbBatchJob->getData(), $twinJob); break; case BatchJobType::STORAGE_EXPORT: $dbBatchJob = $this->updatedStorageExport($dbBatchJob, $dbBatchJob->getData(), $twinJob); break; case BatchJobType::STORAGE_DELETE: $dbBatchJob = $this->updatedStorageDelete($dbBatchJob, $dbBatchJob->getData(), $twinJob); break; case BatchJobType::CAPTURE_THUMB: $dbBatchJob = $this->updatedCaptureThumb($dbBatchJob, $dbBatchJob->getData(), $twinJob); break; default: break; } if (!kConf::get("batch_ignore_duplication")) { if ($dbBatchJob->getStatus() == BatchJob::BATCHJOB_STATUS_FINISHED) { $twinBatchJobs = $dbBatchJob->getTwinJobs(); // update status at all twin jobs foreach ($twinBatchJobs as $twinBatchJob) { if ($twinBatchJob->getStatus() != BatchJob::BATCHJOB_STATUS_FINISHED) { kJobsManager::updateBatchJob($twinBatchJob, BatchJob::BATCHJOB_STATUS_FINISHED); } } } } if ($dbBatchJob->getStatus() == BatchJob::BATCHJOB_STATUS_RETRY && $dbBatchJob->getExecutionAttempts() >= BatchJobPeer::getMaxExecutionAttempts($jobType)) { $dbBatchJob = kJobsManager::updateBatchJob($dbBatchJob, BatchJob::BATCHJOB_STATUS_FAILED); } } catch (Exception $ex) { self::alert($dbBatchJob, $ex); KalturaLog::err("Error:" . $ex->getMessage()); } return true; }