public static function decideProfileFlavorsConvert(BatchJob $parentJob, BatchJob $convertProfileJob, array $flavors, array $conversionProfileFlavorParams, mediaInfo $mediaInfo = null)
{
$entryId = $convertProfileJob->getEntryId();
$originalFlavorAsset = flavorAssetPeer::retrieveOriginalByEntryId($entryId);
if (is_null($originalFlavorAsset)) {
$errDescription = 'Original flavor asset not found';
KalturaLog::err($errDescription);
$convertProfileJob = kJobsManager::failBatchJob($convertProfileJob, $errDescription, BatchJobType::CONVERT_PROFILE);
kBatchManager::updateEntry($convertProfileJob, entryStatus::ERROR_CONVERTING);
return false;
}
$errDescription = null;
$finalFlavors = self::validateConversionProfile($convertProfileJob->getPartnerId(), $entryId, $mediaInfo, $flavors, $conversionProfileFlavorParams, $errDescription);
KalturaLog::log(count($finalFlavors) . " flavors returned from the decision layer");
if (is_null($finalFlavors)) {
$convertProfileJob = kJobsManager::failBatchJob($convertProfileJob, $errDescription);
KalturaLog::log("No flavors created");
throw new Exception($errDescription);
}
if (strlen($errDescription)) {
$err = $convertProfileJob->getDescription() . $errDescription;
$convertProfileJob->setDescription($err);
$convertProfileJob->save();
}
$srcSyncKey = $originalFlavorAsset->getSyncKey(flavorAsset::FILE_SYNC_FLAVOR_ASSET_SUB_TYPE_ASSET);
$conversionsCreated = 0;
$entry = $convertProfileJob->getEntry();
if (!$entry) {
throw new APIException(APIErrors::INVALID_ENTRY, $convertProfileJob, $entryId);
}
$flavorsCollections = array();
// create a convert job per each flavor
foreach ($finalFlavors as $flavor) {
$flavorAsset = kBatchManager::createFlavorAsset($flavor, $entry->getPartnerId(), $entry->getId());
if (!$flavorAsset) {
KalturaLog::log("Flavor asset could not be created, flavor conversion won't be created");
continue;
}
$collectionTag = $flavor->getCollectionTag();
if ($collectionTag) {
$flavorsCollections[$collectionTag][] = $flavor;
continue;
}
KalturaLog::log("Adding flavor conversion with flavor params output id [" . $flavor->getId() . "] and flavor params asset id [" . $flavorAsset->getId() . "]");
$madiaInfoId = $mediaInfo ? $mediaInfo->getId() : null;
$createdJob = kJobsManager::addFlavorConvertJob($srcSyncKey, $flavor, $flavorAsset->getId(), $madiaInfoId, $parentJob);
if ($createdJob) {
$conversionsCreated++;
}
}
foreach ($flavorsCollections as $tag => $flavors) {
switch ($tag) {
case flavorParams::TAG_ISM:
$createdJob = kJobsManager::addConvertIsmCollectionJob($tag, $srcSyncKey, $entry, $parentJob, $flavors);
if ($createdJob) {
$conversionsCreated++;
}
break;
default:
KalturaLog::log("Error: Invalid collection tag [{$tag}]");
break;
}
}
if (!$conversionsCreated) {
$convertProfileJob = kJobsManager::failBatchJob($convertProfileJob, $errDescription);
KalturaLog::log("No flavors created: {$errDescription}");
return false;
}
return true;
}
protected function updatedVirusScanFailed(BatchJob $dbBatchJob, kVirusScanJobData $data, BatchJob $twinJob = null)
{
$entry = entryPeer::retrieveByPK($dbBatchJob->getEntryId());
if ($entry) {
$entry->setStatus(VirusScanPlugin::getEntryStatusCoreValue(VirusScanEntryStatus::INFECTED));
$entry->save();
} else {
KalturaLog::err('Entry not found with id [' . $dbBatchJob->getEntryId() . ']');
throw new Exception('Entry not found with id [' . $dbBatchJob->getEntryId() . ']');
}
$flavorAsset = flavorAssetPeer::retrieveById($data->getFlavorAssetId());
if ($flavorAsset) {
$flavorAsset->setStatus(flavorAsset::FLAVOR_ASSET_STATUS_ERROR);
$flavorAsset->save();
} else {
KalturaLog::err('Flavor asset not found with id [' . $data->getFlavorAssetId() . ']');
throw new Exception('Flavor asset not found with id [' . $data->getFlavorAssetId() . ']');
}
// do not resume flavor asset added event consumption
return $dbBatchJob;
}
public static function handleConvertProfileFinished(BatchJob $dbBatchJob, kConvertProfileJobData $data, BatchJob $twinJob = null)
{
KalturaLog::debug("Convert Profile finished");
$originalflavorAsset = flavorAssetPeer::retrieveOriginalByEntryId($dbBatchJob->getEntryId());
if ($originalflavorAsset->getStatus() == flavorAsset::FLAVOR_ASSET_STATUS_TEMP) {
$originalflavorAsset->setStatus(flavorAsset::FLAVOR_ASSET_STATUS_DELETED);
$originalflavorAsset->setDeletedAt(time());
$originalflavorAsset->save();
}
kBatchManager::updateEntry($dbBatchJob, entryStatus::READY);
kFlowHelper::generateThumbnailsFromFlavor($dbBatchJob->getEntryId(), $dbBatchJob);
return $dbBatchJob;
}
/**
* @param BatchJob $dbBatchJob
* @param kConvertCollectionJobData $data
* @param int $engineType
* @return boolean
*/
public static function handleConvertCollectionFailed(BatchJob $dbBatchJob, kConvertCollectionJobData $data, $engineType)
{
$collectionFlavors = array();
foreach ($data->getFlavors() as $flavor) {
$collectionFlavors[$flavor->getFlavorAssetId()] = $flavor;
}
// find the root job
$rootBatchJob = $dbBatchJob->getRootJob();
$hasIncomplete = false;
$shouldFailProfile = false;
$flavorAssets = assetPeer::retrieveFlavorsByEntryId($dbBatchJob->getEntryId());
foreach ($flavorAssets as $flavorAsset) {
if (isset($collectionFlavors[$flavorAsset->getId()])) {
$flavorAsset->setStatus(flavorAsset::FLAVOR_ASSET_STATUS_ERROR);
$flavorAsset->save();
if (!$rootBatchJob) {
continue;
}
$flavorData = $collectionFlavors[$flavorAsset->getId()];
if ($flavorData->getReadyBehavior() == flavorParamsConversionProfile::READY_BEHAVIOR_REQUIRED) {
$shouldFailProfile = true;
}
continue;
}
if ($flavorAsset->getIsOriginal()) {
continue;
}
if ($flavorAsset->getStatus() == flavorAsset::FLAVOR_ASSET_STATUS_QUEUED || $flavorAsset->getStatus() == flavorAsset::FLAVOR_ASSET_STATUS_CONVERTING || $flavorAsset->getStatus() == flavorAsset::FLAVOR_ASSET_STATUS_IMPORTING || $flavorAsset->getStatus() == flavorAsset::FLAVOR_ASSET_STATUS_VALIDATING) {
$hasIncomplete = true;
}
}
if (!$rootBatchJob) {
return false;
}
if ($rootBatchJob->getJobType() != BatchJobType::CONVERT_PROFILE) {
return false;
}
if ($shouldFailProfile || !$hasIncomplete) {
kJobsManager::failBatchJob($rootBatchJob, "Job " . $dbBatchJob->getId() . " failed");
}
return false;
}
/**
* @param BatchJob $dbBatchJob
* @param kImportJobData $data
* @param BatchJob $twinJob
* @return BatchJob
*/
public static function onImportJobFailed(BatchJob $dbBatchJob, kImportJobData $data, BatchJob $twinJob = null)
{
$statuses = array(EntryDistributionStatus::IMPORT_SUBMITTING, EntryDistributionStatus::IMPORT_UPDATING);
$entryDistributions = EntryDistributionPeer::retrieveByEntryAndStatuses($dbBatchJob->getEntryId(), $statuses);
foreach ($entryDistributions as $entryDistribution) {
/* @var $entryDistribution EntryDistribution */
if ($entryDistribution->getStatus() == EntryDistributionStatus::IMPORT_SUBMITTING) {
$entryDistribution->setStatus(EntryDistributionStatus::ERROR_SUBMITTING);
} elseif ($entryDistribution->getStatus() == EntryDistributionStatus::IMPORT_UPDATING) {
$entryDistribution->setStatus(EntryDistributionStatus::ERROR_UPDATING);
}
$entryDistribution->setErrorType($dbBatchJob->getErrType());
$entryDistribution->setErrorNumber($dbBatchJob->getErrNumber());
$entryDistribution->setErrorDescription($dbBatchJob->getMessage());
$entryDistribution->setDirtyStatus(null);
$entryDistribution->save();
}
}
/**
* @param BatchJob $batchJob
* @param $data
* @param int $type
* @param int $subType
* @return BatchJob
*/
public static function addJob(BatchJob $batchJob, kJobData $data, $type, $subType = null)
{
$batchJob->setJobType($type);
$batchJob->setJobSubType($subType);
$batchJob->setData($data);
if (!$batchJob->getParentJobId() && $batchJob->getEntryId()) {
$currentJob = kBatchManager::getCurrentUpdatingJob();
if ($currentJob && $currentJob->getEntryId() == $batchJob->getEntryId()) {
$batchJob->setParentJobId($currentJob->getId());
$batchJob->setBulkJobId($currentJob->getBulkJobId());
$batchJob->setRootJobId($currentJob->getRootJobId());
} else {
$entry = entryPeer::retrieveByPKNoFilter($batchJob->getEntryId());
// some jobs could be on deleted entry
if ($entry) {
$batchJob->setRootJobId($entry->getBulkUploadId());
$batchJob->setBulkJobId($entry->getBulkUploadId());
}
}
}
$lockInfo = new kLockInfoData($batchJob);
$lockInfo->setEstimatedEffort($data->calculateEstimatedEffort($batchJob));
$lockInfo->setPriority($data->calculatePriority($batchJob));
$lockInfo->setUrgency($data->calculateUrgency($batchJob));
$batchJob->setLockInfo($lockInfo);
if (is_null($batchJob->getStatus())) {
$batchJob = self::updateBatchJob($batchJob, BatchJob::BATCHJOB_STATUS_PENDING);
} else {
$batchJob = self::updateBatchJob($batchJob, $batchJob->getStatus());
}
return $batchJob;
}
public static function decideProfileFlavorsConvert(BatchJob $parentJob, BatchJob $convertProfileJob, array $flavors, array $conversionProfileFlavorParams, $conversionProfileId, mediaInfo $mediaInfo = null)
{
$entryId = $convertProfileJob->getEntryId();
$originalFlavorAsset = assetPeer::retrieveOriginalByEntryId($entryId);
if (is_null($originalFlavorAsset)) {
$errDescription = 'Original flavor asset not found';
self::setError($errDescription, $convertProfileJob, BatchJobType::CONVERT_PROFILE, $convertProfileJob->getEntryId());
return false;
}
$errDescription = null;
try {
$finalFlavors = self::validateConversionProfile($convertProfileJob->getPartnerId(), $entryId, $mediaInfo, $flavors, $conversionProfileFlavorParams, $errDescription);
} catch (Exception $e) {
$code = $e->getCode();
if ($code == KDLErrors::SanityInvalidFrameDim || $code == KDLErrors::NoValidMediaStream) {
$convertProfileJob = kJobsManager::failBatchJob($convertProfileJob, $errDescription);
KalturaLog::err($e->getMessage());
throw $e;
}
}
KalturaLog::log(count($finalFlavors) . " flavors returned from the decision layer");
if (is_null($finalFlavors)) {
$convertProfileJob = kJobsManager::failBatchJob($convertProfileJob, $errDescription);
KalturaLog::log("No flavors created");
//throw new Exception($errDescription); no need to throw alert if the root job failed
}
if (strlen($errDescription)) {
$err = $convertProfileJob->getDescription() . $errDescription;
$convertProfileJob->setDescription($err);
$convertProfileJob->save();
//Check if the error thrown is invalid file - no media content
if (strpos($errDescription, KDLErrors::ToString(KDLErrors::NoValidMediaStream)) !== false) {
throw new Exception(KDLErrors::ToString(KDLErrors::NoValidMediaStream), KDLErrors::NoValidMediaStream);
}
}
$conversionsCreated = 0;
$waitingAssets = 0;
$entry = $convertProfileJob->getEntry();
if (!$entry) {
throw new APIException(APIErrors::INVALID_ENTRY, $convertProfileJob, $entryId);
}
$flavorsCollections = array();
// create a convert job per each flavor
foreach ($finalFlavors as $flavor) {
$flavorAsset = kBatchManager::createFlavorAsset($flavor, $entry->getPartnerId(), $entry->getId());
if (!$flavorAsset) {
continue;
}
$collectionTag = $flavor->getCollectionTag();
/*
* CHANGE: collection porcessing only for ExpressionEncoder jobs
* to allow FFmpeg/ISMV processing
*/
KalturaLog::log("Check for collection case - engines(" . $flavor->getConversionEngines() . ")");
if ($collectionTag && $flavor->getConversionEngines() == conversionEngineType::EXPRESSION_ENCODER3) {
$flavorsCollections[$collectionTag][] = $flavor;
} else {
KalturaLog::log("Adding flavor conversion with flavor params output id [" . $flavor->getId() . "] and flavor params asset id [" . $flavorAsset->getId() . "]");
$madiaInfoId = $mediaInfo ? $mediaInfo->getId() : null;
$createdJob = self::decideFlavorConvert($flavorAsset, $flavor, $originalFlavorAsset, $conversionProfileId, $madiaInfoId, $parentJob);
if ($createdJob) {
$conversionsCreated++;
}
if ($flavorAsset->getStatus() == flavorAsset::ASSET_STATUS_WAIT_FOR_CONVERT) {
$waitingAssets++;
}
}
}
foreach ($flavorsCollections as $tag => $flavors) {
$createdJob = self::decideCollectionConvert($tag, $originalFlavorAsset, $entry, $parentJob, $flavors);
if ($createdJob) {
$conversionsCreated++;
}
}
if (!$conversionsCreated && !$waitingAssets) {
KalturaLog::log("No flavors created: {$errDescription}");
$convertProfileJob = kJobsManager::failBatchJob($convertProfileJob, $errDescription);
return false;
}
return true;
}
public static function handleProvisionProvideFailed(BatchJob $dbBatchJob, kProvisionJobData $data)
{
kBatchManager::updateEntry($dbBatchJob->getEntryId(), entryStatus::ERROR_CONVERTING);
return $dbBatchJob;
}
/**
* @param BatchJob $batchJob
* @param $data
* @param int $type
* @param int $subType
* @return BatchJob
*/
public static function addJob(BatchJob $batchJob, $data, $type, $subType = null)
{
$batchJob->setJobType($type);
$batchJob->setJobSubType($subType);
$batchJob->setData($data);
if (!$batchJob->getParentJobId() && $batchJob->getEntryId()) {
$currentJob = kBatchManager::getCurrentUpdatingJob();
if ($currentJob && $currentJob->getEntryId() == $batchJob->getEntryId()) {
$batchJob->setParentJobId($currentJob->getId());
$batchJob->setBulkJobId($currentJob->getBulkJobId());
$batchJob->setRootJobId($currentJob->getRootJobId());
} else {
$entry = entryPeer::retrieveByPKNoFilter($batchJob->getEntryId());
// some jobs could be on deleted entry
if ($entry) {
$batchJob->setRootJobId($entry->getBulkUploadId());
$batchJob->setBulkJobId($entry->getBulkUploadId());
}
}
}
// validate partner id
$partnerId = $batchJob->getPartnerId();
// if(!$partnerId)
// throw new APIException(APIErrors::PARTNER_NOT_SET);
// validate that partner exists
$partner = PartnerPeer::retrieveByPK($partnerId);
if (!$partner) {
KalturaLog::err("Invalid partner id [{$partnerId}]");
throw new APIException(APIErrors::INVALID_PARTNER_ID, $partnerId);
}
// set the priority and work group
$batchJob->setPriority($partner->getPriority($batchJob->getBulkJobId()));
$batchJob = self::updateBatchJob($batchJob, BatchJob::BATCHJOB_STATUS_PENDING);
// look for identical jobs
$twinJobs = BatchJobPeer::retrieveDuplicated($type, $data);
$twinJob = null;
if (count($twinJobs)) {
foreach ($twinJobs as $currentTwinJob) {
if ($currentTwinJob->getId() != $batchJob->getId()) {
$twinJob = reset($twinJobs);
}
}
}
if (!is_null($twinJob)) {
$batchJob->setTwinJobId($twinJob->getId());
if (!kConf::get("batch_ignore_duplication")) {
$batchJob = self::updateBatchJob($batchJob, $twinJob->getStatus(), $twinJob);
} else {
$batchJob->save();
}
}
return $batchJob;
}
protected function updatedVirusScanFailed(BatchJob $dbBatchJob, kVirusScanJobData $data)
{
$entry = entryPeer::retrieveByPKNoFilter($dbBatchJob->getEntryId());
if ($entry) {
self::setEntryStatusBeforeScanFailure($entry, $entry->getStatus());
$entry->setStatus(VirusScanPlugin::getEntryStatusCoreValue(VirusScanEntryStatus::SCAN_FAILURE));
$entry->save();
myNotificationMgr::createNotification(kNotificationJobData::NOTIFICATION_TYPE_ENTRY_UPDATE, $entry);
} else {
KalturaLog::err('Entry not found with id [' . $dbBatchJob->getEntryId() . ']');
throw new Exception('Entry not found with id [' . $dbBatchJob->getEntryId() . ']');
}
$flavorAsset = assetPeer::retrieveById($data->getFlavorAssetId());
if ($flavorAsset) {
self::setFlavorAssetStatusBeforeScanFailure($flavorAsset, $flavorAsset->getStatus());
$flavorAsset->setStatus(flavorAsset::FLAVOR_ASSET_STATUS_ERROR);
$flavorAsset->save();
} else {
KalturaLog::err('Flavor asset not found with id [' . $data->getFlavorAssetId() . ']');
throw new Exception('Flavor asset not found with id [' . $data->getFlavorAssetId() . ']');
}
// do not resume flavor asset added event consumption
return $dbBatchJob;
}
public static function exportSourceAssetFromJob(BatchJob $dbBatchJob)
{
// convert profile finished - export source flavor
if ($dbBatchJob->getJobType() == BatchJobType::CONVERT_PROFILE) {
$externalStorages = StorageProfilePeer::retrieveAutomaticByPartnerId($dbBatchJob->getPartnerId());
$sourceFlavor = assetPeer::retrieveOriginalByEntryId($dbBatchJob->getEntryId());
if (!$sourceFlavor) {
KalturaLog::debug('Cannot find source flavor for entry id [' . $dbBatchJob->getEntryId() . ']');
} else {
if (!$sourceFlavor->isLocalReadyStatus()) {
KalturaLog::debug('Source flavor id [' . $sourceFlavor->getId() . '] has status [' . $sourceFlavor->getStatus() . '] - not ready for export');
} else {
foreach ($externalStorages as $externalStorage) {
if ($externalStorage->triggerFitsReadyAsset($dbBatchJob->getEntryId())) {
self::exportFlavorAsset($sourceFlavor, $externalStorage);
}
}
}
}
}
// convert collection finished - export ism and ismc files
if ($dbBatchJob->getJobType() == BatchJobType::CONVERT_COLLECTION && $dbBatchJob->getJobSubType() == conversionEngineType::EXPRESSION_ENCODER3) {
$entry = $dbBatchJob->getEntry();
$externalStorages = StorageProfilePeer::retrieveAutomaticByPartnerId($dbBatchJob->getPartnerId());
foreach ($externalStorages as $externalStorage) {
if ($externalStorage->triggerFitsReadyAsset($entry->getId())) {
$ismKey = $entry->getSyncKey(entry::FILE_SYNC_ENTRY_SUB_TYPE_ISM);
if (kFileSyncUtils::fileSync_exists($ismKey)) {
self::export($entry, $externalStorage, $ismKey);
}
$ismcKey = $entry->getSyncKey(entry::FILE_SYNC_ENTRY_SUB_TYPE_ISMC);
if (kFileSyncUtils::fileSync_exists($ismcKey)) {
self::export($entry, $externalStorage, $ismcKey);
}
}
}
}
return true;
}
/**
* @param BatchJob $dbBatchJob
* @param BatchJob $twinJob
* @return bool true if should continue to the next consumer
*/
public function updatedJob(BatchJob $dbBatchJob, BatchJob $twinJob = null)
{
if ($dbBatchJob->getStatus() != BatchJob::BATCHJOB_STATUS_FINISHED) {
return true;
}
// convert profile finished - export source flavor
if ($dbBatchJob->getJobType() == BatchJobType::CONVERT_PROFILE) {
$externalStorages = StorageProfilePeer::retrieveAutomaticByPartnerId($dbBatchJob->getPartnerId());
foreach ($externalStorages as $externalStorage) {
if ($externalStorage->getTrigger() == StorageProfile::STORAGE_TEMP_TRIGGER_FLAVOR_READY || $externalStorage->getTrigger() == StorageProfile::STORAGE_TEMP_TRIGGER_MODERATION_APPROVED && $dbBatchJob->getEntry()->getModerationStatus() == entry::ENTRY_MODERATION_STATUS_APPROVED) {
$sourceFlavor = flavorAssetPeer::retrieveOriginalReadyByEntryId($dbBatchJob->getEntryId());
if ($sourceFlavor) {
$this->exportFlavorAsset($sourceFlavor, $externalStorage);
}
}
}
}
// convert collection finished - export ism and ismc files
if ($dbBatchJob->getJobType() == BatchJobType::CONVERT_COLLECTION && $dbBatchJob->getJobSubType() == conversionEngineType::EXPRESSION_ENCODER3) {
$entry = $dbBatchJob->getEntry();
$externalStorages = StorageProfilePeer::retrieveAutomaticByPartnerId($dbBatchJob->getPartnerId());
foreach ($externalStorages as $externalStorage) {
if ($externalStorage->getTrigger() == StorageProfile::STORAGE_TEMP_TRIGGER_FLAVOR_READY || $externalStorage->getTrigger() == StorageProfile::STORAGE_TEMP_TRIGGER_MODERATION_APPROVED && $entry->getModerationStatus() == entry::ENTRY_MODERATION_STATUS_APPROVED) {
$ismKey = $entry->getSyncKey(entry::FILE_SYNC_ENTRY_SUB_TYPE_ISM);
if (kFileSyncUtils::fileSync_exists($ismKey)) {
$this->export($entry, $externalStorage, $ismKey);
}
$ismcKey = $entry->getSyncKey(entry::FILE_SYNC_ENTRY_SUB_TYPE_ISMC);
if (kFileSyncUtils::fileSync_exists($ismcKey)) {
$this->export($entry, $externalStorage, $ismcKey);
}
}
}
}
return true;
}
