public static function decideProfileFlavorsConvert(BatchJob $parentJob, BatchJob $convertProfileJob, array $flavors, array $conversionProfileFlavorParams, mediaInfo $mediaInfo = null) { $entryId = $convertProfileJob->getEntryId(); $originalFlavorAsset = flavorAssetPeer::retrieveOriginalByEntryId($entryId); if (is_null($originalFlavorAsset)) { $errDescription = 'Original flavor asset not found'; KalturaLog::err($errDescription); $convertProfileJob = kJobsManager::failBatchJob($convertProfileJob, $errDescription, BatchJobType::CONVERT_PROFILE); kBatchManager::updateEntry($convertProfileJob, entryStatus::ERROR_CONVERTING); return false; } $errDescription = null; $finalFlavors = self::validateConversionProfile($convertProfileJob->getPartnerId(), $entryId, $mediaInfo, $flavors, $conversionProfileFlavorParams, $errDescription); KalturaLog::log(count($finalFlavors) . " flavors returned from the decision layer"); if (is_null($finalFlavors)) { $convertProfileJob = kJobsManager::failBatchJob($convertProfileJob, $errDescription); KalturaLog::log("No flavors created"); throw new Exception($errDescription); } if (strlen($errDescription)) { $err = $convertProfileJob->getDescription() . $errDescription; $convertProfileJob->setDescription($err); $convertProfileJob->save(); } $srcSyncKey = $originalFlavorAsset->getSyncKey(flavorAsset::FILE_SYNC_FLAVOR_ASSET_SUB_TYPE_ASSET); $conversionsCreated = 0; $entry = $convertProfileJob->getEntry(); if (!$entry) { throw new APIException(APIErrors::INVALID_ENTRY, $convertProfileJob, $entryId); } $flavorsCollections = array(); // create a convert job per each flavor foreach ($finalFlavors as $flavor) { $flavorAsset = kBatchManager::createFlavorAsset($flavor, $entry->getPartnerId(), $entry->getId()); if (!$flavorAsset) { KalturaLog::log("Flavor asset could not be created, flavor conversion won't be created"); continue; } $collectionTag = $flavor->getCollectionTag(); if ($collectionTag) { $flavorsCollections[$collectionTag][] = $flavor; continue; } KalturaLog::log("Adding flavor conversion with flavor params output id [" . $flavor->getId() . "] and flavor params asset id [" . $flavorAsset->getId() . "]"); $madiaInfoId = $mediaInfo ? $mediaInfo->getId() : null; $createdJob = kJobsManager::addFlavorConvertJob($srcSyncKey, $flavor, $flavorAsset->getId(), $madiaInfoId, $parentJob); if ($createdJob) { $conversionsCreated++; } } foreach ($flavorsCollections as $tag => $flavors) { switch ($tag) { case flavorParams::TAG_ISM: $createdJob = kJobsManager::addConvertIsmCollectionJob($tag, $srcSyncKey, $entry, $parentJob, $flavors); if ($createdJob) { $conversionsCreated++; } break; default: KalturaLog::log("Error: Invalid collection tag [{$tag}]"); break; } } if (!$conversionsCreated) { $convertProfileJob = kJobsManager::failBatchJob($convertProfileJob, $errDescription); KalturaLog::log("No flavors created: {$errDescription}"); return false; } return true; }
protected function updatedVirusScanFailed(BatchJob $dbBatchJob, kVirusScanJobData $data, BatchJob $twinJob = null) { $entry = entryPeer::retrieveByPK($dbBatchJob->getEntryId()); if ($entry) { $entry->setStatus(VirusScanPlugin::getEntryStatusCoreValue(VirusScanEntryStatus::INFECTED)); $entry->save(); } else { KalturaLog::err('Entry not found with id [' . $dbBatchJob->getEntryId() . ']'); throw new Exception('Entry not found with id [' . $dbBatchJob->getEntryId() . ']'); } $flavorAsset = flavorAssetPeer::retrieveById($data->getFlavorAssetId()); if ($flavorAsset) { $flavorAsset->setStatus(flavorAsset::FLAVOR_ASSET_STATUS_ERROR); $flavorAsset->save(); } else { KalturaLog::err('Flavor asset not found with id [' . $data->getFlavorAssetId() . ']'); throw new Exception('Flavor asset not found with id [' . $data->getFlavorAssetId() . ']'); } // do not resume flavor asset added event consumption return $dbBatchJob; }
public static function handleConvertProfileFinished(BatchJob $dbBatchJob, kConvertProfileJobData $data, BatchJob $twinJob = null) { KalturaLog::debug("Convert Profile finished"); $originalflavorAsset = flavorAssetPeer::retrieveOriginalByEntryId($dbBatchJob->getEntryId()); if ($originalflavorAsset->getStatus() == flavorAsset::FLAVOR_ASSET_STATUS_TEMP) { $originalflavorAsset->setStatus(flavorAsset::FLAVOR_ASSET_STATUS_DELETED); $originalflavorAsset->setDeletedAt(time()); $originalflavorAsset->save(); } kBatchManager::updateEntry($dbBatchJob, entryStatus::READY); kFlowHelper::generateThumbnailsFromFlavor($dbBatchJob->getEntryId(), $dbBatchJob); return $dbBatchJob; }
/** * @param BatchJob $dbBatchJob * @param kConvertCollectionJobData $data * @param int $engineType * @return boolean */ public static function handleConvertCollectionFailed(BatchJob $dbBatchJob, kConvertCollectionJobData $data, $engineType) { $collectionFlavors = array(); foreach ($data->getFlavors() as $flavor) { $collectionFlavors[$flavor->getFlavorAssetId()] = $flavor; } // find the root job $rootBatchJob = $dbBatchJob->getRootJob(); $hasIncomplete = false; $shouldFailProfile = false; $flavorAssets = assetPeer::retrieveFlavorsByEntryId($dbBatchJob->getEntryId()); foreach ($flavorAssets as $flavorAsset) { if (isset($collectionFlavors[$flavorAsset->getId()])) { $flavorAsset->setStatus(flavorAsset::FLAVOR_ASSET_STATUS_ERROR); $flavorAsset->save(); if (!$rootBatchJob) { continue; } $flavorData = $collectionFlavors[$flavorAsset->getId()]; if ($flavorData->getReadyBehavior() == flavorParamsConversionProfile::READY_BEHAVIOR_REQUIRED) { $shouldFailProfile = true; } continue; } if ($flavorAsset->getIsOriginal()) { continue; } if ($flavorAsset->getStatus() == flavorAsset::FLAVOR_ASSET_STATUS_QUEUED || $flavorAsset->getStatus() == flavorAsset::FLAVOR_ASSET_STATUS_CONVERTING || $flavorAsset->getStatus() == flavorAsset::FLAVOR_ASSET_STATUS_IMPORTING || $flavorAsset->getStatus() == flavorAsset::FLAVOR_ASSET_STATUS_VALIDATING) { $hasIncomplete = true; } } if (!$rootBatchJob) { return false; } if ($rootBatchJob->getJobType() != BatchJobType::CONVERT_PROFILE) { return false; } if ($shouldFailProfile || !$hasIncomplete) { kJobsManager::failBatchJob($rootBatchJob, "Job " . $dbBatchJob->getId() . " failed"); } return false; }
/** * @param BatchJob $dbBatchJob * @param kImportJobData $data * @param BatchJob $twinJob * @return BatchJob */ public static function onImportJobFailed(BatchJob $dbBatchJob, kImportJobData $data, BatchJob $twinJob = null) { $statuses = array(EntryDistributionStatus::IMPORT_SUBMITTING, EntryDistributionStatus::IMPORT_UPDATING); $entryDistributions = EntryDistributionPeer::retrieveByEntryAndStatuses($dbBatchJob->getEntryId(), $statuses); foreach ($entryDistributions as $entryDistribution) { /* @var $entryDistribution EntryDistribution */ if ($entryDistribution->getStatus() == EntryDistributionStatus::IMPORT_SUBMITTING) { $entryDistribution->setStatus(EntryDistributionStatus::ERROR_SUBMITTING); } elseif ($entryDistribution->getStatus() == EntryDistributionStatus::IMPORT_UPDATING) { $entryDistribution->setStatus(EntryDistributionStatus::ERROR_UPDATING); } $entryDistribution->setErrorType($dbBatchJob->getErrType()); $entryDistribution->setErrorNumber($dbBatchJob->getErrNumber()); $entryDistribution->setErrorDescription($dbBatchJob->getMessage()); $entryDistribution->setDirtyStatus(null); $entryDistribution->save(); } }
/** * @param BatchJob $batchJob * @param $data * @param int $type * @param int $subType * @return BatchJob */ public static function addJob(BatchJob $batchJob, kJobData $data, $type, $subType = null) { $batchJob->setJobType($type); $batchJob->setJobSubType($subType); $batchJob->setData($data); if (!$batchJob->getParentJobId() && $batchJob->getEntryId()) { $currentJob = kBatchManager::getCurrentUpdatingJob(); if ($currentJob && $currentJob->getEntryId() == $batchJob->getEntryId()) { $batchJob->setParentJobId($currentJob->getId()); $batchJob->setBulkJobId($currentJob->getBulkJobId()); $batchJob->setRootJobId($currentJob->getRootJobId()); } else { $entry = entryPeer::retrieveByPKNoFilter($batchJob->getEntryId()); // some jobs could be on deleted entry if ($entry) { $batchJob->setRootJobId($entry->getBulkUploadId()); $batchJob->setBulkJobId($entry->getBulkUploadId()); } } } $lockInfo = new kLockInfoData($batchJob); $lockInfo->setEstimatedEffort($data->calculateEstimatedEffort($batchJob)); $lockInfo->setPriority($data->calculatePriority($batchJob)); $lockInfo->setUrgency($data->calculateUrgency($batchJob)); $batchJob->setLockInfo($lockInfo); if (is_null($batchJob->getStatus())) { $batchJob = self::updateBatchJob($batchJob, BatchJob::BATCHJOB_STATUS_PENDING); } else { $batchJob = self::updateBatchJob($batchJob, $batchJob->getStatus()); } return $batchJob; }
public static function decideProfileFlavorsConvert(BatchJob $parentJob, BatchJob $convertProfileJob, array $flavors, array $conversionProfileFlavorParams, $conversionProfileId, mediaInfo $mediaInfo = null) { $entryId = $convertProfileJob->getEntryId(); $originalFlavorAsset = assetPeer::retrieveOriginalByEntryId($entryId); if (is_null($originalFlavorAsset)) { $errDescription = 'Original flavor asset not found'; self::setError($errDescription, $convertProfileJob, BatchJobType::CONVERT_PROFILE, $convertProfileJob->getEntryId()); return false; } $errDescription = null; try { $finalFlavors = self::validateConversionProfile($convertProfileJob->getPartnerId(), $entryId, $mediaInfo, $flavors, $conversionProfileFlavorParams, $errDescription); } catch (Exception $e) { $code = $e->getCode(); if ($code == KDLErrors::SanityInvalidFrameDim || $code == KDLErrors::NoValidMediaStream) { $convertProfileJob = kJobsManager::failBatchJob($convertProfileJob, $errDescription); KalturaLog::err($e->getMessage()); throw $e; } } KalturaLog::log(count($finalFlavors) . " flavors returned from the decision layer"); if (is_null($finalFlavors)) { $convertProfileJob = kJobsManager::failBatchJob($convertProfileJob, $errDescription); KalturaLog::log("No flavors created"); //throw new Exception($errDescription); no need to throw alert if the root job failed } if (strlen($errDescription)) { $err = $convertProfileJob->getDescription() . $errDescription; $convertProfileJob->setDescription($err); $convertProfileJob->save(); //Check if the error thrown is invalid file - no media content if (strpos($errDescription, KDLErrors::ToString(KDLErrors::NoValidMediaStream)) !== false) { throw new Exception(KDLErrors::ToString(KDLErrors::NoValidMediaStream), KDLErrors::NoValidMediaStream); } } $conversionsCreated = 0; $waitingAssets = 0; $entry = $convertProfileJob->getEntry(); if (!$entry) { throw new APIException(APIErrors::INVALID_ENTRY, $convertProfileJob, $entryId); } $flavorsCollections = array(); // create a convert job per each flavor foreach ($finalFlavors as $flavor) { $flavorAsset = kBatchManager::createFlavorAsset($flavor, $entry->getPartnerId(), $entry->getId()); if (!$flavorAsset) { continue; } $collectionTag = $flavor->getCollectionTag(); /* * CHANGE: collection porcessing only for ExpressionEncoder jobs * to allow FFmpeg/ISMV processing */ KalturaLog::log("Check for collection case - engines(" . $flavor->getConversionEngines() . ")"); if ($collectionTag && $flavor->getConversionEngines() == conversionEngineType::EXPRESSION_ENCODER3) { $flavorsCollections[$collectionTag][] = $flavor; } else { KalturaLog::log("Adding flavor conversion with flavor params output id [" . $flavor->getId() . "] and flavor params asset id [" . $flavorAsset->getId() . "]"); $madiaInfoId = $mediaInfo ? $mediaInfo->getId() : null; $createdJob = self::decideFlavorConvert($flavorAsset, $flavor, $originalFlavorAsset, $conversionProfileId, $madiaInfoId, $parentJob); if ($createdJob) { $conversionsCreated++; } if ($flavorAsset->getStatus() == flavorAsset::ASSET_STATUS_WAIT_FOR_CONVERT) { $waitingAssets++; } } } foreach ($flavorsCollections as $tag => $flavors) { $createdJob = self::decideCollectionConvert($tag, $originalFlavorAsset, $entry, $parentJob, $flavors); if ($createdJob) { $conversionsCreated++; } } if (!$conversionsCreated && !$waitingAssets) { KalturaLog::log("No flavors created: {$errDescription}"); $convertProfileJob = kJobsManager::failBatchJob($convertProfileJob, $errDescription); return false; } return true; }
public static function handleProvisionProvideFailed(BatchJob $dbBatchJob, kProvisionJobData $data) { kBatchManager::updateEntry($dbBatchJob->getEntryId(), entryStatus::ERROR_CONVERTING); return $dbBatchJob; }
/** * @param BatchJob $batchJob * @param $data * @param int $type * @param int $subType * @return BatchJob */ public static function addJob(BatchJob $batchJob, $data, $type, $subType = null) { $batchJob->setJobType($type); $batchJob->setJobSubType($subType); $batchJob->setData($data); if (!$batchJob->getParentJobId() && $batchJob->getEntryId()) { $currentJob = kBatchManager::getCurrentUpdatingJob(); if ($currentJob && $currentJob->getEntryId() == $batchJob->getEntryId()) { $batchJob->setParentJobId($currentJob->getId()); $batchJob->setBulkJobId($currentJob->getBulkJobId()); $batchJob->setRootJobId($currentJob->getRootJobId()); } else { $entry = entryPeer::retrieveByPKNoFilter($batchJob->getEntryId()); // some jobs could be on deleted entry if ($entry) { $batchJob->setRootJobId($entry->getBulkUploadId()); $batchJob->setBulkJobId($entry->getBulkUploadId()); } } } // validate partner id $partnerId = $batchJob->getPartnerId(); // if(!$partnerId) // throw new APIException(APIErrors::PARTNER_NOT_SET); // validate that partner exists $partner = PartnerPeer::retrieveByPK($partnerId); if (!$partner) { KalturaLog::err("Invalid partner id [{$partnerId}]"); throw new APIException(APIErrors::INVALID_PARTNER_ID, $partnerId); } // set the priority and work group $batchJob->setPriority($partner->getPriority($batchJob->getBulkJobId())); $batchJob = self::updateBatchJob($batchJob, BatchJob::BATCHJOB_STATUS_PENDING); // look for identical jobs $twinJobs = BatchJobPeer::retrieveDuplicated($type, $data); $twinJob = null; if (count($twinJobs)) { foreach ($twinJobs as $currentTwinJob) { if ($currentTwinJob->getId() != $batchJob->getId()) { $twinJob = reset($twinJobs); } } } if (!is_null($twinJob)) { $batchJob->setTwinJobId($twinJob->getId()); if (!kConf::get("batch_ignore_duplication")) { $batchJob = self::updateBatchJob($batchJob, $twinJob->getStatus(), $twinJob); } else { $batchJob->save(); } } return $batchJob; }
protected function updatedVirusScanFailed(BatchJob $dbBatchJob, kVirusScanJobData $data) { $entry = entryPeer::retrieveByPKNoFilter($dbBatchJob->getEntryId()); if ($entry) { self::setEntryStatusBeforeScanFailure($entry, $entry->getStatus()); $entry->setStatus(VirusScanPlugin::getEntryStatusCoreValue(VirusScanEntryStatus::SCAN_FAILURE)); $entry->save(); myNotificationMgr::createNotification(kNotificationJobData::NOTIFICATION_TYPE_ENTRY_UPDATE, $entry); } else { KalturaLog::err('Entry not found with id [' . $dbBatchJob->getEntryId() . ']'); throw new Exception('Entry not found with id [' . $dbBatchJob->getEntryId() . ']'); } $flavorAsset = assetPeer::retrieveById($data->getFlavorAssetId()); if ($flavorAsset) { self::setFlavorAssetStatusBeforeScanFailure($flavorAsset, $flavorAsset->getStatus()); $flavorAsset->setStatus(flavorAsset::FLAVOR_ASSET_STATUS_ERROR); $flavorAsset->save(); } else { KalturaLog::err('Flavor asset not found with id [' . $data->getFlavorAssetId() . ']'); throw new Exception('Flavor asset not found with id [' . $data->getFlavorAssetId() . ']'); } // do not resume flavor asset added event consumption return $dbBatchJob; }
public static function exportSourceAssetFromJob(BatchJob $dbBatchJob) { // convert profile finished - export source flavor if ($dbBatchJob->getJobType() == BatchJobType::CONVERT_PROFILE) { $externalStorages = StorageProfilePeer::retrieveAutomaticByPartnerId($dbBatchJob->getPartnerId()); $sourceFlavor = assetPeer::retrieveOriginalByEntryId($dbBatchJob->getEntryId()); if (!$sourceFlavor) { KalturaLog::debug('Cannot find source flavor for entry id [' . $dbBatchJob->getEntryId() . ']'); } else { if (!$sourceFlavor->isLocalReadyStatus()) { KalturaLog::debug('Source flavor id [' . $sourceFlavor->getId() . '] has status [' . $sourceFlavor->getStatus() . '] - not ready for export'); } else { foreach ($externalStorages as $externalStorage) { if ($externalStorage->triggerFitsReadyAsset($dbBatchJob->getEntryId())) { self::exportFlavorAsset($sourceFlavor, $externalStorage); } } } } } // convert collection finished - export ism and ismc files if ($dbBatchJob->getJobType() == BatchJobType::CONVERT_COLLECTION && $dbBatchJob->getJobSubType() == conversionEngineType::EXPRESSION_ENCODER3) { $entry = $dbBatchJob->getEntry(); $externalStorages = StorageProfilePeer::retrieveAutomaticByPartnerId($dbBatchJob->getPartnerId()); foreach ($externalStorages as $externalStorage) { if ($externalStorage->triggerFitsReadyAsset($entry->getId())) { $ismKey = $entry->getSyncKey(entry::FILE_SYNC_ENTRY_SUB_TYPE_ISM); if (kFileSyncUtils::fileSync_exists($ismKey)) { self::export($entry, $externalStorage, $ismKey); } $ismcKey = $entry->getSyncKey(entry::FILE_SYNC_ENTRY_SUB_TYPE_ISMC); if (kFileSyncUtils::fileSync_exists($ismcKey)) { self::export($entry, $externalStorage, $ismcKey); } } } } return true; }
/** * @param BatchJob $dbBatchJob * @param BatchJob $twinJob * @return bool true if should continue to the next consumer */ public function updatedJob(BatchJob $dbBatchJob, BatchJob $twinJob = null) { if ($dbBatchJob->getStatus() != BatchJob::BATCHJOB_STATUS_FINISHED) { return true; } // convert profile finished - export source flavor if ($dbBatchJob->getJobType() == BatchJobType::CONVERT_PROFILE) { $externalStorages = StorageProfilePeer::retrieveAutomaticByPartnerId($dbBatchJob->getPartnerId()); foreach ($externalStorages as $externalStorage) { if ($externalStorage->getTrigger() == StorageProfile::STORAGE_TEMP_TRIGGER_FLAVOR_READY || $externalStorage->getTrigger() == StorageProfile::STORAGE_TEMP_TRIGGER_MODERATION_APPROVED && $dbBatchJob->getEntry()->getModerationStatus() == entry::ENTRY_MODERATION_STATUS_APPROVED) { $sourceFlavor = flavorAssetPeer::retrieveOriginalReadyByEntryId($dbBatchJob->getEntryId()); if ($sourceFlavor) { $this->exportFlavorAsset($sourceFlavor, $externalStorage); } } } } // convert collection finished - export ism and ismc files if ($dbBatchJob->getJobType() == BatchJobType::CONVERT_COLLECTION && $dbBatchJob->getJobSubType() == conversionEngineType::EXPRESSION_ENCODER3) { $entry = $dbBatchJob->getEntry(); $externalStorages = StorageProfilePeer::retrieveAutomaticByPartnerId($dbBatchJob->getPartnerId()); foreach ($externalStorages as $externalStorage) { if ($externalStorage->getTrigger() == StorageProfile::STORAGE_TEMP_TRIGGER_FLAVOR_READY || $externalStorage->getTrigger() == StorageProfile::STORAGE_TEMP_TRIGGER_MODERATION_APPROVED && $entry->getModerationStatus() == entry::ENTRY_MODERATION_STATUS_APPROVED) { $ismKey = $entry->getSyncKey(entry::FILE_SYNC_ENTRY_SUB_TYPE_ISM); if (kFileSyncUtils::fileSync_exists($ismKey)) { $this->export($entry, $externalStorage, $ismKey); } $ismcKey = $entry->getSyncKey(entry::FILE_SYNC_ENTRY_SUB_TYPE_ISMC); if (kFileSyncUtils::fileSync_exists($ismcKey)) { $this->export($entry, $externalStorage, $ismcKey); } } } } return true; }