public static function abortDbBatchJob(BatchJob $dbBatchJob, $force = false) { if (!$force && in_array($dbBatchJob->getStatus(), BatchJobPeer::getClosedStatusList())) { return $dbBatchJob; } $dbBatchJob->setAbort(1); // 1 = true // if not currently locked if (!$dbBatchJob->getSchedulerId()) { $dbBatchJob = self::updateBatchJob($dbBatchJob, BatchJob::BATCHJOB_STATUS_ABORTED); } else { $dbBatchJob->save(); } // aborts all child jobs self::abortChildJobs($dbBatchJob); return $dbBatchJob; }
public static function shouldCreateLockObject(BatchJob $batchJob, $isNew, PropelPDO $con = null) { if ($isNew) { if (in_array($batchJob->getStatus(), self::getSchedulingRequiredStatusList())) { return true; } return false; } $oldStatus = $batchJob->getColumnsOldValue(BatchJobPeer::STATUS); $oldValueInClosed = is_null($oldStatus) ? false : in_array($oldStatus, BatchJobPeer::getClosedStatusList()); $newValue = $batchJob->getStatus(); $newValueInOpen = in_array($newValue, BatchJobPeer::getUnClosedStatusList()); // if the object is not a new object, a batch_job_lock object should exist. // an exception is when we move from a closed state to a open open. // f.i. retry request of an entry that was in closed status and we now restarted it. if (!($oldValueInClosed && $newValueInOpen)) { return false; } $lockEntry = BatchJobLockPeer::retrieveByPK($batchJob->getId()); if ($lockEntry === null) { return true; } return false; }
/** * * @param $id * @param kExclusiveLockKey db_lock_object * @param db_lock_objectstatus - optional. will be used to set the status once the object is free * @return BatchJob */ public static function freeExclusive($id, kExclusiveLockKey $lockKey, $resetExecutionAttempts = false) { $c = new Criteria(); $c->add(BatchJobLockPeer::ID, $id); $c->add(BatchJobLockPeer::SCHEDULER_ID, $lockKey->getSchedulerId()); $c->add(BatchJobLockPeer::WORKER_ID, $lockKey->getWorkerId()); $c->add(BatchJobLockPeer::BATCH_INDEX, $lockKey->getBatchIndex()); $db_lock_object = BatchJobLockPeer::doSelectOne($c); if (!$db_lock_object) { if (BatchJobLockPeer::retrieveByPK($id)) { throw new APIException(APIErrors::FREE_EXCLUSIVE_JOB_FAILED, $id, $lockKey->getSchedulerId(), $lockKey->getWorkerId(), $lockKey->getBatchIndex()); } else { return BatchJobPeer::retrieveByPK($id); } } $db_object = $db_lock_object->getBatchJob(); if ($resetExecutionAttempts || in_array($db_lock_object->getStatus(), BatchJobPeer::getClosedStatusList())) { $db_lock_object->setExecutionAttempts(0); } $db_lock_object->setSchedulerId(null); $db_lock_object->setWorkerId(null); $db_lock_object->setBatchIndex(null); $db_lock_object->setExpiration(null); $db_lock_object->save(); if ($db_object->getStatus() != BatchJob::BATCHJOB_STATUS_ABORTED && $db_object->getExecutionStatus() == BatchJobExecutionStatus::ABORTED) { $db_object = kJobsManager::abortDbBatchJob($db_object); } return $db_object; }
/** * @param BatchJob $dbBatchJob * @param flavorAsset $currentFlavorAsset * @return BatchJob */ public static function handleConvertFinished(BatchJob $dbBatchJob = null, flavorAsset $currentFlavorAsset) { KalturaLog::debug("entry id [" . $currentFlavorAsset->getEntryId() . "] flavor asset id [" . $currentFlavorAsset->getId() . "]"); $profile = null; try { $profile = myPartnerUtils::getConversionProfile2ForEntry($currentFlavorAsset->getEntryId()); KalturaLog::debug("profile [" . $profile->getId() . "]"); } catch (Exception $e) { KalturaLog::err($e->getMessage()); } $currentReadyBehavior = self::getReadyBehavior($currentFlavorAsset, $profile); KalturaLog::debug("Current ready behavior [{$currentReadyBehavior}]"); if ($currentReadyBehavior == flavorParamsConversionProfile::READY_BEHAVIOR_IGNORE) { return $dbBatchJob; } $rootBatchJob = null; if ($dbBatchJob) { $rootBatchJob = $dbBatchJob->getRootJob(); } if ($rootBatchJob) { KalturaLog::debug("root batch job id [" . $rootBatchJob->getId() . "] type [" . $rootBatchJob->getJobType() . "]"); } // update the root job end exit if ($rootBatchJob && $rootBatchJob->getJobType() == BatchJobType::BULKDOWNLOAD) { $siblingJobs = $rootBatchJob->getChildJobs(); foreach ($siblingJobs as $siblingJob) { // checking only conversion child jobs if ($siblingJob->getJobType() != BatchJobType::CONVERT && $siblingJob->getJobType() != BatchJobType::CONVERT_COLLECTION && $siblingJob->getJobType() != BatchJobType::POSTCONVERT) { continue; } // if not complete leave function if (!in_array($siblingJob->getStatus(), BatchJobPeer::getClosedStatusList())) { KalturaLog::debug("job id [" . $siblingJob->getId() . "] status [" . $siblingJob->getStatus() . "]"); return $dbBatchJob; } } KalturaLog::debug("finish bulk download root job"); // all child jobs completed kJobsManager::updateBatchJob($rootBatchJob, BatchJob::BATCHJOB_STATUS_FINISHED); return $dbBatchJob; } $inheritedFlavorParamsIds = array(); $requiredFlavorParamsIds = array(); $flavorParamsConversionProfileItems = array(); if ($profile) { $flavorParamsConversionProfileItems = flavorParamsConversionProfilePeer::retrieveByConversionProfile($profile->getId()); } foreach ($flavorParamsConversionProfileItems as $flavorParamsConversionProfile) { if ($flavorParamsConversionProfile->getReadyBehavior() == flavorParamsConversionProfile::READY_BEHAVIOR_REQUIRED) { $requiredFlavorParamsIds[$flavorParamsConversionProfile->getFlavorParamsId()] = true; } if ($flavorParamsConversionProfile->getReadyBehavior() == flavorParamsConversionProfile::READY_BEHAVIOR_NO_IMPACT) { $inheritedFlavorParamsIds[] = $flavorParamsConversionProfile->getFlavorParamsId(); } } $flavorParamsItems = assetParamsPeer::retrieveByPKs($inheritedFlavorParamsIds); foreach ($flavorParamsItems as $flavorParams) { if ($flavorParams->getReadyBehavior() == flavorParamsConversionProfile::READY_BEHAVIOR_REQUIRED) { $requiredFlavorParamsIds[$flavorParamsConversionProfile->getFlavorParamsId()] = true; } } KalturaLog::debug("required flavor params ids [" . print_r($requiredFlavorParamsIds, true) . "]"); // go over all the flavor assets of the entry $inCompleteFlavorIds = array(); $origianlAssetFlavorId = null; $siblingFlavorAssets = assetPeer::retrieveFlavorsByEntryId($currentFlavorAsset->getEntryId()); foreach ($siblingFlavorAssets as $siblingFlavorAsset) { KalturaLog::debug("sibling flavor asset id [" . $siblingFlavorAsset->getId() . "] flavor params id [" . $siblingFlavorAsset->getFlavorParamsId() . "]"); // don't mark any incomplete flag if ($siblingFlavorAsset->getStatus() == flavorAsset::FLAVOR_ASSET_STATUS_READY) { KalturaLog::debug("sibling flavor asset id [" . $siblingFlavorAsset->getId() . "] is ready"); if (isset($requiredFlavorParamsIds[$siblingFlavorAsset->getFlavorParamsId()])) { unset($requiredFlavorParamsIds[$siblingFlavorAsset->getFlavorParamsId()]); } continue; } $readyBehavior = self::getReadyBehavior($siblingFlavorAsset, $profile); if ($siblingFlavorAsset->getStatus() == flavorAsset::ASSET_STATUS_EXPORTING) { if ($siblingFlavorAsset->getIsOriginal()) { $origianlAssetFlavorId = $siblingFlavorAsset->getFlavorParamsId(); } else { if ($readyBehavior != flavorParamsConversionProfile::READY_BEHAVIOR_IGNORE) { KalturaLog::debug("sibling flavor asset id [" . $siblingFlavorAsset->getId() . "] is incomplete"); $inCompleteFlavorIds[] = $siblingFlavorAsset->getFlavorParamsId(); } } } if ($readyBehavior == flavorParamsConversionProfile::READY_BEHAVIOR_IGNORE) { KalturaLog::debug("sibling flavor asset id [" . $siblingFlavorAsset->getId() . "] is ignored"); continue; } if ($siblingFlavorAsset->getStatus() == flavorAsset::FLAVOR_ASSET_STATUS_QUEUED || $siblingFlavorAsset->getStatus() == flavorAsset::FLAVOR_ASSET_STATUS_CONVERTING || $siblingFlavorAsset->getStatus() == flavorAsset::FLAVOR_ASSET_STATUS_IMPORTING || $siblingFlavorAsset->getStatus() == flavorAsset::FLAVOR_ASSET_STATUS_VALIDATING) { KalturaLog::debug("sibling flavor asset id [" . $siblingFlavorAsset->getId() . "] is incomplete"); $inCompleteFlavorIds[] = $siblingFlavorAsset->getFlavorParamsId(); } if ($readyBehavior == flavorParamsConversionProfile::READY_BEHAVIOR_REQUIRED) { KalturaLog::debug("sibling flavor asset id [" . $siblingFlavorAsset->getId() . "] is required"); $requiredFlavorParamsIds[$siblingFlavorAsset->getFlavorParamsId()] = true; } } KalturaLog::debug("left required flavor params ids [" . print_r($requiredFlavorParamsIds, true) . "]"); KalturaLog::debug("left incomplete flavor ids [" . print_r($inCompleteFlavorIds, true) . "]"); if (count($requiredFlavorParamsIds)) { $inCompleteRequiredFlavorParamsIds = array_keys($requiredFlavorParamsIds); foreach ($inCompleteRequiredFlavorParamsIds as $inCompleteFlavorId) { $inCompleteFlavorIds[] = $inCompleteFlavorId; } KalturaLog::debug('Convert Finished - has In-Complete Required flavors [[' . print_r($inCompleteRequiredFlavorParamsIds, true) . ']'); } elseif ($currentFlavorAsset->getStatus() == asset::ASSET_STATUS_READY && ($currentReadyBehavior == flavorParamsConversionProfile::READY_BEHAVIOR_OPTIONAL || $currentReadyBehavior == flavorParamsConversionProfile::READY_BEHAVIOR_REQUIRED)) { // mark the entry as ready if all required conversions completed or any of the optionals if ($currentFlavorAsset->getentry()->getReplacedEntryId()) { KalturaLog::debug('Entry is temporary replacement and requires all flavors to complete'); } else { KalturaLog::debug('Mark the entry as ready'); kBatchManager::updateEntry($currentFlavorAsset->getEntryId(), entryStatus::READY); } } if ($origianlAssetFlavorId) { $inCompleteFlavorIds = array_diff($inCompleteFlavorIds, array($origianlAssetFlavorId)); } if (!count($inCompleteFlavorIds)) { KalturaLog::debug('Convert Finished'); if ($origianlAssetFlavorId && $rootBatchJob && $rootBatchJob->getJobType() == BatchJobType::CONVERT_PROFILE) { kStorageExporter::exportSourceAssetFromJob($rootBatchJob); } else { // mark the context root job as finished only if all conversion jobs are completed kBatchManager::updateEntry($currentFlavorAsset->getEntryId(), entryStatus::READY); if ($rootBatchJob && $rootBatchJob->getJobType() == BatchJobType::CONVERT_PROFILE) { kJobsManager::updateBatchJob($rootBatchJob, BatchJob::BATCHJOB_STATUS_FINISHED); } } return $dbBatchJob; } KalturaLog::debug('Convert Finished - has In-Complete flavors [' . print_r($inCompleteFlavorIds, true) . ']'); if (!$rootBatchJob || $rootBatchJob->getJobType() != BatchJobType::CONVERT_PROFILE) { return $dbBatchJob; } $childJobs = $rootBatchJob->getChildJobs(); KalturaLog::debug('Child jobs found [' . count($childJobs) . ']'); if (count($childJobs) > 1) { $allDone = true; foreach ($childJobs as $childJob) { if ($childJob->getId() != $rootBatchJob->getId() && $childJob->getStatus() != BatchJob::BATCHJOB_STATUS_FINISHED) { KalturaLog::debug('Child job id [' . $childJob->getId() . '] status [' . $childJob->getStatus() . ']'); $allDone = false; } } if ($allDone) { KalturaLog::debug('All child jobs done, closing profile'); kJobsManager::updateBatchJob($rootBatchJob, BatchJob::BATCHJOB_STATUS_FINISHED); } } return $dbBatchJob; }
/** * @param BatchJob $dbBatchJob */ public static function abortChildJobs(BatchJob $dbBatchJob) { $maxJobsForQuery = 100; $c = new Criteria(); $c->add(BatchJobPeer::STATUS, BatchJobPeer::getClosedStatusList(), Criteria::NOT_IN); $c->setLimit($maxJobsForQuery); // aborts all child jobs in chunks while (true) { $dbChildJobs = $dbBatchJob->getChildJobs($c); foreach ($dbChildJobs as $dbChildJob) { $dbChildJob->setMessage("Parent job was aborted."); if ($dbChildJob->getId() != $dbBatchJob->getId()) { self::abortDbBatchJob($dbChildJob); } } if (count($dbChildJobs) < $maxJobsForQuery) { break; } } }
private static function getQueue(Criteria $c, $schedulerId, $workerId, $max_exe_attempts) { $schd = BatchJobPeer::SCHEDULER_ID; $work = BatchJobPeer::WORKER_ID; $stat = BatchJobPeer::STATUS; $atmp = BatchJobPeer::EXECUTION_ATTEMPTS; $expr = BatchJobPeer::PROCESSOR_EXPIRATION; $recheck = BatchJobPeer::CHECK_AGAIN_TIMEOUT; $schd_id = $schedulerId; $work_id = $workerId; $now = time(); $now_str = date('Y-m-d H:i:s', $now); // same workers unfinished jobs $query1 = "(\r\n\t\t\t\t\t\t\t{$schd} = {$schd_id} \r\n\t\t\t\t\t\tAND {$work} = {$work_id} \r\n\t\t\t\t\t\tAND {$stat} IN (" . BatchJobPeer::getInProcStatusList() . ") \r\n\t\t\t\t\t)"; // "others unfinished jobs " - the expiration should be SMALLER than the current time to make sure the job is not // being processed $closedStatuses = implode(',', BatchJobPeer::getClosedStatusList()); $query2 = "(\r\n\t\t\t\t\t\t\t{$stat} NOT IN ({$closedStatuses})\r\n\t\t\t\t\t\tAND\t{$expr} <= '{$now_str}'\r\n\t\t\t\t\t)"; // "retry jobs" $query3 = "(\r\n\t\t\t\t\t\t{$stat} = " . BatchJob::BATCHJOB_STATUS_RETRY . "\r\n\t\t\t\t\t\tAND {$recheck} <= {$now}\r\n\t\t\t\t\t)"; // "max attempts jobs" $queryMaxAttempts = "(\r\n\t\t\t\t\t\t\t\t{$atmp} <= {$max_exe_attempts}\r\n\t\t\t\t\t\t\t\tOR\r\n\t\t\t\t\t\t\t\t{$atmp} IS NULL\r\n\t\t\t\t\t\t\t)"; $crit1 = $c->getNewCriterion($stat, BatchJob::BATCHJOB_STATUS_PENDING); $crit1->addOr($c->getNewCriterion($schd, $query1, Criteria::CUSTOM)); $crit1->addOr($c->getNewCriterion($schd, $query2, Criteria::CUSTOM)); $crit1->addOr($c->getNewCriterion($schd, $query3, Criteria::CUSTOM)); $c->addAnd($crit1); $c->addAnd($c->getNewCriterion($atmp, $queryMaxAttempts, Criteria::CUSTOM)); $c->addAnd($c->getNewCriterion(BatchJobPeer::DC, kDataCenterMgr::getCurrentDcId())); // $objects = BatchJobPeer::doCount ( $c ); return BatchJobPeer::doCount($c, false, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2)); }
public function objectChanged(BaseObject $object, array $modifiedColumns) { if ($object instanceof entry && in_array(entryPeer::STATUS, $modifiedColumns) && ($object->getStatus() == entryStatus::READY || $object->getStatus() == entryStatus::ERROR_CONVERTING) && $object->getReplacedEntryId()) { kFlowHelper::handleEntryReplacement($object); return true; } if ($object instanceof UploadToken && in_array(UploadTokenPeer::STATUS, $modifiedColumns) && $object->getStatus() == UploadToken::UPLOAD_TOKEN_FULL_UPLOAD) { kFlowHelper::handleUploadFinished($object); return true; } if ($object instanceof BatchJob && $object->getJobType() == BatchJobType::BULKUPLOAD && $object->getStatus() == BatchJob::BATCHJOB_STATUS_ABORTED && in_array(BatchJobPeer::STATUS, $modifiedColumns) && in_array($object->getColumnsOldValue(BatchJobPeer::STATUS), BatchJobPeer::getClosedStatusList())) { $partner = $object->getPartner(); if ($partner->getEnableBulkUploadNotificationsEmails()) { kFlowHelper::sendBulkUploadNotificationEmail($object, MailType::MAIL_TYPE_BULKUPLOAD_ABORTED, array($partner->getAdminName(), $object->getId(), kFlowHelper::createBulkUploadLogUrl($object))); } return true; } if ($object instanceof UserRole && in_array(UserRolePeer::PERMISSION_NAMES, $modifiedColumns)) { $filter = new kuserFilter(); $filter->set('_eq_role_ids', $object->getId()); kJobsManager::addIndexJob($object->getPartnerId(), IndexObjectType::USER, $filter, false); return true; } if ($object instanceof FileSync) { $c = new Criteria(); $c->add(BatchJobLockPeer::OBJECT_ID, $object->getId()); $c->add(BatchJobLockPeer::OBJECT_TYPE, BatchJobObjectType::FILE_SYNC); $c->add(BatchJobLockPeer::JOB_TYPE, BatchJobType::FILESYNC_IMPORT); $c->add(BatchJobLockPeer::STATUS, array(BatchJob::BATCHJOB_STATUS_RETRY, BatchJob::BATCHJOB_STATUS_PENDING), Criteria::IN); $fileSyncImportJobs = BatchJobLockPeer::doSelect($c); foreach ($fileSyncImportJobs as $fileSyncImportJob) { kJobsManager::abortDbBatchJob(BatchJobPeer::retrieveByPK($fileSyncImportJob->getId())); } return true; } if (!$object instanceof flavorAsset || !in_array(assetPeer::STATUS, $modifiedColumns)) { return true; } $entry = entryPeer::retrieveByPKNoFilter($object->getEntryId()); KalturaLog::info("Asset id [" . $object->getId() . "] isOriginal [" . $object->getIsOriginal() . "] status [" . $object->getStatus() . "]"); if ($object->getIsOriginal()) { return true; } if ($object->getStatus() == flavorAsset::FLAVOR_ASSET_STATUS_VALIDATING) { $postConvertAssetType = BatchJob::POSTCONVERT_ASSET_TYPE_FLAVOR; $offset = $entry->getThumbOffset(); // entry getThumbOffset now takes the partner DefThumbOffset into consideration $syncKey = $object->getSyncKey(asset::FILE_SYNC_FLAVOR_ASSET_SUB_TYPE_ASSET); $fileSync = kFileSyncUtils::getLocalFileSyncForKey($syncKey, false); if (!$fileSync) { return true; } $srcFileSyncLocalPath = kFileSyncUtils::getLocalFilePathForKey($syncKey); if ($srcFileSyncLocalPath) { kJobsManager::addPostConvertJob(null, $postConvertAssetType, $srcFileSyncLocalPath, $object->getId(), null, $entry->getCreateThumb(), $offset); } } elseif ($object->getStatus() == flavorAsset::FLAVOR_ASSET_STATUS_READY) { // If we get a ready flavor and the entry is in no content if ($entry->getStatus() == entryStatus::NO_CONTENT) { $entry->setStatus(entryStatus::PENDING); // we change the entry to pending $entry->save(); } } return true; }
public static function cleanExclusiveJobs() { $jobs = kBatchExclusiveLock::getExpiredJobs(); foreach ($jobs as $job) { KalturaLog::log("Cleaning job id[" . $job->getId() . "]"); kJobsManager::updateBatchJob($job, BatchJob::BATCHJOB_STATUS_FATAL); } $c = new Criteria(); $c->add(BatchJobPeer::STATUS, BatchJobPeer::getClosedStatusList(), Criteria::IN); $c->add(BatchJobPeer::BATCH_INDEX, null, Criteria::ISNOTNULL); // The 'closed' jobs should be donn for at least 10min. // before the cleanup starts messing upo with'em // This solves cases when job (convert) completes succesfully, // but the next job (closure)does not get a chance to take over due to the clean-up $c->add(BatchJobPeer::FINISH_TIME, time() - 600, Criteria::LESS_THAN); // MUST be the master DB $jobs = BatchJobPeer::doSelect($c, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2)); foreach ($jobs as $job) { KalturaLog::log("Cleaning job id[" . $job->getId() . "]"); $job->setSchedulerId(null); $job->setWorkerId(null); $job->setBatchIndex(null); $job->setProcessorExpiration(null); $job->save(); } return count($jobs); }