public static function shouldCreateLockObject(BatchJob $batchJob, $isNew, PropelPDO $con = null) { if ($isNew) { if (in_array($batchJob->getStatus(), self::getSchedulingRequiredStatusList())) { return true; } return false; } $oldStatus = $batchJob->getColumnsOldValue(BatchJobPeer::STATUS); $oldValueInClosed = is_null($oldStatus) ? false : in_array($oldStatus, BatchJobPeer::getClosedStatusList()); $newValue = $batchJob->getStatus(); $newValueInOpen = in_array($newValue, BatchJobPeer::getUnClosedStatusList()); // if the object is not a new object, a batch_job_lock object should exist. // an exception is when we move from a closed state to a open open. // f.i. retry request of an entry that was in closed status and we now restarted it. if (!($oldValueInClosed && $newValueInOpen)) { return false; } $lockEntry = BatchJobLockPeer::retrieveByPK($batchJob->getId()); if ($lockEntry === null) { return true; } return false; }
/** * will return $max_count of objects using the peer. * The criteria will be used to filter the basic parameter, the function will encapsulate the inner logic of the BatchJob * and the exclusiveness. * * @param Criteria $c */ private static function getExclusive(Criteria $c, kExclusiveLockKey $lockKey, $max_execution_time, $number_of_objects, $jobType, $maxOffset = null) { $schd = BatchJobLockPeer::SCHEDULER_ID; $work = BatchJobLockPeer::WORKER_ID; $btch = BatchJobLockPeer::BATCH_INDEX; $stat = BatchJobLockPeer::STATUS; $atmp = BatchJobLockPeer::EXECUTION_ATTEMPTS; $expr = BatchJobLockPeer::EXPIRATION; $recheck = BatchJobLockPeer::START_AT; $partnerLoadQuota = PartnerLoadPeer::QUOTA; $schd_id = $lockKey->getSchedulerId(); $work_id = $lockKey->getWorkerId(); $btch_id = $lockKey->getBatchIndex(); $now = time(); $now_str = date('Y-m-d H:i:s', $now); $delayedJobTypes = kConf::get('delayed_job_types'); $apiJobType = kPluginableEnumsManager::coreToApi('BatchJobType', $jobType); // added to support nfs delay if (in_array($apiJobType, $delayedJobTypes)) { $interval = kConf::hasParam('nfs_safety_margin_sec') ? kConf::get('nfs_safety_margin_sec') : 5; $c->add(BatchJobLockPeer::CREATED_AT, time() - $interval, Criteria::LESS_THAN); } $c->add(BatchJobLockPeer::JOB_TYPE, $jobType); $c->add(BatchJobLockPeer::DC, kDataCenterMgr::getCurrentDcId()); $c->add(BatchJobLockPeer::BATCH_VERSION, BatchJobLockPeer::getBatchVersion($jobType), Criteria::LESS_EQUAL); $prioritizers_ratio = BatchJobLockPeer::getPrioritizersRatio($jobType); $shouldUseJoin = BatchJobLockPeer::getMaxJobsForPartner($jobType) != self::UNLIMITED_QUOTA; self::addPrioritizersCondition($c, $prioritizers_ratio, $shouldUseJoin); // Query Parts $unClosedStatuses = implode(',', BatchJobPeer::getUnClosedStatusList()); $statusCondition = "{$stat} IN ({$unClosedStatuses})"; $lockExpiredCondition = "{$expr} <= '{$now_str}'"; $pendingJobsCondition = "( {$stat} = " . BatchJob::BATCHJOB_STATUS_PENDING . " OR ( {$stat} = " . BatchJob::BATCHJOB_STATUS_RETRY . " AND {$recheck} <= '{$now_str}' ))"; $unhandledJobCondition = "{$schd} IS NULL AND {$work} IS NULL AND {$btch} IS NULL "; $newJobsCond = "({$pendingJobsCondition} AND ({$unhandledJobCondition}))"; if ($shouldUseJoin) { $partnerLoadCondition = "(({$partnerLoadQuota} > 0) OR ({$partnerLoadQuota} is null))"; $newJobsCond .= " AND {$partnerLoadCondition}"; } $jobAlreadyHandledByWorker = "{$schd} = {$schd_id} AND {$work} = {$work_id} AND {$btch} = {$btch_id}"; $max_exe_attempts = BatchJobLockPeer::getMaxExecutionAttempts($jobType); $jobWasntExecutedTooMany = "{$atmp} <= {$max_exe_attempts} OR {$atmp} IS NULL"; // Generate query $query = "\t{$statusCondition}\n\t\t\t\t\tAND\t(\n\t\t\t\t\t\t{$lockExpiredCondition}\n\t\t\t\t\t\tOR\t({$newJobsCond})\n\t\t\t\t\t\tOR ({$jobAlreadyHandledByWorker})\n\t\t\t\t\t)\n\t\t\t\t\tAND ({$jobWasntExecutedTooMany})"; $c->add($stat, $query, Criteria::CUSTOM); // In case maxOffset isn't null, we want to take the chunk out of a random offset in between. // That's usefull for load handling if ($maxOffset) { $c->setOffset(rand(0, $maxOffset)); } $c->setLimit($number_of_objects); $objects = BatchJobLockPeer::doSelect($c, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2)); return self::lockObjects($lockKey, $objects, $max_execution_time); }
private static function getParentJobForWaitingAssetConversion($entryId, BatchJob $parentJob = null) { if ($parentJob && $parentJob->getJobType() == BatchJobType::POSTCONVERT) { //In case the flavor conversion is triggered by the ingested flavor add the conversion job //under the convert profile job if available $c = new Criteria(); $c->add(BatchJobPeer::ENTRY_ID, $entryId); $c->add(BatchJobPeer::JOB_TYPE, BatchJobType::CONVERT_PROFILE); $statuses = BatchJobPeer::getUnClosedStatusList(); $statuses[] = BatchJob::BATCHJOB_STATUS_ALMOST_DONE; $c->add(BatchJobPeer::STATUS, $statuses, Criteria::IN); $batchJob = BatchJobPeer::doSelectOne($c); if ($batchJob) { return $batchJob; } } return $parentJob; }
/** * will return $max_count of objects using the peer. * The criteria will be used to filter the basic parameter, the function will encapsulate the inner logic of the BatchJob * and the exclusiveness. * * @param Criteria $c */ private static function getExclusive(Criteria $c, kExclusiveLockKey $lockKey, $max_execution_time, $number_of_objects, $max_exe_attempts) { $schd = BatchJobPeer::SCHEDULER_ID; $work = BatchJobPeer::WORKER_ID; $btch = BatchJobPeer::BATCH_INDEX; $stat = BatchJobPeer::STATUS; $atmp = BatchJobPeer::EXECUTION_ATTEMPTS; $expr = BatchJobPeer::PROCESSOR_EXPIRATION; $recheck = BatchJobPeer::CHECK_AGAIN_TIMEOUT; $schd_id = $lockKey->getSchedulerId(); $work_id = $lockKey->getWorkerId(); $btch_id = $lockKey->getBatchIndex(); $now = time(); $now_str = date('Y-m-d H:i:s', $now); $unClosedStatuses = implode(',', BatchJobPeer::getUnClosedStatusList()); $inProgressStatuses = BatchJobPeer::getInProcStatusList(); $query = "\t\r\n\t\t\t\t\t\t{$stat} IN ({$unClosedStatuses})\r\n\t\t\t\t\tAND\t(\r\n\t\t\t\t\t\t\t{$expr} <= '{$now_str}'\r\n\t\t\t\t\t\tOR\t(\r\n\t\t\t\t\t\t\t\t(\r\n\t\t\t\t\t\t\t\t\t{$stat} = " . BatchJob::BATCHJOB_STATUS_PENDING . " \r\n\t\t\t\t\t\t\t\tOR (\r\n\t\t\t\t\t\t\t\t\t\t{$stat} = " . BatchJob::BATCHJOB_STATUS_RETRY . "\r\n\t\t\t\t\t\t\t\t\tAND {$recheck} <= {$now}\r\n\t\t\t\t\t\t\t\t)\r\n\t\t\t\t\t\t\t) \r\n\t\t\t\t\t\t\tAND (\r\n\t\t\t\t\t\t\t\t\t{$schd} IS NULL\r\n\t\t\t\t\t\t\t\tAND {$work} IS NULL \r\n\t\t\t\t\t\t\t\tAND {$btch} IS NULL \r\n\t\t\t\t\t\t\t)\r\n\t\t\t\t\t\t) \r\n\t\t\t\t\t\tOR (\r\n\t\t\t\t\t\t\t\t{$schd} = {$schd_id} \r\n\t\t\t\t\t\t\tAND {$work} = {$work_id} \r\n\t\t\t\t\t\t\tAND {$btch} = {$btch_id} \r\n\t\t\t\t\t\t\tAND {$stat} IN ({$inProgressStatuses}) \r\n\t\t\t\t\t\t)\r\n\t\t\t\t\t) \r\n\t\t\t\t\tAND (\r\n\t\t\t\t\t\t\t{$atmp} <= {$max_exe_attempts}\r\n\t\t\t\t\t\tOR\t{$atmp} IS NULL\r\n\t\t\t\t\t)"; $c->add($stat, $query, Criteria::CUSTOM); $c->add(BatchJobPeer::DC, kDataCenterMgr::getCurrentDcId()); $c->addAscendingOrderByColumn(BatchJobPeer::ID); $c->setLimit($number_of_objects); // $objects = BatchJobPeer::doSelect ( $c ); $objects = BatchJobPeer::doSelect($c, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2)); return self::lockObjects($lockKey, $objects, $max_execution_time); }
public static function deleteEntry(entry $entry, $partner_id = null, $onlyIfAllJobsDone = false) { if ($entry->getStatus() == entryStatus::DELETED || $entry->getStatus() == entryStatus::BLOCKED) { return; } // don't do this twice ! if ($onlyIfAllJobsDone) { KalturaLog::DEBUG("onlyIfAllJobsDone = " . (int) $onlyIfAllJobsDone); $dbEntryBatchJobs = BatchJobPeer::retrieveByEntryId($entry->getId()); foreach ($dbEntryBatchJobs as $job) { /* @var $job BatchJob */ if (in_array($job->getStatus(), BatchJobPeer::getUnClosedStatusList())) { KalturaLog::DEBUG("Entry [" . $entry->getId() . "] still has an unhandled batchjob [" . $job->getId() . "] with status [" . $job->getStatus() . "] - aborting deletion process."); //mark entry for later deletion $entry->setMarkedForDeletion(true); $entry->save(); return; } } } KalturaLog::log("myEntryUtils::delete Entry [" . $entry->getId() . "] Partner [" . $entry->getPartnerId() . "]"); kJobsManager::abortEntryJobs($entry->getId()); $media_type = $entry->getMediaType(); $need_to_fix_roughcut = false; $thumb_template_file = "&deleted_image.jpg"; KalturaLog::log("media type [{$media_type}]"); switch ($media_type) { case entry::ENTRY_MEDIA_TYPE_AUDIO: $template_file = "&deleted_audio.flv"; $need_to_fix_roughcut = true; break; case entry::ENTRY_MEDIA_TYPE_IMAGE: $template_file = "&deleted_image.jpg"; $need_to_fix_roughcut = false; // no need to add a batch job for images break; case entry::ENTRY_MEDIA_TYPE_VIDEO: $template_file = "&deleted_video.flv"; $need_to_fix_roughcut = true; break; case entry::ENTRY_MEDIA_TYPE_LIVE_STREAM_FLASH: case entry::ENTRY_MEDIA_TYPE_LIVE_STREAM_WINDOWS_MEDIA: case entry::ENTRY_MEDIA_TYPE_LIVE_STREAM_REAL_MEDIA: case entry::ENTRY_MEDIA_TYPE_LIVE_STREAM_QUICKTIME: kJobsManager::addProvisionDeleteJob(null, $entry); break; case entry::ENTRY_MEDIA_TYPE_SHOW: default: $template_file = "&deleted_rc.xml"; $need_to_fix_roughcut = false; break; } // in this case we'll need some batch job to fix all related roughcuts for this entry // use the batch_job mechanism to indicate there is a deleted entry to handle if ($need_to_fix_roughcut) { // Should use a different job type // BatchJob::createDeleteEntryJob ( $entry ); } $entry->putInCustomData("deleted_original_data", $entry->getData()); $entry->putInCustomData("deleted_original_thumb", $entry->getThumbnail()); $content_path = myContentStorage::getFSContentRootPath(); // Remarked by Tan-Tan 27/09/2010 // Handled by kObjectDeleteHandler // $currentDataKey = $entry->getSyncKey(entry::FILE_SYNC_ENTRY_SUB_TYPE_DATA); // replaced__getDataPath // $currentDataEditKey = $entry->getSyncKey(entry::FILE_SYNC_ENTRY_SUB_TYPE_DATA_EDIT); // replaced__getDataPathEdit // $currentThumbKey = $entry->getSyncKey(entry::FILE_SYNC_ENTRY_SUB_TYPE_THUMB); // replaced__getThumbnailPath $entry->setData($entry->getData()); // once to increment the verions $entry->setData($template_file); // the other to set the template $entry->setThumbnail($entry->getThumbnail()); // once to increment the verions $entry->setThumbnail($thumb_template_file); // the other to set the template // Remarked by Tan-Tan 27/09/2010 // Handled by kObjectDeleteHandler // // move file so there will be no access to it // $deleted_content = kFileSyncUtils::deleteSyncFileForKey($currentDataKey); // $deleted_content .= "|" . kFileSyncUtils::deleteSyncFileForKey($currentDataEditKey,false); // for some entries there may not be an edit version // $deleted_content .= "|" . kFileSyncUtils::deleteSyncFileForKey($currentThumbKey,false); // for some entries (empty mix / audio) there may not be a thumb FileSync // Remarked by Tan-Tan 27/09/2010 // $deleted_content is always null anyway // $entry->putInCustomData( "deleted_file_path" , $deleted_content ? $deleted_content : serialize($currentDataKey) ) ; $entry->setStatus(entryStatus::DELETED); //$entry->setCategories(""); // make sure the moderation_status is set to moderation::MODERATION_STATUS_DELETE $entry->setModerationStatus(moderation::MODERATION_STATUS_DELETE); $entry->setModifiedAt(time()); $entry->save(); myNotificationMgr::createNotification(kNotificationJobData::NOTIFICATION_TYPE_ENTRY_DELETE, $entry, null, null, null, null, $entry->getId()); }