function INFO($str) { global $TRACE_INFO; if ($TRACE_INFO) { TRACE($str); } }
public static function getExclusiveFile($path, $process_id = 1, $log_number_of_files = false) { $indicators = glob($path . "/*" . self::INDICATOR_SUFFIX); $count = count($indicators); if ($count > 0 || $log_number_of_files) { TRACE("[" . $count . "] indicator in directory [" . $path . "]"); } if ($indicators == null || count($indicators) == 0) { return null; } foreach ($indicators as $indicator) { $new_indicator = $indicator . "-{$process_id}"; $move_res = @rename($indicator, $new_indicator); // only one server will actually move the indicator ... if ($move_res) { $file = str_replace(kConversionCommand::INDICATOR_SUFFIX, "", $indicator); $file_name = basename($file); // now remove the indicator //unlink( $new_indicator ); // move to in-proc $in_proc = self::inProcFromIndicator($indicator); @rename($new_indicator, $in_proc); return array($file, $file_name, $in_proc); } else { TRACE("[{$indicator}] grabbed by other process"); } // keep on trying ... } return null; }
public function myBatchPartnerUsage() { self::initDb(); $partners_exists = true; $start_pos = 0; $bulk_size = 500; while ($partners_exists) { $c = new Criteria(); // get only free partners $c->addAnd(PartnerPeer::PARTNER_PACKAGE, 1); $c->addAnd(PartnerPeer::MONITOR_USAGE, 1); $c->setOffset($start_pos); $c->setLimit($bulk_size); $partners = PartnerPeer::doSelect($c); if (!$partners) { TRACE("No more partners. offset: {$start_pos} , limit: {$bulk_size} ."); $partners_exists = false; } else { TRACE("Looping " . ($start_pos + $bulk_size - 1) . " partners, offset: {$start_pos} ."); foreach ($partners as $partner) { myPartnerUtils::doPartnerUsage($partner, true); } } unset($partners); $start_pos += $bulk_size; } }
private function checkTimeout(KalturaBatchJob $job) { TRACE($job); if ($job->queueTime + $this->taskConfig->params->maxTimeBeforeFail < time()) { return $this->closeJob($job, KalturaBatchJobErrorTypes::APP, KalturaBatchJobAppErrors::CLOSER_TIMEOUT, 'Timed out', KalturaBatchJobStatus::FAILED); } }
public function run() { // print_r ( $this->kClient ); $r = rand(2, 5); TRACE("Slppeing for [{$r}]"); sleep($r); TRACE("Bye!"); }
public static function log($str) { if (self::$s_context) { self::getLogger()->log(self::$s_context . " " . $str); } else { self::getLogger()->log($str); } global $g_context; if (isset($g_context)) { TRACE($str); } }
public function toFile($file_name, $create_indicator = false) { TRACE("Setting file [{$file_name}] indicator [{$create_indicator}]"); if (file_exists($file_name)) { @unlink($file_name); } file_put_contents($file_name, $this->toString()); // sync - OK if ($create_indicator) { $this->createIndicator($file_name); } }
private function setConversionResult($file_name, kConversionCommand $conv_cmd, kConversionResult $conv_res) { // the result is placed either in the default resuls path of the server OR depending on the conv_cmd if ($conv_cmd->result_path) { $path = $conv_cmd->result_path . "/" . $file_name; } else { $path = $this->res_path . "/" . $file_name; } TRACE("Writing ConvResult to [{$path}]\n" . print_r($conv_res, true)); $conv_res->toFile($path, true); }
public function myBatchFlattenServer($script_name) { $this->script_name = $script_name; $this->register($script_name); SET_CONTEXT("FS"); $MAX_ITERATIONS_DUE_TO_PROPEL_MEMORY_LEAK = 10000000; self::initDb(); list($sleep_between_cycles, $number_of_times_to_skip_writing_sleeping) = self::getSleepParams('app_flatten_'); $last_worker_count = 0; $iteration = 0; $c = new Criteria(); $currentDc = kDataCenterMgr::getCurrentDc(); $c->add(BatchJobPeer::DC, kDataCenterMgr::getCurrentDcId()); $c->add(BatchJobPeer::JOB_TYPE, BatchJobType::FLATTEN); $c->add(BatchJobPeer::STATUS, BatchJob::BATCHJOB_STATUS_PROCESSED); $temp_count = 0; while (1) { self::exitIfDone(); try { sleep($sleep_between_cycles); $jobs = BatchJobPeer::doSelect($c); foreach ($jobs as $job) { $data = json_decode($job->getData(true), true); $entry_id = $data['entryId']; $entry_int_id = $data['entryIntId']; $entry_version = $data['entryVersion']; $file_format = $data['fileFormat']; $entry = entryPeer::retrieveByPK($entry_id); if (!$entry) { // entry is probably deleted if it is not returned from retrieveByPK // close job as failed $job->setStatus(BatchJob::BATCHJOB_STATUS_FAILED); $job->setDescription("could not retrieve entry, probably deleted"); TRACE("could not retrieve entry {$entry_id} , probably deleted"); $job->save(); continue; } $fileSyncKey = $entry->getSyncKey(entry::FILE_SYNC_ENTRY_SUB_TYPE_DOWNLOAD, $file_format); $fullFinalPath = kFileSyncUtils::getLocalFilePathForKey($fileSyncKey); $finalPathNoExt = substr($fullFinalPath, 0, strlen($fullFinalPath) - strlen($file_format)); myContentStorage::fullMkdir($fullFinalPath); $wildcardFinalPath = $finalPathNoExt . "*"; $older_files = glob($wildcardFinalPath); foreach ($older_files as $older_file) { TRACE("removing old file: [{$older_file}]"); @unlink($older_file); } TRACE("Downloading: {$fullFinalPath}"); kFile::downloadUrlToFile($data["serverUrl"], $fullFinalPath); if (!file_exists($fullFinalPath)) { TRACE("file doesnt exist: " . $data["serverUrl"]); $job->setDescription("file doesnt exist: " . $data["serverUrl"]); $job->setStatus(BatchJob::BATCHJOB_STATUS_FAILED); } else { if (filesize($fullFinalPath) < 100000) { @unlink($fullFinalPath); TRACE("file too small: " . $data["serverUrl"]); $job->setDescription("file too small: " . $data["serverUrl"]); $job->setStatus(BatchJob::BATCHJOB_STATUS_FAILED); } else { if ($data['email']) { $downloadLink = $entry->getDownloadUrl() . '/format/' . $file_format; kJobsManager::addMailJob(null, $entry_id, $entry->getPartnerId(), self::KALTURAS_FLATTEN_READY, kMailJobData::MAIL_PRIORITY_NORMAL, kConf::get("batch_flatten_video_sender_email"), kConf::get("batch_flatten_video_sender_name"), $data['email'], array($data['email'], $downloadLink)); } TRACE("Deleting: " . $data["deleteUrl"]); kFile::downloadUrlToString($data["deleteUrl"]); myNotificationMgr::createNotification(kNotificationJobData::NOTIFICATION_TYPE_ENTRY_UPDATE, $entry); $job->setStatus(BatchJob::BATCHJOB_STATUS_FINISHED); $filePath = kFileSyncUtils::getLocalFilePathForKey($fileSyncKey); if (file_exists($filePath)) { try { kFileSyncUtils::createSyncFileForKey($fileSyncKey); } catch (Exception $ex) { TRACE("ignore ERROR: " . $ex->getMessage()); } } else { TRACE("The file [{$filePath}] doesn't exists, not creating FileSync"); } } } $job->save(); } } catch (Exception $ex) { TRACE("ERROR: " . $ex->getMessage()); self::initDb(true); self::failed(); } if ($temp_count == 0) { TRACE("Ended conversion. sleeping for a while (" . $sleep_between_cycles . " seconds). Will write to the log in (" . $sleep_between_cycles * $number_of_times_to_skip_writing_sleeping . ") seconds"); } $temp_count++; if ($temp_count >= $number_of_times_to_skip_writing_sleeping) { $temp_count = 0; } } }
private function updateConvertedEntry($ok, $entry, kConversionResult $conv_res) { $file_before_conversion = $conv_res->conv_cmd->source_file; $file_after_conversion = $conv_res->conv_cmd->target_file; // TODO -get all targets if ($ok == true) { // TODO - write all targets not only primary one TRACE("File [{$file_before_conversion}] converted OK to [{$file_after_conversion}]"); try { // TODO - do we need to create the helpers eagerly ?? // $this->createFlvWrappersForTargets( $conv_res ); } catch (Exception $ex) { TRACE("Error while creating helper files for [{$file_after_conversion}]"); } $entry->setStatusReady(); } else { TRACE("Problem converting file [{$file_before_conversion}]"); $entry->setStatus(entryStatus::ERROR_CONVERTING); } $this->updateConversionInDb($entry, $conv_res); // loop until the file is really ready - sometimes the size of the file or the mtime is wrong for ($i = 0; $i < 15; $i++) { clearstatcache(); if (!file_exists($file_after_conversion) || filesize($file_after_conversion) == 0) { // TRACE ( "Entry id [" . $entry->getId() . "] printing file stats: " . print_r( stat ($file_after_conversion ) , true ) ); TRACE("Entry id [" . $entry->getId() . "]. no such file [{$file_after_conversion}]. Sleeping for 1 second for the [{$i}] time."); sleep(2); } else { break; } } TRACE("Entry id [" . $entry->getId() . "] setting duration"); $entry->setLengthInMsecs(kConversionHelper::getFlvDuration($file_after_conversion)); TRACE("Entry id [" . $entry->getId() . "] duration [" . $entry->getLengthInMsecs() . "]"); // how could it be otherwise ?? if ($entry->getMediaType() == entry::ENTRY_MEDIA_TYPE_VIDEO) { // TODO - move out of this function if the partner is required for more configurations $partner = PartnerPeer::retrieveByPK($entry->getPartnerId()); // TODO - make sure the width & height of the target are part of the kConversionReulst // if ( $conversion_info ) $entry->setDimensions ( $conversion_info->video_width , $conversion_info->video_height ); $offset = $entry->getBestThumbOffset($partner->getDefThumbOffset()); TRACE("Entry id [" . $entry->getId() . "] Thumb offset: [{$offset}]"); // first create the thumb for the entry myEntryUtils::createThumbnailFromEntry($entry, $entry, $offset); // then make sure it will propage to the roughcut if needed myEntryUtils::createRoughcutThumbnailFromEntry($entry, false); $entry->updateVideoDimensions(); TRACE("Entry id [" . $entry->getId() . "] dimensions: [" . $entry->getWidth() . "x" . $entry->getHeight() . "]"); } // send notification - regardless its status myNotificationMgr::createNotification(kNotificationJobData::NOTIFICATION_TYPE_ENTRY_UPDATE, $entry); $entry->save(); }
protected static function fixConvParams(kConversionCommand $conv_cmd) { $conv_params_list = $conv_cmd->conversion_params_list; foreach ($conv_params_list as $conv_params) { TRACE("Before fix: " . print_r($conv_params, true)); kConversionHelper::fillConversionParams($conv_cmd->source_file, $conv_params); TRACE("After fix: " . print_r($conv_params, true)); } }
public static function doPartnerUsage($partner) { $should_block_delete_partner = true; $blocking_days_grace = 7; $block_notification_grace = time() - dateUtils::DAY * $blocking_days_grace; $delete_grace = time() - dateUtils::DAY * 30; $packages = new PartnerPackages(); $partnerPackage = $packages->getPackageDetails($partner->getPartnerPackage()); $report_date = date('Y-m') . '-01'; // We are now working with the DWH and a stored-procedure, and not with record type 6 on partner_activity. $report_date = dateUtils::todayOffset(-3); list($totalStorage, $totalUsage, $totalTraffic) = myPartnerUtils::collectPartnerUsageFromDWH($partner, $partnerPackage, $report_date); $totalUsageGB = $totalUsage / 1024 / 1024; // from KB to GB $percent = round($totalUsageGB / $partnerPackage['cycle_bw'] * 100, 2); TRACE("percent (" . $partner->getId() . ") is: {$percent}"); $email_link_hash = 'pid=' . $partner->getId() . '&h=' . self::getEmailLinkHash($partner->getId(), $partner->getSecret()); $email_link_hash_adOpt = $email_link_hash . '&type=adOptIn'; /* mindtouch partners - extra mail parameter */ $mindtouch_notice = ' '; if ($partner->getType() == 103) { $mindtouch_notice = '<BR><BR>Note: You must be a MindTouch paying customer to upgrade your video service. If you are not a paying MindTouch customer, contact MindTouch: http://www.mindtouch.com/about_mindtouch/contact_mindtouch to get a quote.<BR><BR>'; } if ($percent >= 80 && $percent < 100 && !$partner->getEightyPercentWarning()) { TRACE("partner " . $partner->getId() . " reached 80% - setting first warning"); /* prepare mail job, and set EightyPercentWarning() to true/date */ $partner->setEightyPercentWarning(time()); $partner->setUsageLimitWarning(0); $body_params = array($partner->getAdminName(), $partnerPackage['cycle_bw'], $mindtouch_notice, round($totalUsageGB, 2), $email_link_hash, $email_link_hash_adOpt); myPartnerUtils::notifiyPartner(myPartnerUtils::KALTURA_PACKAGE_EIGHTY_PERCENT_WARNING, $partner, $body_params); } elseif ($percent >= 80 && $percent < 100 && $partner->getEightyPercentWarning() && !$partner->getUsageLimitWarning()) { TRACE("passed the 80%, assume notification sent, nothing to do."); } elseif ($percent < 80 && $partner->getEightyPercentWarning()) { TRACE("partner " . $partner->getId() . " was 80%, now not. clearing warnings"); /* clear getEightyPercentWarning */ $partner->setEightyPercentWarning(0); $partner->setUsageLimitWarning(0); } elseif ($percent >= 100 && !$partner->getUsageLimitWarning()) { TRACE("partner " . $partner->getId() . " reached 100% - setting second warning"); /* prepare mail job, and set getUsageLimitWarning() date */ $partner->setUsageLimitWarning(time()); $body_params = array($partner->getAdminName(), $mindtouch_notice, round($totalUsageGB, 2), $email_link_hash, $email_link_hash_adOpt); myPartnerUtils::notifiyPartner(myPartnerUtils::KALTURA_PACKAGE_LIMIT_WARNING_1, $partner, $body_params); } elseif ($percent >= 100 && $partnerPackage['cycle_fee'] == 0 && $partner->getUsageLimitWarning() > 0 && $partner->getUsageLimitWarning() <= $block_notification_grace && $partner->getUsageLimitWarning() > $delete_grace && $partner->getStatus() != Partner::PARTNER_STATUS_CONTENT_BLOCK) { TRACE("partner " . $partner->getId() . " reached 100% {$blocking_days_grace} days ago - sending block email and blocking partner"); /* send block email and block partner */ $body_params = array($partner->getAdminName(), $mindtouch_notice, round($totalUsageGB, 2), $email_link_hash, $email_link_hash_adOpt); myPartnerUtils::notifiyPartner(myPartnerUtils::KALTURA_PACKAGE_LIMIT_WARNING_2, $partner, $body_params); if ($should_block_delete_partner) { $partner->setStatus(2); } } elseif ($percent >= 120 && $partnerPackage['cycle_fee'] != 0 && $partner->getUsageLimitWarning() <= $block_notification_grace) { $body_params = array($partner->getAdminName(), round($totalUsageGB, 2)); myPartnerUtils::notifiyPartner(myPartnerUtils::KALTURA_PAID_PACKAGE_SUGGEST_UPGRADE, $partner, $body_params); } elseif ($percent >= 100 && $partnerPackage['cycle_fee'] == 0 && $partner->getUsageLimitWarning() > 0 && $partner->getUsageLimitWarning() <= $delete_grace && $partner->getStatus() == Partner::PARTNER_STATUS_CONTENT_BLOCK) { TRACE("partner " . $partner->getId() . " reached 100% a month ago - deleting partner"); /* delete partner */ $body_params = array($partner->getAdminName()); myPartnerUtils::notifiyPartner(myPartnerUtils::KALTURA_DELETE_ACCOUNT, $partner, $body_params); if ($should_block_delete_partner) { $partner->setStatus(0); } } elseif ($percent < 80 && ($partner->getUsageLimitWarning() || $partner->getEightyPercentWarning())) { TRACE("partner " . $partner->getId() . " OK"); // PARTNER OK // resetting status and warnings should only be done manually //$partner->setStatus(1); $partner->setEightyPercentWarning(0); $partner->setUsageLimitWarning(0); } $partner->save(); }
private static function failed(kConversionEngine $converter, kConversionCommand $conv_cmd, $first_failed_index) { TRACE("Error: Engine [" . $converter->getName() . "] failed to convert [{$first_failed_index}]"); }
public static function createFilePath($base_path, $file_name) { $id = self::getEntryIdFromFileName($file_name); // create a new path with the file name $entry = entryPeer::retrieveByPK($id); if ($entry) { TRACE("Found entry for file_name [{$file_name}] -> entry_id [{$id}]"); $int_id = $entry->getIntId(); $path_name = myContentStorage::dirForId($int_id, $id) . "." . pathinfo($file_name, PATHINFO_EXTENSION); } else { TRACE("Did NOT find entry for file_name [{$file_name}] -> entry_id [{$id}]"); $path_name = "AZ/" . pathinfo($file_name, PATHINFO_BASENAME); } // make sure the separator exists between the 2 paths // if ( ! kString::endsWith( $base_path , "/" ) ) $base_path .= "/"; kFile::fullMkdir($base_path . "/" . $path_name); return $base_path . "/" . $path_name; }
//myBatchBase::IGNORE_PREFIX ) continue; if (in_array($batch_name, $batchwatch_job_list)) { $command = file_get_contents($file); // TODO - change to fit the way we start / stop / restart // $cmd_line = "service $batch_name $command"; $cmd_line = "{$PHP_CMD} runBatch.php {$command} {$batch_name} "; TRACE("{$cmd_line}"); $output = array(); exec($cmd_line, $output, $return_var); TRACE("Result: [{$return_var}]\n" . print_r($output, true)); if ($command == "stop") { $path = batchStatus::batchEnd($batch_name); } } else { TRACE("VERY BAD!!: tying to manipulate invalid batch [{$batch_name}]. Ignoring..."); } // remove the file unlink($file); } if ($count > $write_to_the_log_seconds) { $count = 0; } if ($count == 0) { TRACE("Sleeping for [{$sleep_time}] second. Will write to the log in ({$write_to_the_log_seconds}]"); } sleep($sleep_time); $count++; } catch (Exception $ex) { TRACE("Error:" . $ex->getTrace()); } }
public function doMonthlyAggregation($date) { // set the dates $dateParts = explode('-', $date); $currentDate = $dateParts; $currentDate[2] = $currentDate[2] - 3; if ($currentDate[2] <= 0) { $currentDate[1] = $currentDate[1] - 1; if ($currentDate[1] == 0) { $currentDate[1] = 12; $currentDate[0] = $currentDate[0] - 1; } // if $currentDate[2] before reduction = 3, $currentDate[2] after reduction = 0 // if $currentDate[2] = 0 and last_day_for_month return 30, $currentDate[2] = 30 // if $currentDate[2] before reduction = 2, $currentDate[2] after reduction = -1 // if $currentDate[2] = -1 and last_day_for_month return 30, $currentDate[2] = 30 + (-1) = 29 $currentDate[2] = $this->last_day_for_month($currentDate) + $currentDate[2]; } if ($currentDate[1] < 10 && strlen($currentDate[1]) == 1) { $currentDate[1] = '0' . $currentDate[1]; } if ($currentDate[2] < 10 && strlen($currentDate[2]) == 1) { $currentDate[2] = '0' . $currentDate[2]; } $firstOfMonth = $currentDate[0] . '-' . $currentDate[1] . '-01'; $currentDate = implode('-', $currentDate); TRACE("calculating monthly agg. for date {$currentDate}"); // set start points $partners_exists = true; $start_pos = 0; $bulk_size = 500; // loop partners while ($partners_exists) { // pull bulk of partners $c = new Criteria(); $c->addAnd(PartnerPeer::CREATED_AT, $currentDate, Criteria::LESS_EQUAL); $c->setOffset($start_pos); $c->setLimit($bulk_size); $partners = PartnerPeer::doSelect($c); if (!$partners) { TRACE("No more partners. offset: {$start_pos} , limit: {$bulk_size} ."); // set flag to exit while loop $partners_exists = false; } else { TRACE("Looping " . ($start_pos + $bulk_size - 1) . " partners, offset: {$start_pos} ."); // loop bulk of partners foreach ($partners as $partner) { /* if ($partner->getId() != 593 && $partner->getId() != 395 && $partner->getId() != 387 ) continue; TRACE("testing... not skiping partner ".$partner->getId()); */ // get row from partner_activity where date is 1st of current month and type is 6 $partnerActivityCriteria = new Criteria(); $partnerActivityCriteria->addAnd(PartnerActivityPeer::ACTIVITY_DATE, $firstOfMonth); $partnerActivityCriteria->addAnd(PartnerActivityPeer::ACTIVITY, PartnerActivity::PARTNER_ACTIVITY_MONTHLY_AGGREGATION); $partnerActivityCriteria->addAnd(PartnerActivityPeer::PARTNER_ID, $partner->getId()); $activityTotal = PartnerActivityPeer::doSelect($partnerActivityCriteria); if (count($activityTotal) > 1) { TRACE("loaded more than one monthly aggregation row for partner. something went wrong. partner " . $partner->getID()); } elseif (count($activityTotal) == 0 || !$activityTotal) { // no rows for this month, either today is 1st of month or new partner. adding row for partner $partnerActivity = new PartnerActivity(); $partnerActivity->setActivity(PartnerActivity::PARTNER_ACTIVITY_MONTHLY_AGGREGATION); $partnerActivity->setPartnerId($partner->getId()); $partnerActivity->setActivityDate($firstOfMonth); $storageTotal = $this->getStorageAggregationFor($partner->getId(), $currentDate); $storageAddition = $storageTotal / date('t', strtotime($currentDate)); $partnerActivity->setAmount1($storageAddition); $partnerActivity->setAmount2($this->getTrafficFor($partner->getId(), $currentDate)); $total_amount = $partnerActivity->getAmount1() * 1024 + $partnerActivity->getAmount2(); $partnerActivity->setAmount($total_amount); $partnerActivity->save(); } else { $currentStorage = $activityTotal[0]->getAmount1(); $storageTotal = $this->getStorageAggregationFor($partner->getId(), $currentDate); $storageAddition = $storageTotal / date('t', strtotime($currentDate)); $activityTotal[0]->setAmount1($currentStorage + $storageAddition); $currentTraffic = $activityTotal[0]->getAmount2(); $trafficAddition = $this->getTrafficFor($partner->getId(), $currentDate); $activityTotal[0]->setAmount2($currentTraffic + $trafficAddition); // storage is saved in MB, traffic is saved in KB, normalizing storage for correct sum result $total_amount = $activityTotal[0]->getAmount1() * 1024 + $activityTotal[0]->getAmount2(); $activityTotal[0]->setAmount($total_amount); $activityTotal[0]->save(); } unset($partnerActivityCriteria); unset($activityTotal); } } unset($partners); $start_pos += $bulk_size; } }
protected static function exitIfDone() { if (!self::shouldProceed()) { if (self::$s_pending_tasks == 0) { TRACE("Gracefully exiting..."); die; } else { if (self::$s_force_die_time == null) { // set the force_die_time self::$s_force_die_time = time() + self::SECONDS_TO_FORCE_DIE; TRACE("Should exis but still exists [" . self::$s_pending_tasks . "] pending tasts ... Will FORCE DIE in [" . self::SECONDS_TO_FORCE_DIE . "] seconds"); } elseif (time() > self::$s_force_die_time) { TRACE("FORCE DIE !!. There are still [" . self::$s_pending_tasks . "] pending tasts but their time has come !"); die; } else { TRACE("Should exit but still exists [" . self::$s_pending_tasks . "] pending tasts ... Will FORCE DIE in [" . (time() - self::$s_force_die_time) . "] seconds"); } } } /* easeier process management */ $batch_script = substr(self::$batch_script_name, strrpos(str_replace("\\", "/", self::$batch_script_name), "/") + 1); $running_filename = str_replace('.php', '', $batch_script) . '.running.' . getmypid(); $file_path = self::getBatchwatchPath() . "/" . $running_filename; file_put_contents($file_path, date('Y-m-d H:i:s')); // sync - OK }