public function execute() { $this->forceSystemAuthentication(); myDbHelper::$use_alternative_con = null; //myDbHelper::DB_HELPER_CONN_PROPEL2 $uiConfId = $this->getRequestParameter("id"); $uiConf = uiConfPeer::retrieveByPK($uiConfId); $subTypes = array(uiconf::FILE_SYNC_UICONF_SUB_TYPE_DATA, uiconf::FILE_SYNC_UICONF_SUB_TYPE_FEATURES); foreach ($subTypes as $subType) { if ($subType == uiconf::FILE_SYNC_UICONF_SUB_TYPE_DATA) { echo "Data:" . PHP_EOL; } else { if ($subType == uiconf::FILE_SYNC_UICONF_SUB_TYPE_FEATURES) { echo "Features:" . PHP_EOL; } } $syncKey = $uiConf->getSyncKey($subType); if (kFileSyncUtils::file_exists($syncKey)) { echo "File sync already exists." . PHP_EOL; } else { list($rootPath, $filePath) = $uiConf->generateFilePathArr($subType); $fullPath = $rootPath . $filePath; if (file_exists($fullPath)) { kFileSyncUtils::createSyncFileForKey($syncKey); echo "Created successfully." . PHP_EOL; } else { echo "File not found:" . PHP_EOL; echo $fullPath . PHP_EOL; echo "Not creating file sync." . PHP_EOL; } } echo PHP_EOL; } die; }
public function myBatchFlattenServer($script_name) { $this->script_name = $script_name; $this->register($script_name); SET_CONTEXT("FS"); $MAX_ITERATIONS_DUE_TO_PROPEL_MEMORY_LEAK = 10000000; self::initDb(); list($sleep_between_cycles, $number_of_times_to_skip_writing_sleeping) = self::getSleepParams('app_flatten_'); $last_worker_count = 0; $iteration = 0; $c = new Criteria(); $currentDc = kDataCenterMgr::getCurrentDc(); $c->add(BatchJobPeer::DC, kDataCenterMgr::getCurrentDcId()); $c->add(BatchJobPeer::JOB_TYPE, BatchJobType::FLATTEN); $c->add(BatchJobPeer::STATUS, BatchJob::BATCHJOB_STATUS_PROCESSED); $temp_count = 0; while (1) { self::exitIfDone(); try { sleep($sleep_between_cycles); $jobs = BatchJobPeer::doSelect($c); foreach ($jobs as $job) { $data = json_decode($job->getData(true), true); $entry_id = $data['entryId']; $entry_int_id = $data['entryIntId']; $entry_version = $data['entryVersion']; $file_format = $data['fileFormat']; $entry = entryPeer::retrieveByPK($entry_id); if (!$entry) { // entry is probably deleted if it is not returned from retrieveByPK // close job as failed $job->setStatus(BatchJob::BATCHJOB_STATUS_FAILED); $job->setDescription("could not retrieve entry, probably deleted"); TRACE("could not retrieve entry {$entry_id} , probably deleted"); $job->save(); continue; } $fileSyncKey = $entry->getSyncKey(entry::FILE_SYNC_ENTRY_SUB_TYPE_DOWNLOAD, $file_format); $fullFinalPath = kFileSyncUtils::getLocalFilePathForKey($fileSyncKey); $finalPathNoExt = substr($fullFinalPath, 0, strlen($fullFinalPath) - strlen($file_format)); myContentStorage::fullMkdir($fullFinalPath); $wildcardFinalPath = $finalPathNoExt . "*"; $older_files = glob($wildcardFinalPath); foreach ($older_files as $older_file) { TRACE("removing old file: [{$older_file}]"); @unlink($older_file); } TRACE("Downloading: {$fullFinalPath}"); kFile::downloadUrlToFile($data["serverUrl"], $fullFinalPath); if (!file_exists($fullFinalPath)) { TRACE("file doesnt exist: " . $data["serverUrl"]); $job->setDescription("file doesnt exist: " . $data["serverUrl"]); $job->setStatus(BatchJob::BATCHJOB_STATUS_FAILED); } else { if (filesize($fullFinalPath) < 100000) { @unlink($fullFinalPath); TRACE("file too small: " . $data["serverUrl"]); $job->setDescription("file too small: " . $data["serverUrl"]); $job->setStatus(BatchJob::BATCHJOB_STATUS_FAILED); } else { if ($data['email']) { $downloadLink = $entry->getDownloadUrl() . '/format/' . $file_format; kJobsManager::addMailJob(null, $entry_id, $entry->getPartnerId(), self::KALTURAS_FLATTEN_READY, kMailJobData::MAIL_PRIORITY_NORMAL, kConf::get("batch_flatten_video_sender_email"), kConf::get("batch_flatten_video_sender_name"), $data['email'], array($data['email'], $downloadLink)); } TRACE("Deleting: " . $data["deleteUrl"]); kFile::downloadUrlToString($data["deleteUrl"]); myNotificationMgr::createNotification(kNotificationJobData::NOTIFICATION_TYPE_ENTRY_UPDATE, $entry); $job->setStatus(BatchJob::BATCHJOB_STATUS_FINISHED); $filePath = kFileSyncUtils::getLocalFilePathForKey($fileSyncKey); if (file_exists($filePath)) { try { kFileSyncUtils::createSyncFileForKey($fileSyncKey); } catch (Exception $ex) { TRACE("ignore ERROR: " . $ex->getMessage()); } } else { TRACE("The file [{$filePath}] doesn't exists, not creating FileSync"); } } } $job->save(); } } catch (Exception $ex) { TRACE("ERROR: " . $ex->getMessage()); self::initDb(true); self::failed(); } if ($temp_count == 0) { TRACE("Ended conversion. sleeping for a while (" . $sleep_between_cycles . " seconds). Will write to the log in (" . $sleep_between_cycles * $number_of_times_to_skip_writing_sleeping . ") seconds"); } $temp_count++; if ($temp_count >= $number_of_times_to_skip_writing_sleeping) { $temp_count = 0; } } }
private function sentToCenversion($write_to_log = true) { try { $debug = array("before getFileToConvert"); list($before_archiving_file_path, $file_name, $in_proc) = $this->getFileToConvert($write_to_log); $debug[] = "after getFileToConvert [{$before_archiving_file_path}] [{$file_name}]"; if (!$before_archiving_file_path) { return; } // TODO - check if this file failed too many times ... //if ( !$this->shouldHandleFile ( $file_name ) ) $entry_id = self::getEntryIdFromFileName($file_name); $debug[] = "entry_id [{$entry_id}]"; // we have to retrieve the path of the entry - do so by setting the data to the file path (now rather than at the end) $entry = entryPeer::retrieveByPK($entry_id); $conv_profile = myPartnerUtils::getConversionProfileForEntry($entry_id); $debug[] = "conversion profile of class [" . get_class($conv_profile) . "]"; if (!$entry) { KalturaLog::debug("entry id [{$entry_id}] not found!"); return; } // the conversion target should be the entry's dataPath $flv_file_name = kConversionHelper::flvFileName($before_archiving_file_path); $debug[] = "flv_file_name [{$flv_file_name}]"; $entry->setData(null); $entry->setData($flv_file_name); // we assume the output will be of type FLV $entry->save(); $archive_file_sync = $this->archiveFile($before_archiving_file_path); $archived_file_path = $archive_file_sync->getFullPath(); $debug[] = "archived_file_path [{$archived_file_path}]"; if ($conv_profile->getBypassFlv() && kConversionHelper::isFlv($archived_file_path)) { $conv_cmd = $this->createConversionCommandFromConverionProfile($archived_file_path, $archived_file_path, $conv_profile, $entry); $debug[] = "before createConversionInDb[{$entry_id}] [{$archived_file_path}]"; // first update the DB $this->createConversionInDb($entry_id, $archived_file_path, $conv_cmd); // TODO - check if there is a set of convParams for this FLV profile and manye some conversion should be done // for the edit version ?? KalturaLog::debug("Bypassing conversion for entry_id [{$entry_id}] file [{$file_name}]"); $conv_res = new kConversionResult($conv_cmd); $conv_res_info = new kConvResInfo(); $conv_res_info->target = $archived_file_path; $start = microtime(true); // FileSync - soft copy $archived_sync_key = $entry->getSyncKey(entry::FILE_SYNC_ENTRY_SUB_TYPE_ARCHIVE); $data_sync_key = $entry->getSyncKey(entry::FILE_SYNC_ENTRY_SUB_TYPE_DATA); kFileSyncUtils::softCopy($archived_sync_key, $data_sync_key); $end = microtime(true); $conv_res_info->duration = $end - $start; $conv_res_info->conv_str = "NO CONVERT"; $conv_res->appendResInfo($conv_res_info); $this->updateConvertedEntry(true, $entry, $conv_res); $this->removeInProc($in_proc); return; } // FileSync - create file sync for the future place of the converted data $data_sync_key = $entry->getSyncKey(entry::FILE_SYNC_ENTRY_SUB_TYPE_DATA); kFileSyncUtils::createSyncFileForKey($data_sync_key, false, false); $full_target_path = kFileSyncUtils::getLocalFilePathForKey($data_sync_key, true); if ($conv_profile->getProfileTypeSuffix() == "edit") { // FileSync - create file sync for the future place of the converted data in edit flavor $data_edit_sync_key = $entry->getSyncKey(entry::FILE_SYNC_ENTRY_SUB_TYPE_DATA_EDIT); kFileSyncUtils::createSyncFileForKey($data_edit_sync_key, false, false); } $debug[] = "full_target_path [{$full_target_path}]"; $conv_cmd = $this->createConversionCommandFromConverionProfile($archived_file_path, $full_target_path, $conv_profile, $entry); $debug[] = "before createConversionInDb[{$entry_id}] [{$archived_file_path}]"; // first update the DB $this->createConversionInDb($entry_id, $archived_file_path, $conv_cmd); KalturaLog::debug("Setting ConversionCommand for file [{$file_name}]\n" . print_r($conv_cmd, true)); $debug[] = "before saveConversionCommand"; // then save the conversion command $cmd_file_path = $this->saveConversionCommand(); $this->removeInProc($in_proc); KalturaLog::debug("Set ConversionCommand for file [{$file_name}] in [{$cmd_file_path}]"); $debug[] = "end"; } catch (kConversionException $kcoe) { $this->removeInProc($in_proc); KalturaLog::debug("Error:\n" . $kcoe->getMessage() . "\n" . $kcoe->getTraceAsString() . "\n" . print_r($debug)); // update the entry with the error sttus and the error message to the conversion result $conv_res = new kConversionResult($conv_cmd); $conv_res->appendResult($kcoe->getMessage()); $this->updateConvertedEntry(false, $entry, $conv_res); } catch (Exception $ex) { $this->removeInProc($in_proc); KalturaLog::debug("Error:\n" . $ex->getMessage() . "\n" . $ex->getTraceAsString() . "\n" . print_r($debug)); // if this failed for some unknown reason - set it for reconversion $indicator = $this->setFileToReConvert($before_archiving_file_path, $file_name); KalturaLog::debug("... will reconvert [" . print_r($indicator, true) . "]"); throw $ex; } }
protected function archiveFile($file_name) { KalturaLog::debug("Archiving file [" . $file_name . "]"); $id = self::getEntryIdFromFileName($file_name); $entry = entryPeer::retrieveByPKNoFilter($id); $entry->setArchiveExtension(pathinfo($file_name, PATHINFO_EXTENSION)); $sync_key = $entry->getSyncKey(entry::FILE_SYNC_ENTRY_SUB_TYPE_ARCHIVE); $file_sync = kFileSyncUtils::createSyncFileForKey($sync_key, false, false); $target = kFileSyncUtils::getLocalFilePathForKey($sync_key, false); KalturaLog::debug("Archiving file [" . $file_name . "] to [" . $target . "]}"); // MOVE - there is no need to copy because we the ConvCommand will include the file path anyway if ($file_name == $target) { KalturaLog::debug("File [{$file_name}] already archived"); return $file_sync; } // there is a file in the archive but the current file does not exist if (!file_exists($file_name) && file_exists($target)) { return $file_sync; } if (file_exists($target) && filesize($target) == filesize($file_name)) { return $file_sync; } else { // move to archive and override if exists myContentStorage::moveFile($file_name, $target, true, false); } kFileSyncUtils::markLocalFileSyncAsReady($sync_key); return $file_sync; }