public function execute()
{
$this->forceSystemAuthentication();
myDbHelper::$use_alternative_con = null;
//myDbHelper::DB_HELPER_CONN_PROPEL2
$uiConfId = $this->getRequestParameter("id");
$uiConf = uiConfPeer::retrieveByPK($uiConfId);
$subTypes = array(uiconf::FILE_SYNC_UICONF_SUB_TYPE_DATA, uiconf::FILE_SYNC_UICONF_SUB_TYPE_FEATURES);
foreach ($subTypes as $subType) {
if ($subType == uiconf::FILE_SYNC_UICONF_SUB_TYPE_DATA) {
echo "Data:" . PHP_EOL;
} else {
if ($subType == uiconf::FILE_SYNC_UICONF_SUB_TYPE_FEATURES) {
echo "Features:" . PHP_EOL;
}
}
$syncKey = $uiConf->getSyncKey($subType);
if (kFileSyncUtils::file_exists($syncKey)) {
echo "File sync already exists." . PHP_EOL;
} else {
list($rootPath, $filePath) = $uiConf->generateFilePathArr($subType);
$fullPath = $rootPath . $filePath;
if (file_exists($fullPath)) {
kFileSyncUtils::createSyncFileForKey($syncKey);
echo "Created successfully." . PHP_EOL;
} else {
echo "File not found:" . PHP_EOL;
echo $fullPath . PHP_EOL;
echo "Not creating file sync." . PHP_EOL;
}
}
echo PHP_EOL;
}
die;
}
public function myBatchFlattenServer($script_name)
{
$this->script_name = $script_name;
$this->register($script_name);
SET_CONTEXT("FS");
$MAX_ITERATIONS_DUE_TO_PROPEL_MEMORY_LEAK = 10000000;
self::initDb();
list($sleep_between_cycles, $number_of_times_to_skip_writing_sleeping) = self::getSleepParams('app_flatten_');
$last_worker_count = 0;
$iteration = 0;
$c = new Criteria();
$currentDc = kDataCenterMgr::getCurrentDc();
$c->add(BatchJobPeer::DC, kDataCenterMgr::getCurrentDcId());
$c->add(BatchJobPeer::JOB_TYPE, BatchJobType::FLATTEN);
$c->add(BatchJobPeer::STATUS, BatchJob::BATCHJOB_STATUS_PROCESSED);
$temp_count = 0;
while (1) {
self::exitIfDone();
try {
sleep($sleep_between_cycles);
$jobs = BatchJobPeer::doSelect($c);
foreach ($jobs as $job) {
$data = json_decode($job->getData(true), true);
$entry_id = $data['entryId'];
$entry_int_id = $data['entryIntId'];
$entry_version = $data['entryVersion'];
$file_format = $data['fileFormat'];
$entry = entryPeer::retrieveByPK($entry_id);
if (!$entry) {
// entry is probably deleted if it is not returned from retrieveByPK
// close job as failed
$job->setStatus(BatchJob::BATCHJOB_STATUS_FAILED);
$job->setDescription("could not retrieve entry, probably deleted");
TRACE("could not retrieve entry {$entry_id} , probably deleted");
$job->save();
continue;
}
$fileSyncKey = $entry->getSyncKey(entry::FILE_SYNC_ENTRY_SUB_TYPE_DOWNLOAD, $file_format);
$fullFinalPath = kFileSyncUtils::getLocalFilePathForKey($fileSyncKey);
$finalPathNoExt = substr($fullFinalPath, 0, strlen($fullFinalPath) - strlen($file_format));
myContentStorage::fullMkdir($fullFinalPath);
$wildcardFinalPath = $finalPathNoExt . "*";
$older_files = glob($wildcardFinalPath);
foreach ($older_files as $older_file) {
TRACE("removing old file: [{$older_file}]");
@unlink($older_file);
}
TRACE("Downloading: {$fullFinalPath}");
kFile::downloadUrlToFile($data["serverUrl"], $fullFinalPath);
if (!file_exists($fullFinalPath)) {
TRACE("file doesnt exist: " . $data["serverUrl"]);
$job->setDescription("file doesnt exist: " . $data["serverUrl"]);
$job->setStatus(BatchJob::BATCHJOB_STATUS_FAILED);
} else {
if (filesize($fullFinalPath) < 100000) {
@unlink($fullFinalPath);
TRACE("file too small: " . $data["serverUrl"]);
$job->setDescription("file too small: " . $data["serverUrl"]);
$job->setStatus(BatchJob::BATCHJOB_STATUS_FAILED);
} else {
if ($data['email']) {
$downloadLink = $entry->getDownloadUrl() . '/format/' . $file_format;
kJobsManager::addMailJob(null, $entry_id, $entry->getPartnerId(), self::KALTURAS_FLATTEN_READY, kMailJobData::MAIL_PRIORITY_NORMAL, kConf::get("batch_flatten_video_sender_email"), kConf::get("batch_flatten_video_sender_name"), $data['email'], array($data['email'], $downloadLink));
}
TRACE("Deleting: " . $data["deleteUrl"]);
kFile::downloadUrlToString($data["deleteUrl"]);
myNotificationMgr::createNotification(kNotificationJobData::NOTIFICATION_TYPE_ENTRY_UPDATE, $entry);
$job->setStatus(BatchJob::BATCHJOB_STATUS_FINISHED);
$filePath = kFileSyncUtils::getLocalFilePathForKey($fileSyncKey);
if (file_exists($filePath)) {
try {
kFileSyncUtils::createSyncFileForKey($fileSyncKey);
} catch (Exception $ex) {
TRACE("ignore ERROR: " . $ex->getMessage());
}
} else {
TRACE("The file [{$filePath}] doesn't exists, not creating FileSync");
}
}
}
$job->save();
}
} catch (Exception $ex) {
TRACE("ERROR: " . $ex->getMessage());
self::initDb(true);
self::failed();
}
if ($temp_count == 0) {
TRACE("Ended conversion. sleeping for a while (" . $sleep_between_cycles . " seconds). Will write to the log in (" . $sleep_between_cycles * $number_of_times_to_skip_writing_sleeping . ") seconds");
}
$temp_count++;
if ($temp_count >= $number_of_times_to_skip_writing_sleeping) {
$temp_count = 0;
}
}
}
private function sentToCenversion($write_to_log = true)
{
try {
$debug = array("before getFileToConvert");
list($before_archiving_file_path, $file_name, $in_proc) = $this->getFileToConvert($write_to_log);
$debug[] = "after getFileToConvert [{$before_archiving_file_path}] [{$file_name}]";
if (!$before_archiving_file_path) {
return;
}
// TODO - check if this file failed too many times ...
//if ( !$this->shouldHandleFile ( $file_name ) )
$entry_id = self::getEntryIdFromFileName($file_name);
$debug[] = "entry_id [{$entry_id}]";
// we have to retrieve the path of the entry - do so by setting the data to the file path (now rather than at the end)
$entry = entryPeer::retrieveByPK($entry_id);
$conv_profile = myPartnerUtils::getConversionProfileForEntry($entry_id);
$debug[] = "conversion profile of class [" . get_class($conv_profile) . "]";
if (!$entry) {
KalturaLog::debug("entry id [{$entry_id}] not found!");
return;
}
// the conversion target should be the entry's dataPath
$flv_file_name = kConversionHelper::flvFileName($before_archiving_file_path);
$debug[] = "flv_file_name [{$flv_file_name}]";
$entry->setData(null);
$entry->setData($flv_file_name);
// we assume the output will be of type FLV
$entry->save();
$archive_file_sync = $this->archiveFile($before_archiving_file_path);
$archived_file_path = $archive_file_sync->getFullPath();
$debug[] = "archived_file_path [{$archived_file_path}]";
if ($conv_profile->getBypassFlv() && kConversionHelper::isFlv($archived_file_path)) {
$conv_cmd = $this->createConversionCommandFromConverionProfile($archived_file_path, $archived_file_path, $conv_profile, $entry);
$debug[] = "before createConversionInDb[{$entry_id}] [{$archived_file_path}]";
// first update the DB
$this->createConversionInDb($entry_id, $archived_file_path, $conv_cmd);
// TODO - check if there is a set of convParams for this FLV profile and manye some conversion should be done
// for the edit version ??
KalturaLog::debug("Bypassing conversion for entry_id [{$entry_id}] file [{$file_name}]");
$conv_res = new kConversionResult($conv_cmd);
$conv_res_info = new kConvResInfo();
$conv_res_info->target = $archived_file_path;
$start = microtime(true);
// FileSync - soft copy
$archived_sync_key = $entry->getSyncKey(entry::FILE_SYNC_ENTRY_SUB_TYPE_ARCHIVE);
$data_sync_key = $entry->getSyncKey(entry::FILE_SYNC_ENTRY_SUB_TYPE_DATA);
kFileSyncUtils::softCopy($archived_sync_key, $data_sync_key);
$end = microtime(true);
$conv_res_info->duration = $end - $start;
$conv_res_info->conv_str = "NO CONVERT";
$conv_res->appendResInfo($conv_res_info);
$this->updateConvertedEntry(true, $entry, $conv_res);
$this->removeInProc($in_proc);
return;
}
// FileSync - create file sync for the future place of the converted data
$data_sync_key = $entry->getSyncKey(entry::FILE_SYNC_ENTRY_SUB_TYPE_DATA);
kFileSyncUtils::createSyncFileForKey($data_sync_key, false, false);
$full_target_path = kFileSyncUtils::getLocalFilePathForKey($data_sync_key, true);
if ($conv_profile->getProfileTypeSuffix() == "edit") {
// FileSync - create file sync for the future place of the converted data in edit flavor
$data_edit_sync_key = $entry->getSyncKey(entry::FILE_SYNC_ENTRY_SUB_TYPE_DATA_EDIT);
kFileSyncUtils::createSyncFileForKey($data_edit_sync_key, false, false);
}
$debug[] = "full_target_path [{$full_target_path}]";
$conv_cmd = $this->createConversionCommandFromConverionProfile($archived_file_path, $full_target_path, $conv_profile, $entry);
$debug[] = "before createConversionInDb[{$entry_id}] [{$archived_file_path}]";
// first update the DB
$this->createConversionInDb($entry_id, $archived_file_path, $conv_cmd);
KalturaLog::debug("Setting ConversionCommand for file [{$file_name}]\n" . print_r($conv_cmd, true));
$debug[] = "before saveConversionCommand";
// then save the conversion command
$cmd_file_path = $this->saveConversionCommand();
$this->removeInProc($in_proc);
KalturaLog::debug("Set ConversionCommand for file [{$file_name}] in [{$cmd_file_path}]");
$debug[] = "end";
} catch (kConversionException $kcoe) {
$this->removeInProc($in_proc);
KalturaLog::debug("Error:\n" . $kcoe->getMessage() . "\n" . $kcoe->getTraceAsString() . "\n" . print_r($debug));
// update the entry with the error sttus and the error message to the conversion result
$conv_res = new kConversionResult($conv_cmd);
$conv_res->appendResult($kcoe->getMessage());
$this->updateConvertedEntry(false, $entry, $conv_res);
} catch (Exception $ex) {
$this->removeInProc($in_proc);
KalturaLog::debug("Error:\n" . $ex->getMessage() . "\n" . $ex->getTraceAsString() . "\n" . print_r($debug));
// if this failed for some unknown reason - set it for reconversion
$indicator = $this->setFileToReConvert($before_archiving_file_path, $file_name);
KalturaLog::debug("... will reconvert [" . print_r($indicator, true) . "]");
throw $ex;
}
}
protected function archiveFile($file_name)
{
KalturaLog::debug("Archiving file [" . $file_name . "]");
$id = self::getEntryIdFromFileName($file_name);
$entry = entryPeer::retrieveByPKNoFilter($id);
$entry->setArchiveExtension(pathinfo($file_name, PATHINFO_EXTENSION));
$sync_key = $entry->getSyncKey(entry::FILE_SYNC_ENTRY_SUB_TYPE_ARCHIVE);
$file_sync = kFileSyncUtils::createSyncFileForKey($sync_key, false, false);
$target = kFileSyncUtils::getLocalFilePathForKey($sync_key, false);
KalturaLog::debug("Archiving file [" . $file_name . "] to [" . $target . "]}");
// MOVE - there is no need to copy because we the ConvCommand will include the file path anyway
if ($file_name == $target) {
KalturaLog::debug("File [{$file_name}] already archived");
return $file_sync;
}
// there is a file in the archive but the current file does not exist
if (!file_exists($file_name) && file_exists($target)) {
return $file_sync;
}
if (file_exists($target) && filesize($target) == filesize($file_name)) {
return $file_sync;
} else {
// move to archive and override if exists
myContentStorage::moveFile($file_name, $target, true, false);
}
kFileSyncUtils::markLocalFileSyncAsReady($sync_key);
return $file_sync;
}
