/**
* Serves multiple files for synchronization between datacenters
*/
public function execute()
{
$fileSyncIds = $this->getRequestParameter("ids");
$hash = $this->getRequestParameter("hash");
// validate hash
$currentDc = kDataCenterMgr::getCurrentDc();
$currentDcId = $currentDc["id"];
$expectedHash = md5($currentDc["secret"] . $fileSyncIds);
if ($hash !== $expectedHash) {
$error = "Invalid hash - ids [{$fileSyncIds}] got [{$hash}] expected [{$expectedHash}]";
KalturaLog::err($error);
KExternalErrors::dieError(KExternalErrors::INVALID_TOKEN);
}
// load file syncs
$fileSyncs = FileSyncPeer::retrieveByPks(explode(',', $fileSyncIds));
if ($fileSyncs) {
KalturaMonitorClient::initApiMonitor(false, 'extwidget.serveMultiFile', $fileSyncs[0]->getPartnerId());
}
// resolve file syncs
$filePaths = array();
foreach ($fileSyncs as $fileSync) {
if ($fileSync->getDc() != $currentDcId) {
$error = "FileSync id [" . $fileSync->getId() . "] does not belong to this DC";
KalturaLog::err($error);
KExternalErrors::dieError(KExternalErrors::BAD_QUERY);
}
// resolve if file_sync is link
$fileSyncResolved = kFileSyncUtils::resolve($fileSync);
// check if file sync path leads to a file or a directory
$resolvedPath = $fileSyncResolved->getFullPath();
if (is_dir($resolvedPath)) {
$error = "FileSync id [" . $fileSync->getId() . "] is a directory";
KalturaLog::err($error);
KExternalErrors::dieError(KExternalErrors::BAD_QUERY);
}
if (!file_exists($resolvedPath)) {
$error = "Path [{$resolvedPath}] for fileSync id [" . $fileSync->getId() . "] does not exist";
KalturaLog::err($error);
continue;
}
$filePaths[$fileSync->getId()] = $resolvedPath;
}
$boundary = md5(uniqid('', true));
header('Content-Type: multipart/form-data; boundary=' . $boundary);
foreach ($filePaths as $id => $filePath) {
echo "--{$boundary}\n";
echo "Content-Type: application/octet-stream\n";
echo "Content-Disposition: form-data; name=\"{$id}\"\n\n";
readfile($filePath);
echo "\n";
}
echo "--{$boundary}--\n";
KExternalErrors::dieGracefully();
}
public static function validateKs($job)
{
$dcParams = kDataCenterMgr::getCurrentDc();
$token = $dcParams["secret"];
$createdString = md5($job->getId() . $token);
$ks = kCurrentContext::$ks_object;
if ($createdString == $ks->additional_data) {
return true;
}
return false;
}
public function calculateId()
{
$dc = kDataCenterMgr::getCurrentDc();
for ($i = 0; $i < 10; $i++) {
$id = $dc["id"] . '_' . md5(microtime(true));
$existingObject = UploadTokenPeer::retrieveByPk($id);
if (!$existingObject) {
return $id;
}
}
throw new Exception("Could not calculate unique id for upload token");
}
private static function calculateId()
{
$dc = kDataCenterMgr::getCurrentDc();
for ($i = 0; $i < 10; ++$i) {
$id = $dc["id"] . '_' . kString::generateStringId();
$existing_object = entryPeer::retrieveByPk($id);
if (!$existing_object) {
return $id;
}
}
die;
}
private function calculateId()
{
$dc = kDataCenterMgr::getCurrentDc();
for ($i = 0; $i < 10; $i++) {
$id = $dc["id"] . '_' . kString::generateStringId();
$existingObject = AppTokenPeer::retrieveByPkNoFilter($id);
if ($existingObject) {
KalturaLog::log("ID [{$id}] already exists");
} else {
return $id;
}
}
throw new Exception("Could not find unique id for AppToken");
}
/**
* generate unique string id for annotation
*/
public function getUniqueAnnotationId()
{
$dc = kDataCenterMgr::getCurrentDc();
for ($i = 0; $i < 10; $i++) {
$id = $dc["id"] . '_' . kString::generateStringId();
$existingObject = AnnotationPeer::retrieveByPK($id);
if ($existingObject) {
KalturaLog::log(__METHOD__ . ": id [{$id}] already exists");
} else {
return $id;
}
}
throw new Exception("Could not find unique id for annotation");
}
/**
* @return string
*/
public static function generateKs($partnerId, $tokenPrefix)
{
$partner = PartnerPeer::retrieveByPK($partnerId);
$userSecret = $partner->getSecret();
//actionslimit:1
$privileges = kSessionBase::PRIVILEGE_SET_ROLE . ":" . self::EXTERNAL_INTEGRATION_SERVICES_ROLE_NAME;
$privileges .= "," . kSessionBase::PRIVILEGE_ACTIONS_LIMIT . ":1";
$dcParams = kDataCenterMgr::getCurrentDc();
$token = $dcParams["secret"];
$additionalData = md5($tokenPrefix . $token);
$ks = "";
$creationSucces = kSessionUtils::startKSession($partnerId, $userSecret, "", $ks, self::THREE_DAYS_IN_SECONDS, KalturaSessionType::USER, "", $privileges, null, $additionalData);
if ($creationSucces >= 0) {
return $ks;
}
return false;
}
protected function calculateId()
{
$allChars = '0123456789abcdefghijklmnopqrstuvwxyz';
$dcChars = str_split($allChars, strlen($allChars) / count(kDataCenterMgr::getAllDcs()));
$dc = kDataCenterMgr::getCurrentDc();
$dcId = (int) $dc["id"];
$currentDcChars = $dcChars[$dcId];
for ($i = 0; $i < 10; $i++) {
$dcChar = substr($currentDcChars, rand(0, strlen($currentDcChars) - 1), 1);
if (!$dcChar) {
$dcChar = '0';
}
$id = $dcChar . kString::generateStringId(3);
$existingObject = ShortLinkPeer::retrieveByPK($id);
if ($existingObject) {
KalturaLog::log("id [{$id}] already exists");
} else {
return $id;
}
}
throw new Exception("Could not find unique id for short link");
}
public static function calculateId($widget)
{
$dc = kDataCenterMgr::getCurrentDc();
for ($i = 0; $i < 10; ++$i) {
$id = $dc["id"] . '_' . kString::generateStringId();
$existing_widget = widgetPeer::retrieveByPk($id);
if (!$existing_widget) {
return $id;
}
}
die;
}
/**
* @param FileSyncKey $key
* @param $file_root
* @param $real_path
* @param $strict
* @return SyncFile
*/
public static function createSyncFileLinkForKey(FileSyncKey $target_key, FileSyncKey $source_key, $strict = true)
{
KalturaLog::log(__METHOD__ . " - target_key [{$target_key}], source_key [{$source_key}]");
// TODO - see that if in strict mode - there are no duplicate keys -> update existing records AND set the other DC's records to PENDING
$dc = kDataCenterMgr::getCurrentDc();
$dc_id = $dc["id"];
$sourceFile = self::getLocalFileSyncForKey($source_key, $strict);
if (!$sourceFile) {
KalturaLog::log(__METHOD__ . " - Warning: no source but NOT strict. target_key [{$target_key}], source_key [{$source_key}] ");
return null;
}
$sourceFile = self::resolve($sourceFile);
// we only want to link to a source and not to a link.
// create a FileSync for the current DC with status READY
$current_dc_file_sync = FileSync::createForFileSyncKey($target_key);
$current_dc_file_sync->setDc($dc_id);
$current_dc_file_sync->setPartnerId($target_key->partner_id);
$current_dc_file_sync->setFileSize(-1);
$current_dc_file_sync->setStatus(FileSync::FILE_SYNC_STATUS_READY);
$current_dc_file_sync->setReadyAt(time());
$current_dc_file_sync->setOriginal(1);
$current_dc_file_sync->setLinkedId($sourceFile->getId());
$current_dc_file_sync->setFileType(FileSync::FILE_SYNC_FILE_TYPE_LINK);
$current_dc_file_sync->save();
//increment link_count for this DC source]
self::incrementLinkCountForFileSync($sourceFile);
$c = new Criteria();
$c = FileSyncPeer::getCriteriaForFileSyncKey($source_key);
$file_sync_list = FileSyncPeer::doSelect($c);
$source_file_syncs = array();
foreach ($file_sync_list as $file_sync) {
$file_sync = self::resolve($file_sync);
// we only want to link to a source and not to a link.
$source_file_syncs[$file_sync->getDc()] = $file_sync;
}
// create records for all other DCs with status READY
// link is always ready since no one is going to fetch it.
$other_dcs = kDataCenterMgr::getAllDcs();
foreach ($other_dcs as $remote_dc) {
// TODO - maybe we should create the file sync
if (!isset($source_file_syncs[$remote_dc["id"]])) {
continue;
}
$remote_dc_file_sync = FileSync::createForFileSyncKey($target_key);
$remote_dc_file_sync->setDc($remote_dc["id"]);
$remote_dc_file_sync->setStatus(FileSync::FILE_SYNC_STATUS_READY);
$remote_dc_file_sync->setOriginal(0);
$remote_dc_file_sync->setFileType(FileSync::FILE_SYNC_FILE_TYPE_LINK);
$remote_dc_file_sync->setReadyAt(time());
$remote_dc_file_sync->setPartnerID($target_key->partner_id);
$remote_dc_file_sync->setLinkedId($source_file_syncs[$remote_dc["id"]]->getId());
$remote_dc_file_sync->save();
// increment link_cont for remote DCs sources
self::incrementLinkCountForFileSync($source_file_syncs[$remote_dc["id"]]);
kEventsManager::raiseEvent(new kObjectAddedEvent($remote_dc_file_sync));
}
kEventsManager::raiseEvent(new kObjectAddedEvent($current_dc_file_sync));
}
protected static function createLiveReportExportDownloadUrl($partner_id, $file_name, $expiry, $applicationUrlTemplate)
{
// Extract simple download name
$regex = "/^{$partner_id}_Export_[a-zA-Z0-9]+_(?<fileName>[\\w\\-]+.csv)\$/";
if (!preg_match($regex, $file_name, $matches)) {
KalturaLog::err("File name doesn't match expected format");
return null;
}
$downloadName = $matches['fileName'];
// Add dc to enable redirection
$dc = kDataCenterMgr::getCurrentDc();
$file_name = $dc['id'] . "_" . $file_name;
$ksStr = "";
$partner = PartnerPeer::retrieveByPK($partner_id);
$secret = $partner->getSecret();
$privilege = ks::PRIVILEGE_DOWNLOAD . ":" . $file_name;
$ksStr = kSessionBase::generateSession($partner->getKSVersion(), $partner->getAdminSecret(), null, ks::TYPE_KS, $partner_id, $expiry, $privilege);
if ($applicationUrlTemplate) {
$url = str_replace("[ks]", $ksStr, $applicationUrlTemplate);
$url = str_replace("[id]", $file_name, $url);
} else {
//url is built with DC url in order to be directed to the same DC of the saved file
$url = kDataCenterMgr::getCurrentDcUrl() . "/api_v3/index.php/service/liveReports/action/serveReport/ks/{$ksStr}/id/{$file_name}/{$downloadName}";
}
return $url;
}
/**
* @param FileSyncKey $key
* @param $file_root
* @param $real_path
* @return SyncFile
*/
public static function createSyncFileLinkForKey(FileSyncKey $target_key, FileSyncKey $source_key)
{
KalturaLog::debug("target_key [{$target_key}], source_key [{$source_key}]");
// TODO - see that if in strict mode - there are no duplicate keys -> update existing records AND set the other DC's records to PENDING
$dc = kDataCenterMgr::getCurrentDc();
$dc_id = $dc["id"];
// load all source file syncs
$c = new Criteria();
$c = FileSyncPeer::getCriteriaForFileSyncKey($source_key);
$file_sync_list = FileSyncPeer::doSelect($c);
if (!$file_sync_list) {
KalturaLog::notice("Warning: no source. target_key [{$target_key}], source_key [{$source_key}] ");
return null;
}
$source_file_syncs = array();
foreach ($file_sync_list as $file_sync) {
$file_sync = self::resolve($file_sync);
// we only want to link to a source and not to a link.
$source_file_syncs[] = $file_sync;
}
// find the current dc file sync
$current_dc_source_file = null;
foreach ($source_file_syncs as $source_file_sync) {
if ($source_file_sync->getDc() == $dc_id) {
$current_dc_source_file = $source_file_sync;
}
}
if (!$current_dc_source_file) {
$current_dc_source_file = reset($source_file_syncs);
}
// create the remote file syncs
foreach ($source_file_syncs as $source_file_sync) {
$remote_dc_file_sync = FileSync::createForFileSyncKey($target_key);
$remote_dc_file_sync->setDc($source_file_sync->getDc());
$remote_dc_file_sync->setStatus($source_file_sync->getStatus());
$remote_dc_file_sync->setOriginal($current_dc_source_file == $source_file_sync);
$remote_dc_file_sync->setFileSize(-1);
if ($source_file_sync->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_URL) {
$remote_dc_file_sync->setFileType(FileSync::FILE_SYNC_FILE_TYPE_URL);
$remote_dc_file_sync->setFileRoot($source_file_sync->getFileRoot());
$remote_dc_file_sync->setFilePath($source_file_sync->getFilePath());
} else {
$remote_dc_file_sync->setFileType(FileSync::FILE_SYNC_FILE_TYPE_LINK);
}
$remote_dc_file_sync->setLinkedId($source_file_sync->getId());
self::incrementLinkCountForFileSync($source_file_sync);
$remote_dc_file_sync->setPartnerID($target_key->partner_id);
$remote_dc_file_sync->save();
if ($current_dc_source_file == $source_file_sync) {
$current_dc_target_file = $remote_dc_file_sync;
} else {
kEventsManager::raiseEvent(new kObjectAddedEvent($remote_dc_file_sync));
}
}
kEventsManager::raiseEvent(new kObjectAddedEvent($current_dc_target_file));
}
/**
* @param FileSyncKey $key
* @param $file_root
* @param $real_path
* @return SyncFile
*/
public static function createSyncFileLinkForKey(FileSyncKey $target_key, FileSyncKey $source_key)
{
KalturaLog::log(__METHOD__ . " - target_key [{$target_key}], source_key [{$source_key}]");
// TODO - see that if in strict mode - there are no duplicate keys -> update existing records AND set the other DC's records to PENDING
$dc = kDataCenterMgr::getCurrentDc();
$dc_id = $dc["id"];
list($sourceFile, $local) = self::getReadyFileSyncForKey($source_key, true, false);
if (!$sourceFile) {
KalturaLog::log(__METHOD__ . " - Warning: no source. target_key [{$target_key}], source_key [{$source_key}] ");
return null;
}
$sourceFile = self::resolve($sourceFile);
// we only want to link to a source and not to a link.
// create a FileSync for the current DC with status READY
$current_dc_file_sync = FileSync::createForFileSyncKey($target_key);
$current_dc_file_sync->setPartnerId($target_key->partner_id);
$current_dc_file_sync->setFileSize(-1);
$current_dc_file_sync->setStatus($sourceFile->getStatus());
$current_dc_file_sync->setOriginal(1);
if ($sourceFile->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_URL) {
$current_dc_file_sync->setFileType(FileSync::FILE_SYNC_FILE_TYPE_URL);
$current_dc_file_sync->setDc($sourceFile->getDc());
$current_dc_file_sync->setFileRoot($sourceFile->getFileRoot());
$current_dc_file_sync->setFilePath($sourceFile->getFilePath());
} else {
$current_dc_file_sync->setFileType(FileSync::FILE_SYNC_FILE_TYPE_LINK);
$current_dc_file_sync->setDc($dc_id);
$current_dc_file_sync->setLinkedId($sourceFile->getId());
}
$current_dc_file_sync->save();
//increment link_count for this DC source]
if ($sourceFile->getFileType() != FileSync::FILE_SYNC_FILE_TYPE_URL) {
self::incrementLinkCountForFileSync($sourceFile);
}
$c = new Criteria();
$c = FileSyncPeer::getCriteriaForFileSyncKey($source_key);
$file_sync_list = FileSyncPeer::doSelect($c);
$source_file_syncs = array();
foreach ($file_sync_list as $file_sync) {
$file_sync = self::resolve($file_sync);
// we only want to link to a source and not to a link.
$source_file_syncs[$file_sync->getDc()] = $file_sync;
}
foreach ($source_file_syncs as $remote_dc_id => $source_file_sync) {
if ($source_file_sync->getDc() == $current_dc_file_sync->getDc()) {
continue;
}
$remote_dc_file_sync = FileSync::createForFileSyncKey($target_key);
$remote_dc_file_sync->setDc($remote_dc_id);
$remote_dc_file_sync->setStatus($source_file_sync->getStatus());
$remote_dc_file_sync->setOriginal(0);
if ($source_file_sync->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_URL) {
$remote_dc_file_sync->setFileType(FileSync::FILE_SYNC_FILE_TYPE_URL);
$remote_dc_file_sync->setFileRoot($source_file_sync->getFileRoot());
$remote_dc_file_sync->setFilePath($source_file_sync->getFilePath());
} else {
$remote_dc_file_sync->setFileType(FileSync::FILE_SYNC_FILE_TYPE_LINK);
$remote_dc_file_sync->setLinkedId($source_file_sync->getId());
}
$remote_dc_file_sync->setPartnerID($target_key->partner_id);
$remote_dc_file_sync->save();
// increment link_cont for remote DCs sources
if ($source_file_sync->getFileType() != FileSync::FILE_SYNC_FILE_TYPE_URL) {
self::incrementLinkCountForFileSync($source_file_sync);
}
kEventsManager::raiseEvent(new kObjectAddedEvent($remote_dc_file_sync));
}
kEventsManager::raiseEvent(new kObjectAddedEvent($current_dc_file_sync));
}
public function myBatchFlattenServer($script_name)
{
$this->script_name = $script_name;
$this->register($script_name);
SET_CONTEXT("FS");
$MAX_ITERATIONS_DUE_TO_PROPEL_MEMORY_LEAK = 10000000;
self::initDb();
list($sleep_between_cycles, $number_of_times_to_skip_writing_sleeping) = self::getSleepParams('app_flatten_');
$last_worker_count = 0;
$iteration = 0;
$c = new Criteria();
$currentDc = kDataCenterMgr::getCurrentDc();
$c->add(BatchJobPeer::DC, kDataCenterMgr::getCurrentDcId());
$c->add(BatchJobPeer::JOB_TYPE, BatchJobType::FLATTEN);
$c->add(BatchJobPeer::STATUS, BatchJob::BATCHJOB_STATUS_PROCESSED);
$temp_count = 0;
while (1) {
self::exitIfDone();
try {
sleep($sleep_between_cycles);
$jobs = BatchJobPeer::doSelect($c);
foreach ($jobs as $job) {
$data = json_decode($job->getData(true), true);
$entry_id = $data['entryId'];
$entry_int_id = $data['entryIntId'];
$entry_version = $data['entryVersion'];
$file_format = $data['fileFormat'];
$entry = entryPeer::retrieveByPK($entry_id);
if (!$entry) {
// entry is probably deleted if it is not returned from retrieveByPK
// close job as failed
$job->setStatus(BatchJob::BATCHJOB_STATUS_FAILED);
$job->setDescription("could not retrieve entry, probably deleted");
TRACE("could not retrieve entry {$entry_id} , probably deleted");
$job->save();
continue;
}
$fileSyncKey = $entry->getSyncKey(entry::FILE_SYNC_ENTRY_SUB_TYPE_DOWNLOAD, $file_format);
$fullFinalPath = kFileSyncUtils::getLocalFilePathForKey($fileSyncKey);
$finalPathNoExt = substr($fullFinalPath, 0, strlen($fullFinalPath) - strlen($file_format));
myContentStorage::fullMkdir($fullFinalPath);
$wildcardFinalPath = $finalPathNoExt . "*";
$older_files = glob($wildcardFinalPath);
foreach ($older_files as $older_file) {
TRACE("removing old file: [{$older_file}]");
@unlink($older_file);
}
TRACE("Downloading: {$fullFinalPath}");
kFile::downloadUrlToFile($data["serverUrl"], $fullFinalPath);
if (!file_exists($fullFinalPath)) {
TRACE("file doesnt exist: " . $data["serverUrl"]);
$job->setDescription("file doesnt exist: " . $data["serverUrl"]);
$job->setStatus(BatchJob::BATCHJOB_STATUS_FAILED);
} else {
if (filesize($fullFinalPath) < 100000) {
@unlink($fullFinalPath);
TRACE("file too small: " . $data["serverUrl"]);
$job->setDescription("file too small: " . $data["serverUrl"]);
$job->setStatus(BatchJob::BATCHJOB_STATUS_FAILED);
} else {
if ($data['email']) {
$downloadLink = $entry->getDownloadUrl() . '/format/' . $file_format;
kJobsManager::addMailJob(null, $entry_id, $entry->getPartnerId(), self::KALTURAS_FLATTEN_READY, kMailJobData::MAIL_PRIORITY_NORMAL, kConf::get("batch_flatten_video_sender_email"), kConf::get("batch_flatten_video_sender_name"), $data['email'], array($data['email'], $downloadLink));
}
TRACE("Deleting: " . $data["deleteUrl"]);
kFile::downloadUrlToString($data["deleteUrl"]);
myNotificationMgr::createNotification(kNotificationJobData::NOTIFICATION_TYPE_ENTRY_UPDATE, $entry);
$job->setStatus(BatchJob::BATCHJOB_STATUS_FINISHED);
$filePath = kFileSyncUtils::getLocalFilePathForKey($fileSyncKey);
if (file_exists($filePath)) {
try {
kFileSyncUtils::createSyncFileForKey($fileSyncKey);
} catch (Exception $ex) {
TRACE("ignore ERROR: " . $ex->getMessage());
}
} else {
TRACE("The file [{$filePath}] doesn't exists, not creating FileSync");
}
}
}
$job->save();
}
} catch (Exception $ex) {
TRACE("ERROR: " . $ex->getMessage());
self::initDb(true);
self::failed();
}
if ($temp_count == 0) {
TRACE("Ended conversion. sleeping for a while (" . $sleep_between_cycles . " seconds). Will write to the log in (" . $sleep_between_cycles * $number_of_times_to_skip_writing_sleeping . ") seconds");
}
$temp_count++;
if ($temp_count >= $number_of_times_to_skip_writing_sleeping) {
$temp_count = 0;
}
}
}
private function calculateId()
{
$dc = kDataCenterMgr::getCurrentDc();
for ($i = 0; $i < 10; $i++) {
$id = $dc["id"] . '_' . kString::generateStringId();
$existingObject = flavorAssetPeer::retrieveById($id);
if ($existingObject) {
KalturaLog::log(__METHOD__ . ": id [{$id}] already exists");
} else {
return $id;
}
}
throw new Exception("Could not find unique id for flavorAsset");
}
/**
* This function returns the FILE SYSTEM path to the root content folder.
* @return string the content folder file system path
*/
public static function getFSContentRootPath()
{
$dc = kDataCenterMgr::getCurrentDc();
return $dc["root"];
//return realpath(sfConfig::get('sf_root_dir')."/../../").'/';
}
public static function enforceCdnDelivery($partnerId)
{
$host = requestUtils::getHost();
$cdnHost = myPartnerUtils::getCdnHost($partnerId);
$dc = kDataCenterMgr::getCurrentDc();
$external_url = $dc["external_url"];
// allow access only via cdn or via proxy from secondary datacenter
if ($host != $cdnHost && $host != $external_url) {
$uri = $_SERVER["REQUEST_URI"];
if (strpos($uri, "/forceproxy/true") === false) {
$uri .= "/forceproxy/true/";
}
header('Location:' . $cdnHost . $uri);
header("X-Kaltura:enforce-cdn");
die;
}
}
