/** * Serves multiple files for synchronization between datacenters */ public function execute() { $fileSyncIds = $this->getRequestParameter("ids"); $hash = $this->getRequestParameter("hash"); // validate hash $currentDc = kDataCenterMgr::getCurrentDc(); $currentDcId = $currentDc["id"]; $expectedHash = md5($currentDc["secret"] . $fileSyncIds); if ($hash !== $expectedHash) { $error = "Invalid hash - ids [{$fileSyncIds}] got [{$hash}] expected [{$expectedHash}]"; KalturaLog::err($error); KExternalErrors::dieError(KExternalErrors::INVALID_TOKEN); } // load file syncs $fileSyncs = FileSyncPeer::retrieveByPks(explode(',', $fileSyncIds)); if ($fileSyncs) { KalturaMonitorClient::initApiMonitor(false, 'extwidget.serveMultiFile', $fileSyncs[0]->getPartnerId()); } // resolve file syncs $filePaths = array(); foreach ($fileSyncs as $fileSync) { if ($fileSync->getDc() != $currentDcId) { $error = "FileSync id [" . $fileSync->getId() . "] does not belong to this DC"; KalturaLog::err($error); KExternalErrors::dieError(KExternalErrors::BAD_QUERY); } // resolve if file_sync is link $fileSyncResolved = kFileSyncUtils::resolve($fileSync); // check if file sync path leads to a file or a directory $resolvedPath = $fileSyncResolved->getFullPath(); if (is_dir($resolvedPath)) { $error = "FileSync id [" . $fileSync->getId() . "] is a directory"; KalturaLog::err($error); KExternalErrors::dieError(KExternalErrors::BAD_QUERY); } if (!file_exists($resolvedPath)) { $error = "Path [{$resolvedPath}] for fileSync id [" . $fileSync->getId() . "] does not exist"; KalturaLog::err($error); continue; } $filePaths[$fileSync->getId()] = $resolvedPath; } $boundary = md5(uniqid('', true)); header('Content-Type: multipart/form-data; boundary=' . $boundary); foreach ($filePaths as $id => $filePath) { echo "--{$boundary}\n"; echo "Content-Type: application/octet-stream\n"; echo "Content-Disposition: form-data; name=\"{$id}\"\n\n"; readfile($filePath); echo "\n"; } echo "--{$boundary}--\n"; KExternalErrors::dieGracefully(); }
public static function validateKs($job) { $dcParams = kDataCenterMgr::getCurrentDc(); $token = $dcParams["secret"]; $createdString = md5($job->getId() . $token); $ks = kCurrentContext::$ks_object; if ($createdString == $ks->additional_data) { return true; } return false; }
public function calculateId() { $dc = kDataCenterMgr::getCurrentDc(); for ($i = 0; $i < 10; $i++) { $id = $dc["id"] . '_' . md5(microtime(true)); $existingObject = UploadTokenPeer::retrieveByPk($id); if (!$existingObject) { return $id; } } throw new Exception("Could not calculate unique id for upload token"); }
private static function calculateId() { $dc = kDataCenterMgr::getCurrentDc(); for ($i = 0; $i < 10; ++$i) { $id = $dc["id"] . '_' . kString::generateStringId(); $existing_object = entryPeer::retrieveByPk($id); if (!$existing_object) { return $id; } } die; }
private function calculateId() { $dc = kDataCenterMgr::getCurrentDc(); for ($i = 0; $i < 10; $i++) { $id = $dc["id"] . '_' . kString::generateStringId(); $existingObject = AppTokenPeer::retrieveByPkNoFilter($id); if ($existingObject) { KalturaLog::log("ID [{$id}] already exists"); } else { return $id; } } throw new Exception("Could not find unique id for AppToken"); }
/** * generate unique string id for annotation */ public function getUniqueAnnotationId() { $dc = kDataCenterMgr::getCurrentDc(); for ($i = 0; $i < 10; $i++) { $id = $dc["id"] . '_' . kString::generateStringId(); $existingObject = AnnotationPeer::retrieveByPK($id); if ($existingObject) { KalturaLog::log(__METHOD__ . ": id [{$id}] already exists"); } else { return $id; } } throw new Exception("Could not find unique id for annotation"); }
/** * @return string */ public static function generateKs($partnerId, $tokenPrefix) { $partner = PartnerPeer::retrieveByPK($partnerId); $userSecret = $partner->getSecret(); //actionslimit:1 $privileges = kSessionBase::PRIVILEGE_SET_ROLE . ":" . self::EXTERNAL_INTEGRATION_SERVICES_ROLE_NAME; $privileges .= "," . kSessionBase::PRIVILEGE_ACTIONS_LIMIT . ":1"; $dcParams = kDataCenterMgr::getCurrentDc(); $token = $dcParams["secret"]; $additionalData = md5($tokenPrefix . $token); $ks = ""; $creationSucces = kSessionUtils::startKSession($partnerId, $userSecret, "", $ks, self::THREE_DAYS_IN_SECONDS, KalturaSessionType::USER, "", $privileges, null, $additionalData); if ($creationSucces >= 0) { return $ks; } return false; }
protected function calculateId() { $allChars = '0123456789abcdefghijklmnopqrstuvwxyz'; $dcChars = str_split($allChars, strlen($allChars) / count(kDataCenterMgr::getAllDcs())); $dc = kDataCenterMgr::getCurrentDc(); $dcId = (int) $dc["id"]; $currentDcChars = $dcChars[$dcId]; for ($i = 0; $i < 10; $i++) { $dcChar = substr($currentDcChars, rand(0, strlen($currentDcChars) - 1), 1); if (!$dcChar) { $dcChar = '0'; } $id = $dcChar . kString::generateStringId(3); $existingObject = ShortLinkPeer::retrieveByPK($id); if ($existingObject) { KalturaLog::log("id [{$id}] already exists"); } else { return $id; } } throw new Exception("Could not find unique id for short link"); }
public static function calculateId($widget) { $dc = kDataCenterMgr::getCurrentDc(); for ($i = 0; $i < 10; ++$i) { $id = $dc["id"] . '_' . kString::generateStringId(); $existing_widget = widgetPeer::retrieveByPk($id); if (!$existing_widget) { return $id; } } die; }
/** * @param FileSyncKey $key * @param $file_root * @param $real_path * @param $strict * @return SyncFile */ public static function createSyncFileLinkForKey(FileSyncKey $target_key, FileSyncKey $source_key, $strict = true) { KalturaLog::log(__METHOD__ . " - target_key [{$target_key}], source_key [{$source_key}]"); // TODO - see that if in strict mode - there are no duplicate keys -> update existing records AND set the other DC's records to PENDING $dc = kDataCenterMgr::getCurrentDc(); $dc_id = $dc["id"]; $sourceFile = self::getLocalFileSyncForKey($source_key, $strict); if (!$sourceFile) { KalturaLog::log(__METHOD__ . " - Warning: no source but NOT strict. target_key [{$target_key}], source_key [{$source_key}] "); return null; } $sourceFile = self::resolve($sourceFile); // we only want to link to a source and not to a link. // create a FileSync for the current DC with status READY $current_dc_file_sync = FileSync::createForFileSyncKey($target_key); $current_dc_file_sync->setDc($dc_id); $current_dc_file_sync->setPartnerId($target_key->partner_id); $current_dc_file_sync->setFileSize(-1); $current_dc_file_sync->setStatus(FileSync::FILE_SYNC_STATUS_READY); $current_dc_file_sync->setReadyAt(time()); $current_dc_file_sync->setOriginal(1); $current_dc_file_sync->setLinkedId($sourceFile->getId()); $current_dc_file_sync->setFileType(FileSync::FILE_SYNC_FILE_TYPE_LINK); $current_dc_file_sync->save(); //increment link_count for this DC source] self::incrementLinkCountForFileSync($sourceFile); $c = new Criteria(); $c = FileSyncPeer::getCriteriaForFileSyncKey($source_key); $file_sync_list = FileSyncPeer::doSelect($c); $source_file_syncs = array(); foreach ($file_sync_list as $file_sync) { $file_sync = self::resolve($file_sync); // we only want to link to a source and not to a link. $source_file_syncs[$file_sync->getDc()] = $file_sync; } // create records for all other DCs with status READY // link is always ready since no one is going to fetch it. $other_dcs = kDataCenterMgr::getAllDcs(); foreach ($other_dcs as $remote_dc) { // TODO - maybe we should create the file sync if (!isset($source_file_syncs[$remote_dc["id"]])) { continue; } $remote_dc_file_sync = FileSync::createForFileSyncKey($target_key); $remote_dc_file_sync->setDc($remote_dc["id"]); $remote_dc_file_sync->setStatus(FileSync::FILE_SYNC_STATUS_READY); $remote_dc_file_sync->setOriginal(0); $remote_dc_file_sync->setFileType(FileSync::FILE_SYNC_FILE_TYPE_LINK); $remote_dc_file_sync->setReadyAt(time()); $remote_dc_file_sync->setPartnerID($target_key->partner_id); $remote_dc_file_sync->setLinkedId($source_file_syncs[$remote_dc["id"]]->getId()); $remote_dc_file_sync->save(); // increment link_cont for remote DCs sources self::incrementLinkCountForFileSync($source_file_syncs[$remote_dc["id"]]); kEventsManager::raiseEvent(new kObjectAddedEvent($remote_dc_file_sync)); } kEventsManager::raiseEvent(new kObjectAddedEvent($current_dc_file_sync)); }
protected static function createLiveReportExportDownloadUrl($partner_id, $file_name, $expiry, $applicationUrlTemplate) { // Extract simple download name $regex = "/^{$partner_id}_Export_[a-zA-Z0-9]+_(?<fileName>[\\w\\-]+.csv)\$/"; if (!preg_match($regex, $file_name, $matches)) { KalturaLog::err("File name doesn't match expected format"); return null; } $downloadName = $matches['fileName']; // Add dc to enable redirection $dc = kDataCenterMgr::getCurrentDc(); $file_name = $dc['id'] . "_" . $file_name; $ksStr = ""; $partner = PartnerPeer::retrieveByPK($partner_id); $secret = $partner->getSecret(); $privilege = ks::PRIVILEGE_DOWNLOAD . ":" . $file_name; $ksStr = kSessionBase::generateSession($partner->getKSVersion(), $partner->getAdminSecret(), null, ks::TYPE_KS, $partner_id, $expiry, $privilege); if ($applicationUrlTemplate) { $url = str_replace("[ks]", $ksStr, $applicationUrlTemplate); $url = str_replace("[id]", $file_name, $url); } else { //url is built with DC url in order to be directed to the same DC of the saved file $url = kDataCenterMgr::getCurrentDcUrl() . "/api_v3/index.php/service/liveReports/action/serveReport/ks/{$ksStr}/id/{$file_name}/{$downloadName}"; } return $url; }
/** * @param FileSyncKey $key * @param $file_root * @param $real_path * @return SyncFile */ public static function createSyncFileLinkForKey(FileSyncKey $target_key, FileSyncKey $source_key) { KalturaLog::debug("target_key [{$target_key}], source_key [{$source_key}]"); // TODO - see that if in strict mode - there are no duplicate keys -> update existing records AND set the other DC's records to PENDING $dc = kDataCenterMgr::getCurrentDc(); $dc_id = $dc["id"]; // load all source file syncs $c = new Criteria(); $c = FileSyncPeer::getCriteriaForFileSyncKey($source_key); $file_sync_list = FileSyncPeer::doSelect($c); if (!$file_sync_list) { KalturaLog::notice("Warning: no source. target_key [{$target_key}], source_key [{$source_key}] "); return null; } $source_file_syncs = array(); foreach ($file_sync_list as $file_sync) { $file_sync = self::resolve($file_sync); // we only want to link to a source and not to a link. $source_file_syncs[] = $file_sync; } // find the current dc file sync $current_dc_source_file = null; foreach ($source_file_syncs as $source_file_sync) { if ($source_file_sync->getDc() == $dc_id) { $current_dc_source_file = $source_file_sync; } } if (!$current_dc_source_file) { $current_dc_source_file = reset($source_file_syncs); } // create the remote file syncs foreach ($source_file_syncs as $source_file_sync) { $remote_dc_file_sync = FileSync::createForFileSyncKey($target_key); $remote_dc_file_sync->setDc($source_file_sync->getDc()); $remote_dc_file_sync->setStatus($source_file_sync->getStatus()); $remote_dc_file_sync->setOriginal($current_dc_source_file == $source_file_sync); $remote_dc_file_sync->setFileSize(-1); if ($source_file_sync->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_URL) { $remote_dc_file_sync->setFileType(FileSync::FILE_SYNC_FILE_TYPE_URL); $remote_dc_file_sync->setFileRoot($source_file_sync->getFileRoot()); $remote_dc_file_sync->setFilePath($source_file_sync->getFilePath()); } else { $remote_dc_file_sync->setFileType(FileSync::FILE_SYNC_FILE_TYPE_LINK); } $remote_dc_file_sync->setLinkedId($source_file_sync->getId()); self::incrementLinkCountForFileSync($source_file_sync); $remote_dc_file_sync->setPartnerID($target_key->partner_id); $remote_dc_file_sync->save(); if ($current_dc_source_file == $source_file_sync) { $current_dc_target_file = $remote_dc_file_sync; } else { kEventsManager::raiseEvent(new kObjectAddedEvent($remote_dc_file_sync)); } } kEventsManager::raiseEvent(new kObjectAddedEvent($current_dc_target_file)); }
/** * @param FileSyncKey $key * @param $file_root * @param $real_path * @return SyncFile */ public static function createSyncFileLinkForKey(FileSyncKey $target_key, FileSyncKey $source_key) { KalturaLog::log(__METHOD__ . " - target_key [{$target_key}], source_key [{$source_key}]"); // TODO - see that if in strict mode - there are no duplicate keys -> update existing records AND set the other DC's records to PENDING $dc = kDataCenterMgr::getCurrentDc(); $dc_id = $dc["id"]; list($sourceFile, $local) = self::getReadyFileSyncForKey($source_key, true, false); if (!$sourceFile) { KalturaLog::log(__METHOD__ . " - Warning: no source. target_key [{$target_key}], source_key [{$source_key}] "); return null; } $sourceFile = self::resolve($sourceFile); // we only want to link to a source and not to a link. // create a FileSync for the current DC with status READY $current_dc_file_sync = FileSync::createForFileSyncKey($target_key); $current_dc_file_sync->setPartnerId($target_key->partner_id); $current_dc_file_sync->setFileSize(-1); $current_dc_file_sync->setStatus($sourceFile->getStatus()); $current_dc_file_sync->setOriginal(1); if ($sourceFile->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_URL) { $current_dc_file_sync->setFileType(FileSync::FILE_SYNC_FILE_TYPE_URL); $current_dc_file_sync->setDc($sourceFile->getDc()); $current_dc_file_sync->setFileRoot($sourceFile->getFileRoot()); $current_dc_file_sync->setFilePath($sourceFile->getFilePath()); } else { $current_dc_file_sync->setFileType(FileSync::FILE_SYNC_FILE_TYPE_LINK); $current_dc_file_sync->setDc($dc_id); $current_dc_file_sync->setLinkedId($sourceFile->getId()); } $current_dc_file_sync->save(); //increment link_count for this DC source] if ($sourceFile->getFileType() != FileSync::FILE_SYNC_FILE_TYPE_URL) { self::incrementLinkCountForFileSync($sourceFile); } $c = new Criteria(); $c = FileSyncPeer::getCriteriaForFileSyncKey($source_key); $file_sync_list = FileSyncPeer::doSelect($c); $source_file_syncs = array(); foreach ($file_sync_list as $file_sync) { $file_sync = self::resolve($file_sync); // we only want to link to a source and not to a link. $source_file_syncs[$file_sync->getDc()] = $file_sync; } foreach ($source_file_syncs as $remote_dc_id => $source_file_sync) { if ($source_file_sync->getDc() == $current_dc_file_sync->getDc()) { continue; } $remote_dc_file_sync = FileSync::createForFileSyncKey($target_key); $remote_dc_file_sync->setDc($remote_dc_id); $remote_dc_file_sync->setStatus($source_file_sync->getStatus()); $remote_dc_file_sync->setOriginal(0); if ($source_file_sync->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_URL) { $remote_dc_file_sync->setFileType(FileSync::FILE_SYNC_FILE_TYPE_URL); $remote_dc_file_sync->setFileRoot($source_file_sync->getFileRoot()); $remote_dc_file_sync->setFilePath($source_file_sync->getFilePath()); } else { $remote_dc_file_sync->setFileType(FileSync::FILE_SYNC_FILE_TYPE_LINK); $remote_dc_file_sync->setLinkedId($source_file_sync->getId()); } $remote_dc_file_sync->setPartnerID($target_key->partner_id); $remote_dc_file_sync->save(); // increment link_cont for remote DCs sources if ($source_file_sync->getFileType() != FileSync::FILE_SYNC_FILE_TYPE_URL) { self::incrementLinkCountForFileSync($source_file_sync); } kEventsManager::raiseEvent(new kObjectAddedEvent($remote_dc_file_sync)); } kEventsManager::raiseEvent(new kObjectAddedEvent($current_dc_file_sync)); }
public function myBatchFlattenServer($script_name) { $this->script_name = $script_name; $this->register($script_name); SET_CONTEXT("FS"); $MAX_ITERATIONS_DUE_TO_PROPEL_MEMORY_LEAK = 10000000; self::initDb(); list($sleep_between_cycles, $number_of_times_to_skip_writing_sleeping) = self::getSleepParams('app_flatten_'); $last_worker_count = 0; $iteration = 0; $c = new Criteria(); $currentDc = kDataCenterMgr::getCurrentDc(); $c->add(BatchJobPeer::DC, kDataCenterMgr::getCurrentDcId()); $c->add(BatchJobPeer::JOB_TYPE, BatchJobType::FLATTEN); $c->add(BatchJobPeer::STATUS, BatchJob::BATCHJOB_STATUS_PROCESSED); $temp_count = 0; while (1) { self::exitIfDone(); try { sleep($sleep_between_cycles); $jobs = BatchJobPeer::doSelect($c); foreach ($jobs as $job) { $data = json_decode($job->getData(true), true); $entry_id = $data['entryId']; $entry_int_id = $data['entryIntId']; $entry_version = $data['entryVersion']; $file_format = $data['fileFormat']; $entry = entryPeer::retrieveByPK($entry_id); if (!$entry) { // entry is probably deleted if it is not returned from retrieveByPK // close job as failed $job->setStatus(BatchJob::BATCHJOB_STATUS_FAILED); $job->setDescription("could not retrieve entry, probably deleted"); TRACE("could not retrieve entry {$entry_id} , probably deleted"); $job->save(); continue; } $fileSyncKey = $entry->getSyncKey(entry::FILE_SYNC_ENTRY_SUB_TYPE_DOWNLOAD, $file_format); $fullFinalPath = kFileSyncUtils::getLocalFilePathForKey($fileSyncKey); $finalPathNoExt = substr($fullFinalPath, 0, strlen($fullFinalPath) - strlen($file_format)); myContentStorage::fullMkdir($fullFinalPath); $wildcardFinalPath = $finalPathNoExt . "*"; $older_files = glob($wildcardFinalPath); foreach ($older_files as $older_file) { TRACE("removing old file: [{$older_file}]"); @unlink($older_file); } TRACE("Downloading: {$fullFinalPath}"); kFile::downloadUrlToFile($data["serverUrl"], $fullFinalPath); if (!file_exists($fullFinalPath)) { TRACE("file doesnt exist: " . $data["serverUrl"]); $job->setDescription("file doesnt exist: " . $data["serverUrl"]); $job->setStatus(BatchJob::BATCHJOB_STATUS_FAILED); } else { if (filesize($fullFinalPath) < 100000) { @unlink($fullFinalPath); TRACE("file too small: " . $data["serverUrl"]); $job->setDescription("file too small: " . $data["serverUrl"]); $job->setStatus(BatchJob::BATCHJOB_STATUS_FAILED); } else { if ($data['email']) { $downloadLink = $entry->getDownloadUrl() . '/format/' . $file_format; kJobsManager::addMailJob(null, $entry_id, $entry->getPartnerId(), self::KALTURAS_FLATTEN_READY, kMailJobData::MAIL_PRIORITY_NORMAL, kConf::get("batch_flatten_video_sender_email"), kConf::get("batch_flatten_video_sender_name"), $data['email'], array($data['email'], $downloadLink)); } TRACE("Deleting: " . $data["deleteUrl"]); kFile::downloadUrlToString($data["deleteUrl"]); myNotificationMgr::createNotification(kNotificationJobData::NOTIFICATION_TYPE_ENTRY_UPDATE, $entry); $job->setStatus(BatchJob::BATCHJOB_STATUS_FINISHED); $filePath = kFileSyncUtils::getLocalFilePathForKey($fileSyncKey); if (file_exists($filePath)) { try { kFileSyncUtils::createSyncFileForKey($fileSyncKey); } catch (Exception $ex) { TRACE("ignore ERROR: " . $ex->getMessage()); } } else { TRACE("The file [{$filePath}] doesn't exists, not creating FileSync"); } } } $job->save(); } } catch (Exception $ex) { TRACE("ERROR: " . $ex->getMessage()); self::initDb(true); self::failed(); } if ($temp_count == 0) { TRACE("Ended conversion. sleeping for a while (" . $sleep_between_cycles . " seconds). Will write to the log in (" . $sleep_between_cycles * $number_of_times_to_skip_writing_sleeping . ") seconds"); } $temp_count++; if ($temp_count >= $number_of_times_to_skip_writing_sleeping) { $temp_count = 0; } } }
private function calculateId() { $dc = kDataCenterMgr::getCurrentDc(); for ($i = 0; $i < 10; $i++) { $id = $dc["id"] . '_' . kString::generateStringId(); $existingObject = flavorAssetPeer::retrieveById($id); if ($existingObject) { KalturaLog::log(__METHOD__ . ": id [{$id}] already exists"); } else { return $id; } } throw new Exception("Could not find unique id for flavorAsset"); }
/** * This function returns the FILE SYSTEM path to the root content folder. * @return string the content folder file system path */ public static function getFSContentRootPath() { $dc = kDataCenterMgr::getCurrentDc(); return $dc["root"]; //return realpath(sfConfig::get('sf_root_dir')."/../../").'/'; }
public static function enforceCdnDelivery($partnerId) { $host = requestUtils::getHost(); $cdnHost = myPartnerUtils::getCdnHost($partnerId); $dc = kDataCenterMgr::getCurrentDc(); $external_url = $dc["external_url"]; // allow access only via cdn or via proxy from secondary datacenter if ($host != $cdnHost && $host != $external_url) { $uri = $_SERVER["REQUEST_URI"]; if (strpos($uri, "/forceproxy/true") === false) { $uri .= "/forceproxy/true/"; } header('Location:' . $cdnHost . $uri); header("X-Kaltura:enforce-cdn"); die; } }