/**
* @param string $remotePath
* @param string $localPath
* @param string $errDescription
* @return boolean
*/
private function fetchFile($remotePath, $localPath, &$errDescription)
{
KalturaLog::debug("Fetch url [{$remotePath}] to file [{$localPath}]");
try {
$folder = substr($localPath, 0, strrpos($localPath, '/'));
if (!file_exists($folder)) {
mkdir($folder, 777, true);
}
$curlWrapper = new KCurlWrapper($remotePath);
$curlHeaderResponse = $curlWrapper->getHeader(true);
if (!$curlHeaderResponse || $curlWrapper->getError()) {
$errDescription = "Error: " . $curlWrapper->getError();
return false;
}
if (!$curlHeaderResponse->isGoodCode()) {
$errDescription = "HTTP Error: " . $curlHeaderResponse->code . " " . $curlHeaderResponse->codeName;
return false;
}
$fileSize = null;
if (isset($curlHeaderResponse->headers['content-length'])) {
$fileSize = $curlHeaderResponse->headers['content-length'];
}
$curlWrapper->close();
KalturaLog::debug("Executing curl");
$curlWrapper = new KCurlWrapper($remotePath);
$res = $curlWrapper->exec($localPath);
KalturaLog::debug("Curl results: {$res}");
if (!$res || $curlWrapper->getError()) {
$errDescription = "Error: " . $curlWrapper->getError();
$curlWrapper->close();
return false;
}
$curlWrapper->close();
if (!file_exists($localPath)) {
$errDescription = "Error: output file doesn't exist";
return false;
}
if ($fileSize) {
clearstatcache();
if (filesize($localPath) != $fileSize) {
$errDescription = "Error: output file have a wrong size";
return false;
}
}
} catch (Exception $ex) {
$errDescription = "Error: " . $ex->getMessage();
return false;
}
return true;
}
/**
* @param string $srcFileSyncRemoteUrl
* @param string $srcFileSyncLocalPath
* @param string $errDescription
* @return string
*/
private function fetchFile($srcFileSyncRemoteUrl, $srcFileSyncLocalPath, &$errDescription = null)
{
KalturaLog::debug("fetchFile({$srcFileSyncRemoteUrl}, {$srcFileSyncLocalPath})");
try {
$curlWrapper = new KCurlWrapper($srcFileSyncRemoteUrl);
$curlHeaderResponse = $curlWrapper->getHeader(true);
if (!$curlHeaderResponse || $curlWrapper->getError()) {
$errDescription = "Error: " . $curlWrapper->getError();
return false;
}
if ($curlHeaderResponse->code != KCurlHeaderResponse::HTTP_STATUS_OK) {
$errDescription = "HTTP Error: " . $curlHeaderResponse->code . " " . $curlHeaderResponse->codeName;
return false;
}
$fileSize = null;
if (isset($curlHeaderResponse->headers['content-length'])) {
$fileSize = $curlHeaderResponse->headers['content-length'];
}
$curlWrapper->close();
KalturaLog::debug("Executing curl");
$curlWrapper = new KCurlWrapper($srcFileSyncRemoteUrl);
$res = $curlWrapper->exec($srcFileSyncLocalPath);
KalturaLog::debug("Curl results: {$res}");
if (!$res || $curlWrapper->getError()) {
$errDescription = "Error: " . $curlWrapper->getError();
$curlWrapper->close();
return false;
}
$curlWrapper->close();
if (!file_exists($srcFileSyncLocalPath)) {
$errDescription = "Error: output file doesn't exist";
return false;
}
if ($fileSize) {
clearstatcache();
if (filesize($srcFileSyncLocalPath) != $fileSize) {
$errDescription = "Error: output file have a wrong size";
return false;
}
}
} catch (Exception $ex) {
$errDescription = "Error: " . $ex->getMessage();
return false;
}
return true;
}
private function fetchFile(KalturaBatchJob $job, KalturaImportJobData $data)
{
KalturaLog::debug("fetchFile({$job->id})");
$jobSubType = $job->jobSubType;
if (in_array($jobSubType, array(kFileTransferMgrType::SCP, kFileTransferMgrType::SFTP))) {
// use SSH file transfer manager for SFTP/SCP
return $this->fetchFileSsh($job, $data);
}
try {
$sourceUrl = $data->srcFileUrl;
KalturaLog::debug("sourceUrl [{$sourceUrl}]");
$this->updateJob($job, 'Downloading file header', KalturaBatchJobStatus::QUEUED, 1);
$fileSize = null;
$resumeOffset = 0;
if ($data->destFileLocalPath && file_exists($data->destFileLocalPath)) {
$curlWrapper = new KCurlWrapper($sourceUrl, $this->taskConfig->params->curlVerbose);
$useNoBody = $job->executionAttempts > 1;
// if the process crashed first time, tries with no body instead of range 0-0
$curlHeaderResponse = $curlWrapper->getHeader($useNoBody);
if (!$curlHeaderResponse || !count($curlHeaderResponse->headers)) {
$this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $curlWrapper->getErrorNumber(), "Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::FAILED);
return $job;
}
if ($curlWrapper->getError()) {
KalturaLog::err("Headers error: " . $curlWrapper->getError());
KalturaLog::err("Headers error number: " . $curlWrapper->getErrorNumber());
$curlWrapper->close();
$curlWrapper = new KCurlWrapper($sourceUrl, $this->taskConfig->params->curlVerbose);
}
if (!$curlHeaderResponse->isGoodCode()) {
$this->closeJob($job, KalturaBatchJobErrorTypes::HTTP, $curlHeaderResponse->code, "HTTP Error: " . $curlHeaderResponse->code . " " . $curlHeaderResponse->codeName, KalturaBatchJobStatus::FAILED);
return $job;
}
if (isset($curlHeaderResponse->headers['content-length'])) {
$fileSize = $curlHeaderResponse->headers['content-length'];
}
$curlWrapper->close();
if ($fileSize) {
clearstatcache();
$actualFileSize = kFile::fileSize($data->destFileLocalPath);
if ($actualFileSize >= $fileSize) {
return $this->moveFile($job, $data->destFileLocalPath, $fileSize);
} else {
$resumeOffset = $actualFileSize;
}
}
}
$curlWrapper = new KCurlWrapper($sourceUrl, $this->taskConfig->params->curlVerbose);
$curlWrapper->setTimeout($this->taskConfig->params->curlTimeout);
if ($resumeOffset) {
$curlWrapper->setResumeOffset($resumeOffset);
} else {
// creates a temp file path
$destFile = $this->getTempFilePath($sourceUrl);
KalturaLog::debug("destFile [{$destFile}]");
$data->destFileLocalPath = $destFile;
$this->updateJob($job, "Downloading file, size: {$fileSize}", KalturaBatchJobStatus::PROCESSING, 2, $data);
}
KalturaLog::debug("Executing curl");
$res = $curlWrapper->exec($data->destFileLocalPath);
KalturaLog::debug("Curl results: {$res}");
if (!$res || $curlWrapper->getError()) {
$errNumber = $curlWrapper->getErrorNumber();
if ($errNumber != CURLE_OPERATION_TIMEOUTED) {
$this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $errNumber, "Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::RETRY);
$curlWrapper->close();
return $job;
} else {
clearstatcache();
$actualFileSize = kFile::fileSize($data->destFileLocalPath);
if ($actualFileSize == $resumeOffset) {
$this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $errNumber, "Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::RETRY);
$curlWrapper->close();
return $job;
}
}
}
$curlWrapper->close();
if (!file_exists($data->destFileLocalPath)) {
$this->closeJob($job, KalturaBatchJobErrorTypes::APP, KalturaBatchJobAppErrors::OUTPUT_FILE_DOESNT_EXIST, "Error: output file doesn't exist", KalturaBatchJobStatus::RETRY);
return $job;
}
// check the file size only if its first or second retry
// in case it failed few times, taks the file as is
if ($fileSize) {
clearstatcache();
$actualFileSize = kFile::fileSize($data->destFileLocalPath);
if ($actualFileSize < $fileSize) {
$percent = floor($actualFileSize * 100 / $fileSize);
$this->updateJob($job, "Downloaded size: {$actualFileSize}({$percent}%)", KalturaBatchJobStatus::PROCESSING, $percent, $data);
$this->kClient->batch->resetJobExecutionAttempts($job->id, $this->getExclusiveLockKey(), $job->jobType);
// $this->closeJob($job, KalturaBatchJobErrorTypes::APP, KalturaBatchJobAppErrors::OUTPUT_FILE_WRONG_SIZE, "Expected file size[$fileSize] actual file size[$actualFileSize]", KalturaBatchJobStatus::RETRY);
return $job;
}
}
$this->updateJob($job, 'File imported, copy to shared folder', KalturaBatchJobStatus::PROCESSED, 90);
$job = $this->moveFile($job, $data->destFileLocalPath, $fileSize);
} catch (Exception $ex) {
$this->closeJob($job, KalturaBatchJobErrorTypes::RUNTIME, $ex->getCode(), "Error: " . $ex->getMessage(), KalturaBatchJobStatus::FAILED);
}
return $job;
}
/**
* Fetches the header for the given $url and closes the job on any errors
* @param KalturaBatchJob $job
* @param string $url
* @return false|KCurlHeaderResponse
*/
private function fetchHeader(KalturaBatchJob &$job, $url)
{
KalturaLog::debug('Fetching header for [' . $url . ']');
$this->updateJob($job, 'Downloading header for [' . $url . ']', KalturaBatchJobStatus::PROCESSING);
// fetch the http headers
$curlWrapper = new KCurlWrapper($url);
$curlHeaderResponse = $curlWrapper->getHeader();
$curlError = $curlWrapper->getError();
$curlWrapper->close();
if (!$curlHeaderResponse || $curlError) {
// error fetching headers
$msg = 'Error: ' . $curlWrapper->getError();
KalturaLog::err($msg);
$this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $curlWrapper->getErrorNumber(), $msg, KalturaBatchJobStatus::FAILED);
return false;
}
if (!$curlHeaderResponse->isGoodCode()) {
// some error exists in the response
$msg = 'HTTP Error: ' . $curlHeaderResponse->code . ' ' . $curlHeaderResponse->codeName;
KalturaLog::err($msg);
$this->closeJob($job, KalturaBatchJobErrorTypes::HTTP, $curlHeaderResponse->code, $msg, KalturaBatchJobStatus::FAILED);
return false;
}
// header fetched successfully - return it
return $curlHeaderResponse;
}
/**
* @param string $remotePath
* @param string $localPath
* @param string $errDescription
* @return boolean
*/
private function fetchFile($remotePath, $localPath, &$errDescription)
{
KalturaLog::debug("Fetch url [{$remotePath}] to file [{$localPath}]");
try {
$folder = substr($localPath, 0, strrpos($localPath, '/'));
if (!file_exists($folder)) {
mkdir($folder, 777, true);
}
$curlWrapper = new KCurlWrapper();
$curlHeaderResponse = $curlWrapper->getHeader($remotePath, true);
$removeServer = isset($curlHeaderResponse->headers['X-Me']) ? $curlHeaderResponse->headers['X-Me'] : "unknown";
if (!$curlHeaderResponse || $curlWrapper->getError()) {
$errDescription = "Error: ({$removeServer}) " . $curlWrapper->getError();
return false;
}
if (!$curlHeaderResponse->isGoodCode()) {
$errDescription = "HTTP Error: ({$removeServer}) " . $curlHeaderResponse->code . " " . $curlHeaderResponse->codeName;
return false;
}
$fileSize = null;
if (isset($curlHeaderResponse->headers['content-length'])) {
$fileSize = $curlHeaderResponse->headers['content-length'];
}
$curlWrapper->close();
KalturaLog::debug("Executing curl");
// overcome a 32bit issue with curl fetching >=4gb files
if (intval("9223372036854775807") == 2147483647 && $fileSize >= 4 * 1024 * 1024 * 1024) {
unlink($localPath);
$cmd = "curl -s {$remotePath} -o {$localPath}";
KalturaLog::debug($cmd);
exec($cmd);
} else {
$curlWrapper = new KCurlWrapper();
$res = $curlWrapper->exec($remotePath, $localPath);
KalturaLog::debug("Curl results: {$res}");
if (!$res || $curlWrapper->getError()) {
$errDescription = "Error: ({$removeServer}) " . $curlWrapper->getError();
$curlWrapper->close();
return false;
}
$curlWrapper->close();
}
if (!file_exists($localPath)) {
$errDescription = "Error: output file doesn't exist";
return false;
}
if ($fileSize) {
clearstatcache();
if (kFile::fileSize($localPath) != $fileSize) {
$errDescription = "Error: output file have a wrong size";
return false;
}
}
} catch (Exception $ex) {
$errDescription = "Error: " . $ex->getMessage();
return false;
}
return true;
}
/**
* @param string $srcFileSyncRemoteUrl
* @param string $srcFileSyncLocalPath
* @param string $errDescription
* @return string
*/
private function fetchFile($srcFileSyncRemoteUrl, $srcFileSyncLocalPath, &$errDescription = null)
{
KalturaLog::debug("fetchFile({$srcFileSyncRemoteUrl}, {$srcFileSyncLocalPath})");
try {
$curlWrapper = new KCurlWrapper($srcFileSyncRemoteUrl);
$curlHeaderResponse = $curlWrapper->getHeader(true);
if (!$curlHeaderResponse || $curlWrapper->getError()) {
$errDescription = "Error: " . $curlWrapper->getError();
return false;
}
if ($curlHeaderResponse->code != KCurlHeaderResponse::HTTP_STATUS_OK) {
$errDescription = "HTTP Error: " . $curlHeaderResponse->code . " " . $curlHeaderResponse->codeName;
return false;
}
$fileSize = null;
KalturaLog::debug("Headers:\n" . print_r($curlHeaderResponse->headers, true));
if (isset($curlHeaderResponse->headers['content-length'])) {
$fileSize = $curlHeaderResponse->headers['content-length'];
}
$curlWrapper->close();
if ($fileSize && file_exists($srcFileSyncLocalPath)) {
clearstatcache();
$actualFileSize = filesize($srcFileSyncLocalPath);
if ($actualFileSize == $fileSize) {
KalturaLog::log("File [{$srcFileSyncLocalPath}] already exists with right size[{$fileSize}]");
return true;
}
KalturaLog::log("File [{$srcFileSyncLocalPath}] already exists with wrong size[{$actualFileSize}] expected size[{$fileSize}]");
KalturaLog::debug("Unlink file[{$srcFileSyncLocalPath}]");
unlink($srcFileSyncLocalPath);
}
KalturaLog::debug("Executing curl");
$curlWrapper = new KCurlWrapper($srcFileSyncRemoteUrl);
$res = $curlWrapper->exec($srcFileSyncLocalPath);
KalturaLog::debug("Curl results: {$res}");
if (!$res || $curlWrapper->getError()) {
$errDescription = "Error: " . $curlWrapper->getError();
$curlWrapper->close();
return false;
}
$curlWrapper->close();
if (!file_exists($srcFileSyncLocalPath)) {
$errDescription = "Error: output file[{$srcFileSyncLocalPath}] doesn't exist";
return false;
}
clearstatcache();
$actualFileSize = filesize($srcFileSyncLocalPath);
KalturaLog::debug("Fetched file to [{$srcFileSyncLocalPath}] size[{$actualFileSize}]");
if ($fileSize) {
if ($actualFileSize != $fileSize) {
$errDescription = "Error: output file[{$srcFileSyncLocalPath}] have a wrong size[{$actualFileSize}] expected size[{$fileSize}]";
return false;
}
}
} catch (Exception $ex) {
$errDescription = "Error: " . $ex->getMessage();
return false;
}
return true;
}
private function fetchFile(KalturaBatchJob $job, KalturaImportJobData $data)
{
KalturaLog::debug("fetchFile({$job->id})");
try {
$sourceUrl = $data->srcFileUrl;
KalturaLog::debug("sourceUrl [{$sourceUrl}]");
$this->updateJob($job, 'Downloading file header', KalturaBatchJobStatus::QUEUED, 1);
$curlWrapper = new KCurlWrapper($sourceUrl);
$useNoBody = $job->executionAttempts > 1;
// if the process crashed first time, tries with no body instead of range 0-0
$curlHeaderResponse = $curlWrapper->getHeader($useNoBody);
if (!$curlHeaderResponse || $curlWrapper->getError()) {
$this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $curlWrapper->getErrorNumber(), "Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::FAILED);
return $job;
}
if (!$curlHeaderResponse->isGoodCode()) {
$this->closeJob($job, KalturaBatchJobErrorTypes::HTTP, $curlHeaderResponse->code, "HTTP Error: " . $curlHeaderResponse->code . " " . $curlHeaderResponse->codeName, KalturaBatchJobStatus::FAILED);
return $job;
}
$fileSize = null;
if (isset($curlHeaderResponse->headers['content-length'])) {
$fileSize = $curlHeaderResponse->headers['content-length'];
}
$curlWrapper->close();
$resumeOffset = 0;
if ($fileSize && $data->destFileLocalPath && file_exists($data->destFileLocalPath)) {
clearstatcache();
$actualFileSize = filesize($data->destFileLocalPath);
if ($actualFileSize >= $fileSize) {
return $this->moveFile($job, $data->destFileLocalPath, $fileSize);
} else {
$resumeOffset = $actualFileSize;
}
}
$curlWrapper = new KCurlWrapper($sourceUrl);
$curlWrapper->setTimeout($this->taskConfig->params->curlTimeout);
if ($resumeOffset) {
$curlWrapper->setResumeOffset($resumeOffset);
} else {
// creates a temp file path
$rootPath = $this->taskConfig->params->localTempPath;
$res = self::createDir($rootPath);
if (!$res) {
KalturaLog::err("Cannot continue import without temp directory");
die;
}
$uniqid = uniqid('import_');
$destFile = realpath($rootPath) . "/{$uniqid}";
KalturaLog::debug("destFile [{$destFile}]");
// in case the url has added arguments, remove them (and reveal the URL path)
// in order to find the file extension
$urlPathEndIndex = strpos($sourceUrl, "?");
if ($urlPathEndIndex !== false) {
$sourceUrlPath = substr($sourceUrl, 0, $urlPathEndIndex);
} else {
$sourceUrlPath = $sourceUrl;
}
$ext = pathinfo($sourceUrlPath, PATHINFO_EXTENSION);
if (strlen($ext)) {
$destFile .= ".{$ext}";
}
$data->destFileLocalPath = $destFile;
$this->updateJob($job, "Downloading file, size: {$fileSize}", KalturaBatchJobStatus::PROCESSING, 2, $data);
}
KalturaLog::debug("Executing curl");
$res = $curlWrapper->exec($data->destFileLocalPath);
KalturaLog::debug("Curl results: {$res}");
if (!$res || $curlWrapper->getError()) {
$errNumber = $curlWrapper->getErrorNumber();
if ($errNumber != CURLE_OPERATION_TIMEOUTED) {
$this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $errNumber, "Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::RETRY);
$curlWrapper->close();
return $job;
} else {
clearstatcache();
$actualFileSize = filesize($data->destFileLocalPath);
if ($actualFileSize == $resumeOffset) {
$this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $errNumber, "Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::RETRY);
$curlWrapper->close();
return $job;
}
}
}
$curlWrapper->close();
if (!file_exists($data->destFileLocalPath)) {
$this->closeJob($job, KalturaBatchJobErrorTypes::APP, KalturaBatchJobAppErrors::OUTPUT_FILE_DOESNT_EXIST, "Error: output file doesn't exist", KalturaBatchJobStatus::RETRY);
return $job;
}
// check the file size only if its first or second retry
// in case it failed few times, taks the file as is
if ($fileSize) {
clearstatcache();
$actualFileSize = filesize($data->destFileLocalPath);
if ($actualFileSize < $fileSize) {
$percent = floor($actualFileSize * 100 / $fileSize);
$this->updateJob($job, "Downloaded size: {$actualFileSize}({$percent}%)", KalturaBatchJobStatus::PROCESSING, $percent, $data);
$this->kClient->batch->resetJobExecutionAttempts($job->id, $this->getExclusiveLockKey(), $job->jobType);
// $this->closeJob($job, KalturaBatchJobErrorTypes::APP, KalturaBatchJobAppErrors::OUTPUT_FILE_WRONG_SIZE, "Expected file size[$fileSize] actual file size[$actualFileSize]", KalturaBatchJobStatus::RETRY);
return $job;
}
}
$this->updateJob($job, 'File imported, copy to shared folder', KalturaBatchJobStatus::PROCESSED, 90);
$job = $this->moveFile($job, $data->destFileLocalPath, $fileSize);
} catch (Exception $ex) {
$this->closeJob($job, KalturaBatchJobErrorTypes::RUNTIME, $ex->getCode(), "Error: " . $ex->getMessage(), KalturaBatchJobStatus::FAILED);
}
return $job;
}
private static function retrieveCaptionFile($captionObj, $captionUrl, $destFolder)
{
KalturaLog::debug("Caption object:\n" . print_r($captionObj, 1));
KalturaLog::debug("Executing curl to retrieve caption asset file from - {$captionUrl}");
$curlWrapper = new KCurlWrapper();
$cptFilePath = $destFolder . ".temp." . $captionObj->languageCode . ".srt";
$res = $curlWrapper->exec($captionUrl, $cptFilePath);
KalturaLog::debug("Curl results: {$res}");
if (!$res || $curlWrapper->getError()) {
$errDescription = "Error: " . $curlWrapper->getError();
$curlWrapper->close();
KalturaLog::err("Failed to curl the caption file url({$captionUrl}). Error ({$errDescription})");
return null;
}
$curlWrapper->close();
if (!file_exists($cptFilePath)) {
KalturaLog::err("Error: output file ({$cptFilePath}) doesn't exist");
return null;
}
KalturaLog::debug("Finished");
return $cptFilePath;
}
/**
* This Function can work in two ways if destFilePath is provided it will copy the url to the dest file else will return the file as string as an output
* @param string $url - URL to get data from
* @param $destFilePath - Optional URL to copy data to
* @param int $maxFileSize - Optional max file size allowed for the retrieval action
* @throws Exception | file as string | if $destFilePath provide - true or false
*/
public static function getDataFromFile($url, $destFilePath = null, $maxFileSize = null)
{
if (!is_null($maxFileSize)) {
$curlWrapper = new KCurlWrapper();
$curlHeaderResponse = $curlWrapper->getHeader($url, true);
$curlWrapper->close();
if (!$curlHeaderResponse || $curlWrapper->getError()) {
throw new Exception("Failed to retrive Curl header response from file path [{$url}] with Error " . $curlWrapper->getError());
}
if (!$curlHeaderResponse->isGoodCode()) {
throw new Exception("Non Valid Error: {$curlHeaderResponse->code}" . " " . $curlHeaderResponse->codeName);
}
if (isset($curlHeaderResponse->headers['content-length'])) {
$fileSize = $curlHeaderResponse->headers['content-length'];
if ($fileSize > $maxFileSize) {
throw new Exception("File size [{$fileSize}] Excedded Max Siae Allowed [{$maxFileSize}]");
}
KalturaLog::debug("File size [{$fileSize}] validated");
} else {
KalturaLog::debug("File size validation skipped");
}
}
$curlWrapper = new KCurlWrapper();
$res = $curlWrapper->exec($url, $destFilePath);
$curlWrapper->close();
return $res;
}
/**
*
* @param $wmData
* @param string $destFileSyncLocalPath
* @param string $cmdLine
* @param string $ffmpegBin
* @param string $mediaInfoBin
* @return string
*/
public static function buildWatermarkedCommandLine($wmData, $destFileSyncLocalPath, $cmdLine, $ffmpegBin = "ffmpeg", $mediaInfoBin = "mediainfo")
{
KalturaLog::log("After:cmdline({$cmdLine})");
if (!isset($mediaInfoBin) || strlen($mediaInfoBin) == 0) {
$mediaInfoBin = "mediainfo";
}
/*
* evaluate WM scale and margins, if any
*/
KalturaLog::log("Watermark data({$mediaInfoBin}):\n" . print_r($wmData, 1));
if (isset($wmData->scale) && is_string($wmData->scale)) {
$wmData->scale = explode("x", $wmData->scale);
}
if (isset($wmData->margins) && is_string($wmData->margins)) {
$wmData->margins = explode("x", $wmData->margins);
}
KalturaLog::log("Watermark data:\n" . print_r($wmData, 1));
/*
* Retrieve watermark image file,
* either from image entry or from external url
* If both set, prefer image entry
*/
$imageDownloadUrl = null;
$errStr = null;
if (isset($wmData->imageEntry)) {
$version = null;
try {
$imgEntry = KBatchBase::$kClient->baseEntry->get($wmData->imageEntry, $version);
} catch (Exception $ex) {
$imgEntry = null;
$errStr = "Exception on retrieval of an image entry({$wmData->imageEntry}),\nexception:" . print_r($ex, 1);
}
if (isset($imgEntry)) {
KalturaLog::log("Watermark entry: {$wmData->imageEntry}");
$imageDownloadUrl = $imgEntry->downloadUrl;
} else {
if (!isset($errStr)) {
$errStr = "Failed to retrieve an image entry({$wmData->imageEntry})";
}
}
if (!isset($imgEntry)) {
KalturaLog::err($errStr);
}
}
if (!isset($imageDownloadUrl)) {
if (isset($wmData->url)) {
$imageDownloadUrl = $wmData->url;
} else {
KalturaLog::err("Missing watermark image data, neither via image-entry-id nor via external url.");
return null;
}
}
$wmTmpFilepath = $destFileSyncLocalPath . ".wmtmp";
KalturaLog::log("imageDownloadUrl({$imageDownloadUrl}), wmTmpFilepath({$wmTmpFilepath})");
/*
* Get the watermark image file
*/
$curlWrapper = new KCurlWrapper();
$res = $curlWrapper->exec($imageDownloadUrl, $wmTmpFilepath);
KalturaLog::debug("Curl results: {$res}");
if (!$res || $curlWrapper->getError()) {
$errDescription = "Error: " . $curlWrapper->getError();
$curlWrapper->close();
KalturaLog::err("Failed to curl the caption file url({$imageDownloadUrl}). Error ({$errDescription})");
return null;
}
$curlWrapper->close();
if (!file_exists($wmTmpFilepath)) {
KalturaLog::err("Error: output file ({$wmTmpFilepath}) doesn't exist");
return null;
}
KalturaLog::log("Successfully retrieved the watermark image file ({$wmTmpFilepath}) ");
/*
* Query the image file for format and dims
*/
$output = array();
exec("{$mediaInfoBin} {$wmTmpFilepath} | grep -i \"width\\|height\\|format\"", $ouput, $rv);
if ($rv != 0) {
KalturaLog::err("Failed to retrieve media data from watermark file ({$wmTmpFilepath}). rv({$rv}). Carry on without watermark.");
return null;
}
foreach ($ouput as $line) {
if (!isset($wmData->format) && stristr($line, "format") != false) {
$str = stristr($line, ":");
$wmData->format = strtolower(trim($str, ": "));
} else {
if (stristr($line, "width") != false) {
$str = stristr($line, ":");
$wmData->width = (int) trim($str, ": ");
} else {
if (stristr($line, "height") != false) {
$str = stristr($line, ":");
$wmData->height = (int) trim($str, ": ");
}
}
}
}
switch ($wmData->format) {
case "jpeg":
$wmData->format = "jpg";
case "jpg":
case "png":
default:
rename($wmTmpFilepath, "{$wmTmpFilepath}.{$wmData->format}");
$wmTmpFilepath = "{$wmTmpFilepath}.{$wmData->format}";
break;
}
KalturaLog::log("Updated Watermark data:\n" . print_r($wmData, 1));
/*
* Evaluate WM scaling params and scale it accordingly
*/
$wid = null;
$hgt = null;
if (!isset($wmData->scale) && ($wmData->width % 2 != 0 || $wmData->height % 2 != 0)) {
$wmData->scale = array();
$wmData->width -= $wmData->width % 2;
$wmData->height -= $wmData->height % 2;
$wmData->scale[0] = $wmData->width;
$wmData->scale[1] = $wmData->height;
}
if (isset($wmData->scale)) {
$wid = in_array($wmData->scale[0], array(null, 0, -1)) ? -1 : $wmData->scale[0];
$hgt = in_array($wmData->scale[1], array(null, 0, -1)) ? -1 : $wmData->scale[1];
if ($wid <= 0) {
$wid = round($hgt * $wmData->width / $wmData->height);
}
if ($hgt <= 0) {
$hgt = round($wid * $wmData->height / $wmData->width);
}
if (!($wid > 0 && $hgt > 0)) {
$wid = $wmData->width;
$hgt = $wmData->height;
}
if ($wid > 0) {
$wid -= $wid % 2;
}
if ($hgt > 0) {
$hgt -= $hgt % 2;
}
$wmFilePath = $wmTmpFilepath;
} else {
$wmFilePath = $wmTmpFilepath;
$wid = $wmData->width;
$hgt = $wmData->height;
}
/* Samples -
"[1]scale=100:100,setsar=100/100[logo];[0:v]crop=100:100:iw-ow-10:300,setsar=100/100[cropped];[cropped][logo]blend=all_expr='if(eq(mod(X,2),mod(Y,2)),A,B)'[blended];[0:v][blended]overlay=main_w-overlay_w-10:300[out]"
"[1]scale=100:100,setsar=100/100[logo];[0:v][logo]overlay=main_w-overlay_w-10:300[out]" -map "[out]"
*/
$cmdLine = str_replace(array(KDLCmdlinePlaceholders::WaterMarkFileName, KDLCmdlinePlaceholders::WaterMarkWidth, KDLCmdlinePlaceholders::WaterMarkHeight), array($wmFilePath, $wid, $hgt), $cmdLine);
KalturaLog::log("After:cmdline({$cmdLine})");
return $cmdLine;
}
public static function handleImportContent($curlInfo, $importData, $params)
{
if (!($curlInfo->headers['content-length'] < 16000 && $curlInfo->headers['content-type'] == 'text/html')) {
return $importData;
}
KalturaLog::debug('content-length [' . $curlInfo->headers['content-length'] . '] content-type [' . $curlInfo->headers['content-type'] . ']');
KalturaLog::info('Handle Import data: Webex Plugin');
$matches = null;
$recordId = null;
if (isset($curlInfo->headers['set-cookie'])) {
$recordId = $curlInfo->getCookieValue($curlInfo->headers['set-cookie'], 'recordId');
if ($recordId == null) {
throw new Exception('recordId value not found');
}
} else {
throw new Exception('set-cookie was not found in header');
}
$data = file_get_contents($importData->destFileLocalPath);
KalturaLog::info("data:\n\n{$data}\n\n");
if (!preg_match("/href='([^']+)';/", $data, $matches)) {
throw new Exception('Starting URL not found');
}
$url2 = $matches[1];
$curlWrapper = new KCurlWrapper();
$curlWrapper->setOpt(CURLOPT_COOKIE, 'DetectionBrowserStatus=3|1|32|1|11|2;' . $curlInfo->headers["set-cookie"]);
$result = $curlWrapper->exec($url2);
KalturaLog::info("result:\n\n{$result}\n\n");
if (!preg_match("/var prepareTicket = '([^']+)';/", $result, $matches)) {
throw new Exception('prepareTicket parameter not found');
}
$prepareTicket = $matches[1];
if (!preg_match('/function (download\\(\\).+prepareTicket;)/s', $result, $matches)) {
throw new Exception('download function not found');
}
if (!preg_match('/http.+prepareTicket/', $matches[0], $matches)) {
throw new Exception('prepareTicket URL not found');
}
$url3 = $matches[0];
$url3 = str_replace(array('"', ' ', '+', 'recordId', 'prepareTicket=prepareTicket'), array('', '', '', $recordId, "prepareTicket={$prepareTicket}"), $url3);
if (!preg_match("/var downloadUrl = '(http[^']+)' \\+ ticket;/", $result, $matches)) {
throw new Exception('Download URL not found');
}
$url4 = $matches[1];
$status = null;
$iterations = isset($params->webex->iterations) && !is_null($params->webex->iterations) ? intval($params->webex->iterations) : 10;
$sleep = isset($params->webex->sleep) && !is_null($params->webex->sleep) ? intval($params->webex->sleep) : 3;
for ($i = 0; $i < $iterations; $i++) {
$result = $curlWrapper->exec($url3);
KalturaLog::info("result ({$i}):\n\n{$result}\n\n");
if (!preg_match("/window\\.parent\\.func_prepare\\('([^']+)','([^']*)','([^']*)'\\);/", $result, $matches)) {
KalturaLog::err("Invalid result returned for prepareTicket request - should contain call to the func_prepare method\n {$result}");
throw new Exception('Invalid result: func_prepare function not found');
}
$status = $matches[1];
if ($status == 'OKOK') {
break;
}
sleep($sleep);
}
if ($status != 'OKOK') {
KalturaLog::info("Invalid result returned for prepareTicket request. Last result:\n " . $result);
throw new kTemporaryException('Invalid result returned for prepareTicket request');
}
$ticket = $matches[3];
$url4 .= $ticket;
$curlWrapper->setOpt(CURLOPT_RETURNTRANSFER, false);
$fileName = pathinfo($importData->destFileLocalPath, PATHINFO_FILENAME);
$destFileLocalPath = preg_replace("/{$fileName}\\.[\\w\\d]+/", "{$fileName}.arf", $importData->destFileLocalPath);
$importData->destFileLocalPath = $destFileLocalPath;
KalturaLog::info('destination: ' . $importData->destFileLocalPath);
$result = $curlWrapper->exec($url4, $importData->destFileLocalPath);
if (!$result) {
$code = $curlWrapper->getErrorNumber();
$message = $curlWrapper->getError();
throw new Exception($message, $code);
}
$curlWrapper->close();
$importData->fileSize = kFile::fileSize($importData->destFileLocalPath);
return $importData;
}
private function fetchFile(KalturaBatchJob $job, KalturaImportJobData $data)
{
$jobSubType = $job->jobSubType;
$sshProtocols = array(kFileTransferMgrType::SCP, kFileTransferMgrType::SFTP);
if (in_array($jobSubType, $sshProtocols)) {
// use SSH file transfer manager for SFTP/SCP
return $this->fetchFileSsh($job, $data);
}
try {
$sourceUrl = $data->srcFileUrl;
$this->updateJob($job, 'Downloading file header', KalturaBatchJobStatus::QUEUED);
$fileSize = null;
$resumeOffset = 0;
if ($data->destFileLocalPath && file_exists($data->destFileLocalPath)) {
$curlWrapper = new KCurlWrapper(self::$taskConfig->params);
$useNoBody = $job->executionAttempts > 1;
// if the process crashed first time, tries with no body instead of range 0-0
$curlHeaderResponse = $curlWrapper->getHeader($sourceUrl, $useNoBody);
if (!$curlHeaderResponse || !count($curlHeaderResponse->headers)) {
$this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $curlWrapper->getErrorNumber(), "Couldn't read file. Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::FAILED);
return $job;
}
if ($curlWrapper->getError()) {
KalturaLog::err("Headers error: " . $curlWrapper->getError());
KalturaLog::err("Headers error number: " . $curlWrapper->getErrorNumber());
$curlWrapper->close();
$curlWrapper = new KCurlWrapper(self::$taskConfig->params);
}
if (!$curlHeaderResponse->isGoodCode()) {
$this->closeJob($job, KalturaBatchJobErrorTypes::HTTP, $curlHeaderResponse->code, "Failed while reading file. HTTP Error: " . $curlHeaderResponse->code . " " . $curlHeaderResponse->codeName, KalturaBatchJobStatus::FAILED);
$curlWrapper->close();
return $job;
}
if (isset($curlHeaderResponse->headers['content-length'])) {
$fileSize = $curlHeaderResponse->headers['content-length'];
}
$curlWrapper->close();
if ($fileSize) {
clearstatcache();
$actualFileSize = kFile::fileSize($data->destFileLocalPath);
if ($actualFileSize >= $fileSize) {
return $this->moveFile($job, $data->destFileLocalPath, $fileSize);
} else {
$resumeOffset = $actualFileSize;
}
}
}
$curlWrapper = new KCurlWrapper(self::$taskConfig->params);
if (is_null($fileSize)) {
// Read file size
$curlHeaderResponse = $curlWrapper->getHeader($sourceUrl, true);
if ($curlHeaderResponse && count($curlHeaderResponse->headers) && !$curlWrapper->getError() && isset($curlHeaderResponse->headers['content-length'])) {
$fileSize = $curlHeaderResponse->headers['content-length'];
}
//Close the curl used to fetch the header and create a new one.
//When fetching headers we set curl options that than are not reset once header is fetched.
//Not all servers support all the options so we need to remove them from our headers.
$curlWrapper->close();
$curlWrapper = new KCurlWrapper(self::$taskConfig->params);
}
if ($resumeOffset) {
$curlWrapper->setResumeOffset($resumeOffset);
} else {
// creates a temp file path
$destFile = $this->getTempFilePath($sourceUrl);
KalturaLog::debug("destFile [{$destFile}]");
$data->destFileLocalPath = $destFile;
$data->fileSize = is_null($fileSize) ? -1 : $fileSize;
$this->updateJob($job, "Downloading file, size: {$fileSize}", KalturaBatchJobStatus::PROCESSING, $data);
}
$res = $curlWrapper->exec($sourceUrl, $data->destFileLocalPath);
KalturaLog::debug("Curl results: {$res}");
if (!$res || $curlWrapper->getError()) {
$errNumber = $curlWrapper->getErrorNumber();
if ($errNumber != CURLE_OPERATION_TIMEOUTED) {
$this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $errNumber, "Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::RETRY);
$curlWrapper->close();
return $job;
} else {
clearstatcache();
$actualFileSize = kFile::fileSize($data->destFileLocalPath);
if ($actualFileSize == $resumeOffset) {
$this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $errNumber, "No new information. Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::RETRY);
$curlWrapper->close();
return $job;
}
if (!$fileSize) {
$this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $errNumber, "Received timeout, but no filesize available. Completed size [{$actualFileSize}]" . $curlWrapper->getError(), KalturaBatchJobStatus::RETRY);
$curlWrapper->close();
return $job;
}
}
}
$curlWrapper->close();
if (!file_exists($data->destFileLocalPath)) {
$this->closeJob($job, KalturaBatchJobErrorTypes::APP, KalturaBatchJobAppErrors::OUTPUT_FILE_DOESNT_EXIST, "Error: output file doesn't exist", KalturaBatchJobStatus::RETRY);
return $job;
}
// check the file size only if its first or second retry
// in case it failed few times, taks the file as is
if ($fileSize) {
clearstatcache();
$actualFileSize = kFile::fileSize($data->destFileLocalPath);
if ($actualFileSize < $fileSize) {
$percent = floor($actualFileSize * 100 / $fileSize);
$this->updateJob($job, "Downloaded size: {$actualFileSize}({$percent}%)", KalturaBatchJobStatus::PROCESSING, $data);
self::$kClient->batch->resetJobExecutionAttempts($job->id, $this->getExclusiveLockKey(), $job->jobType);
// $this->closeJob($job, KalturaBatchJobErrorTypes::APP, KalturaBatchJobAppErrors::OUTPUT_FILE_WRONG_SIZE, "Expected file size[$fileSize] actual file size[$actualFileSize]", KalturaBatchJobStatus::RETRY);
return $job;
}
KalturaLog::info("headers " . print_r($curlHeaderResponse, true));
$pluginInstances = KalturaPluginManager::getPluginInstances('IKalturaImportHandler');
foreach ($pluginInstances as $pluginInstance) {
/* @var $pluginInstance IKalturaImportHandler */
$data = $pluginInstance->handleImportContent($curlHeaderResponse, $data, KBatchBase::$taskConfig->params);
}
}
$this->updateJob($job, 'File imported, copy to shared folder', KalturaBatchJobStatus::PROCESSED);
$job = $this->moveFile($job, $data->destFileLocalPath);
} catch (kTemporaryException $tex) {
$data->destFileLocalPath = KalturaClient::getKalturaNullValue();
$tex->setData($data);
throw $tex;
} catch (Exception $ex) {
$data->destFileLocalPath = KalturaClient::getKalturaNullValue();
if ($ex->getMessage() == KCurlWrapper::COULD_NOT_CONNECT_TO_HOST_ERROR) {
throw new kTemporaryException($ex->getMessage(), $ex->getCode(), $data);
}
$this->closeJob($job, KalturaBatchJobErrorTypes::RUNTIME, $ex->getCode(), "Error: " . $ex->getMessage(), KalturaBatchJobStatus::FAILED, $data);
}
return $job;
}
