/** * @param string $remotePath * @param string $localPath * @param string $errDescription * @return boolean */ private function fetchFile($remotePath, $localPath, &$errDescription) { KalturaLog::debug("Fetch url [{$remotePath}] to file [{$localPath}]"); try { $folder = substr($localPath, 0, strrpos($localPath, '/')); if (!file_exists($folder)) { mkdir($folder, 777, true); } $curlWrapper = new KCurlWrapper($remotePath); $curlHeaderResponse = $curlWrapper->getHeader(true); if (!$curlHeaderResponse || $curlWrapper->getError()) { $errDescription = "Error: " . $curlWrapper->getError(); return false; } if (!$curlHeaderResponse->isGoodCode()) { $errDescription = "HTTP Error: " . $curlHeaderResponse->code . " " . $curlHeaderResponse->codeName; return false; } $fileSize = null; if (isset($curlHeaderResponse->headers['content-length'])) { $fileSize = $curlHeaderResponse->headers['content-length']; } $curlWrapper->close(); KalturaLog::debug("Executing curl"); // overcome a 32bit issue with curl fetching >=4gb files if (intval("9223372036854775807") == 2147483647 && $fileSize >= 4 * 1024 * 1024 * 1024) { unlink($localPath); $cmd = "curl -s {$remotePath} -o {$localPath}"; KalturaLog::debug($cmd); exec($cmd); } else { $curlWrapper = new KCurlWrapper($remotePath); $res = $curlWrapper->exec($localPath); KalturaLog::debug("Curl results: {$res}"); if (!$res || $curlWrapper->getError()) { $errDescription = "Error: " . $curlWrapper->getError(); $curlWrapper->close(); return false; } $curlWrapper->close(); } if (!file_exists($localPath)) { $errDescription = "Error: output file doesn't exist"; return false; } if ($fileSize) { clearstatcache(); if (kFile::fileSize($localPath) != $fileSize) { $errDescription = "Error: output file have a wrong size"; return false; } } } catch (Exception $ex) { $errDescription = "Error: " . $ex->getMessage(); return false; } return true; }
/** * @param string $remotePath * @param string $localPath * @param string $errDescription * @return boolean */ private function fetchFile($remotePath, $localPath, &$errDescription) { KalturaLog::debug("Fetch url [{$remotePath}] to file [{$localPath}]"); try { $folder = substr($localPath, 0, strrpos($localPath, '/')); if (!file_exists($folder)) { mkdir($folder, 777, true); } $curlWrapper = new KCurlWrapper($remotePath); $curlHeaderResponse = $curlWrapper->getHeader(true); if (!$curlHeaderResponse || $curlWrapper->getError()) { $errDescription = "Error: " . $curlWrapper->getError(); return false; } if (!$curlHeaderResponse->isGoodCode()) { $errDescription = "HTTP Error: " . $curlHeaderResponse->code . " " . $curlHeaderResponse->codeName; return false; } $fileSize = null; if (isset($curlHeaderResponse->headers['content-length'])) { $fileSize = $curlHeaderResponse->headers['content-length']; } $curlWrapper->close(); KalturaLog::debug("Executing curl"); $curlWrapper = new KCurlWrapper($remotePath); $res = $curlWrapper->exec($localPath); KalturaLog::debug("Curl results: {$res}"); if (!$res || $curlWrapper->getError()) { $errDescription = "Error: " . $curlWrapper->getError(); $curlWrapper->close(); return false; } $curlWrapper->close(); if (!file_exists($localPath)) { $errDescription = "Error: output file doesn't exist"; return false; } if ($fileSize) { clearstatcache(); if (filesize($localPath) != $fileSize) { $errDescription = "Error: output file have a wrong size"; return false; } } } catch (Exception $ex) { $errDescription = "Error: " . $ex->getMessage(); return false; } return true; }
/** * @param string $srcFileSyncRemoteUrl * @param string $srcFileSyncLocalPath * @param string $errDescription * @return string */ private function fetchFile($srcFileSyncRemoteUrl, $srcFileSyncLocalPath, &$errDescription = null) { KalturaLog::debug("fetchFile({$srcFileSyncRemoteUrl}, {$srcFileSyncLocalPath})"); try { $curlWrapper = new KCurlWrapper($srcFileSyncRemoteUrl); $curlHeaderResponse = $curlWrapper->getHeader(true); if (!$curlHeaderResponse || $curlWrapper->getError()) { $errDescription = "Error: " . $curlWrapper->getError(); return false; } if ($curlHeaderResponse->code != KCurlHeaderResponse::HTTP_STATUS_OK) { $errDescription = "HTTP Error: " . $curlHeaderResponse->code . " " . $curlHeaderResponse->codeName; return false; } $fileSize = null; if (isset($curlHeaderResponse->headers['content-length'])) { $fileSize = $curlHeaderResponse->headers['content-length']; } $curlWrapper->close(); KalturaLog::debug("Executing curl"); $curlWrapper = new KCurlWrapper($srcFileSyncRemoteUrl); $res = $curlWrapper->exec($srcFileSyncLocalPath); KalturaLog::debug("Curl results: {$res}"); if (!$res || $curlWrapper->getError()) { $errDescription = "Error: " . $curlWrapper->getError(); $curlWrapper->close(); return false; } $curlWrapper->close(); if (!file_exists($srcFileSyncLocalPath)) { $errDescription = "Error: output file doesn't exist"; return false; } if ($fileSize) { clearstatcache(); if (filesize($srcFileSyncLocalPath) != $fileSize) { $errDescription = "Error: output file have a wrong size"; return false; } } } catch (Exception $ex) { $errDescription = "Error: " . $ex->getMessage(); return false; } return true; }
private function fetchFile(KalturaBatchJob $job, KalturaImportJobData $data) { KalturaLog::debug("fetchFile({$job->id})"); $jobSubType = $job->jobSubType; if (in_array($jobSubType, array(kFileTransferMgrType::SCP, kFileTransferMgrType::SFTP))) { // use SSH file transfer manager for SFTP/SCP return $this->fetchFileSsh($job, $data); } try { $sourceUrl = $data->srcFileUrl; KalturaLog::debug("sourceUrl [{$sourceUrl}]"); $this->updateJob($job, 'Downloading file header', KalturaBatchJobStatus::QUEUED, 1); $fileSize = null; $resumeOffset = 0; if ($data->destFileLocalPath && file_exists($data->destFileLocalPath)) { $curlWrapper = new KCurlWrapper($sourceUrl, $this->taskConfig->params->curlVerbose); $useNoBody = $job->executionAttempts > 1; // if the process crashed first time, tries with no body instead of range 0-0 $curlHeaderResponse = $curlWrapper->getHeader($useNoBody); if (!$curlHeaderResponse || !count($curlHeaderResponse->headers)) { $this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $curlWrapper->getErrorNumber(), "Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::FAILED); return $job; } if ($curlWrapper->getError()) { KalturaLog::err("Headers error: " . $curlWrapper->getError()); KalturaLog::err("Headers error number: " . $curlWrapper->getErrorNumber()); $curlWrapper->close(); $curlWrapper = new KCurlWrapper($sourceUrl, $this->taskConfig->params->curlVerbose); } if (!$curlHeaderResponse->isGoodCode()) { $this->closeJob($job, KalturaBatchJobErrorTypes::HTTP, $curlHeaderResponse->code, "HTTP Error: " . $curlHeaderResponse->code . " " . $curlHeaderResponse->codeName, KalturaBatchJobStatus::FAILED); return $job; } if (isset($curlHeaderResponse->headers['content-length'])) { $fileSize = $curlHeaderResponse->headers['content-length']; } $curlWrapper->close(); if ($fileSize) { clearstatcache(); $actualFileSize = kFile::fileSize($data->destFileLocalPath); if ($actualFileSize >= $fileSize) { return $this->moveFile($job, $data->destFileLocalPath, $fileSize); } else { $resumeOffset = $actualFileSize; } } } $curlWrapper = new KCurlWrapper($sourceUrl, $this->taskConfig->params->curlVerbose); $curlWrapper->setTimeout($this->taskConfig->params->curlTimeout); if ($resumeOffset) { $curlWrapper->setResumeOffset($resumeOffset); } else { // creates a temp file path $destFile = $this->getTempFilePath($sourceUrl); KalturaLog::debug("destFile [{$destFile}]"); $data->destFileLocalPath = $destFile; $this->updateJob($job, "Downloading file, size: {$fileSize}", KalturaBatchJobStatus::PROCESSING, 2, $data); } KalturaLog::debug("Executing curl"); $res = $curlWrapper->exec($data->destFileLocalPath); KalturaLog::debug("Curl results: {$res}"); if (!$res || $curlWrapper->getError()) { $errNumber = $curlWrapper->getErrorNumber(); if ($errNumber != CURLE_OPERATION_TIMEOUTED) { $this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $errNumber, "Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::RETRY); $curlWrapper->close(); return $job; } else { clearstatcache(); $actualFileSize = kFile::fileSize($data->destFileLocalPath); if ($actualFileSize == $resumeOffset) { $this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $errNumber, "Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::RETRY); $curlWrapper->close(); return $job; } } } $curlWrapper->close(); if (!file_exists($data->destFileLocalPath)) { $this->closeJob($job, KalturaBatchJobErrorTypes::APP, KalturaBatchJobAppErrors::OUTPUT_FILE_DOESNT_EXIST, "Error: output file doesn't exist", KalturaBatchJobStatus::RETRY); return $job; } // check the file size only if its first or second retry // in case it failed few times, taks the file as is if ($fileSize) { clearstatcache(); $actualFileSize = kFile::fileSize($data->destFileLocalPath); if ($actualFileSize < $fileSize) { $percent = floor($actualFileSize * 100 / $fileSize); $this->updateJob($job, "Downloaded size: {$actualFileSize}({$percent}%)", KalturaBatchJobStatus::PROCESSING, $percent, $data); $this->kClient->batch->resetJobExecutionAttempts($job->id, $this->getExclusiveLockKey(), $job->jobType); // $this->closeJob($job, KalturaBatchJobErrorTypes::APP, KalturaBatchJobAppErrors::OUTPUT_FILE_WRONG_SIZE, "Expected file size[$fileSize] actual file size[$actualFileSize]", KalturaBatchJobStatus::RETRY); return $job; } } $this->updateJob($job, 'File imported, copy to shared folder', KalturaBatchJobStatus::PROCESSED, 90); $job = $this->moveFile($job, $data->destFileLocalPath, $fileSize); } catch (Exception $ex) { $this->closeJob($job, KalturaBatchJobErrorTypes::RUNTIME, $ex->getCode(), "Error: " . $ex->getMessage(), KalturaBatchJobStatus::FAILED); } return $job; }
/** * Download a file from $sourceUrl to $fileDestination * @param KalturaBatchJob $job * @param string $sourceUrl * @param string $fileDestination * @param KCurlHeaderResponse $curlHeaderResponse header fetched for the $sourceUrl */ private function fetchFile(KalturaBatchJob &$job, $sourceUrl, $fileDestination, $curlHeaderResponse = null, $fileSize = null) { KalturaLog::debug('fetchFile - job id [' . $job->id . '], source url [' . $sourceUrl . '], destination [' . $fileDestination . ']'); if (!$fileSize) { // fetch header if not given if (!$curlHeaderResponse) { $curlHeaderResponse = $this->fetchHeader($job, $sourceUrl); if (!$curlHeaderResponse) { return false; // job already closed with an error } } // try to get file size from headers $fileSize = null; if (isset($curlHeaderResponse->headers['content-length'])) { $fileSize = $curlHeaderResponse->headers['content-length']; } } // if file already exists - check if we can start from specific offset on exising partial content $resumeOffset = 0; if ($fileSize && $fileDestination && file_exists($fileDestination)) { clearstatcache(); $actualFileSize = filesize($fileDestination); if ($actualFileSize >= $fileSize) { // file download finished ? KalturaLog::debug('File exists with size [' . $actualFileSize . '] - checking if finished...'); return $this->checkFile($job, $fileDestination, $fileSize); } else { // will resume from the current offset KalturaLog::debug('File partialy exists - resume offset set to [' . $actualFileSize . ']'); $resumeOffset = $actualFileSize; } } // get http body $curlWrapper = new KCurlWrapper($sourceUrl); $curlWrapper->setTimeout($this->taskConfig->params->curlTimeout); if ($resumeOffset) { // will resume from the current offset $curlWrapper->setResumeOffset($resumeOffset); } else { // create destination directory if doesn't already exist $res = self::createDir(dirname($fileDestination)); if (!$res) { $msg = 'Error: Cannot create destination directory [' . dirname($fileDestination) . ']'; KalturaLog::err($msg); $this->closeJob($job, KalturaBatchJobErrorTypes::APP, KalturaBatchJobAppErrors::CANNOT_CREATE_DIRECTORY, $msg, KalturaBatchJobStatus::RETRY); return false; } // about to start downloading $this->updateJob($job, "Downloading file, size: {$fileSize}", KalturaBatchJobStatus::PROCESSING); } KalturaLog::debug("Executing curl for downloading file at [{$sourceUrl}]"); $res = $curlWrapper->exec($fileDestination); // download file KalturaLog::debug("Curl results: {$res}"); // handle errors if (!$res || $curlWrapper->getError()) { $errNumber = $curlWrapper->getErrorNumber(); if ($errNumber != CURLE_OPERATION_TIMEOUTED) { // an error other than timeout occured - cannot continue (timeout is handled with resuming) $msg = 'Error: ' . $curlWrapper->getError(); KalturaLog::err($msg); $this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $errNumber, $msg, KalturaBatchJobStatus::RETRY); $curlWrapper->close(); return false; } else { // timeout error occured KalturaLog::debug('Timeout occured'); clearstatcache(); $actualFileSize = filesize($fileDestination); if ($actualFileSize == $resumeOffset) { // no downloading was done at all - error $msg = 'Error: ' . $curlWrapper->getError(); KalturaLog::err($msg); $this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $errNumber, $msg, KalturaBatchJobStatus::RETRY); $curlWrapper->close(); return false; } } } $curlWrapper->close(); if (!file_exists($fileDestination)) { // destination file does not exist for an unknown reason $msg = "Error: output file doesn't exist"; KalturaLog::err($msg); $this->closeJob($job, KalturaBatchJobErrorTypes::APP, KalturaBatchJobAppErrors::OUTPUT_FILE_DOESNT_EXIST, $msg, KalturaBatchJobStatus::RETRY); return false; } if ($fileSize) { clearstatcache(); $actualFileSize = filesize($fileDestination); if ($actualFileSize < $fileSize) { // part of file was downloaded - will resume in next run KalturaLog::debug('File partialy downloaded - will resumt in next run'); $this->updateJob($job, "Downloaded size: {$actualFileSize}", KalturaBatchJobStatus::PROCESSING); $this->kClient->batch->resetJobExecutionAttempts($job->id, $this->getExclusiveLockKey(), $job->jobType); return false; } } KalturaLog::debug('File downloaded completely - will now check if done...'); $this->updateJob($job, 'File downloaded', KalturaBatchJobStatus::PROCESSING); // file downloaded completely - check it return $this->checkFile($job, $fileDestination, $fileSize); }
private function fetchFile(KalturaBatchJob $job, KalturaImportJobData $data) { KalturaLog::debug("fetchFile({$job->id})"); try { $sourceUrl = $data->srcFileUrl; KalturaLog::debug("sourceUrl [{$sourceUrl}]"); $this->updateJob($job, 'Downloading file header', KalturaBatchJobStatus::QUEUED, 1); $curlWrapper = new KCurlWrapper($sourceUrl); $useNoBody = $job->executionAttempts > 1; // if the process crashed first time, tries with no body instead of range 0-0 $curlHeaderResponse = $curlWrapper->getHeader($useNoBody); if (!$curlHeaderResponse || $curlWrapper->getError()) { $this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $curlWrapper->getErrorNumber(), "Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::FAILED); return $job; } if (!$curlHeaderResponse->isGoodCode()) { $this->closeJob($job, KalturaBatchJobErrorTypes::HTTP, $curlHeaderResponse->code, "HTTP Error: " . $curlHeaderResponse->code . " " . $curlHeaderResponse->codeName, KalturaBatchJobStatus::FAILED); return $job; } $fileSize = null; if (isset($curlHeaderResponse->headers['content-length'])) { $fileSize = $curlHeaderResponse->headers['content-length']; } $curlWrapper->close(); $resumeOffset = 0; if ($fileSize && $data->destFileLocalPath && file_exists($data->destFileLocalPath)) { clearstatcache(); $actualFileSize = filesize($data->destFileLocalPath); if ($actualFileSize >= $fileSize) { return $this->moveFile($job, $data->destFileLocalPath, $fileSize); } else { $resumeOffset = $actualFileSize; } } $curlWrapper = new KCurlWrapper($sourceUrl); $curlWrapper->setTimeout($this->taskConfig->params->curlTimeout); if ($resumeOffset) { $curlWrapper->setResumeOffset($resumeOffset); } else { // creates a temp file path $rootPath = $this->taskConfig->params->localTempPath; $res = self::createDir($rootPath); if (!$res) { KalturaLog::err("Cannot continue import without temp directory"); die; } $uniqid = uniqid('import_'); $destFile = realpath($rootPath) . "/{$uniqid}"; KalturaLog::debug("destFile [{$destFile}]"); // in case the url has added arguments, remove them (and reveal the URL path) // in order to find the file extension $urlPathEndIndex = strpos($sourceUrl, "?"); if ($urlPathEndIndex !== false) { $sourceUrlPath = substr($sourceUrl, 0, $urlPathEndIndex); } else { $sourceUrlPath = $sourceUrl; } $ext = pathinfo($sourceUrlPath, PATHINFO_EXTENSION); if (strlen($ext)) { $destFile .= ".{$ext}"; } $data->destFileLocalPath = $destFile; $this->updateJob($job, "Downloading file, size: {$fileSize}", KalturaBatchJobStatus::PROCESSING, 2, $data); } KalturaLog::debug("Executing curl"); $res = $curlWrapper->exec($data->destFileLocalPath); KalturaLog::debug("Curl results: {$res}"); if (!$res || $curlWrapper->getError()) { $errNumber = $curlWrapper->getErrorNumber(); if ($errNumber != CURLE_OPERATION_TIMEOUTED) { $this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $errNumber, "Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::RETRY); $curlWrapper->close(); return $job; } else { clearstatcache(); $actualFileSize = filesize($data->destFileLocalPath); if ($actualFileSize == $resumeOffset) { $this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $errNumber, "Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::RETRY); $curlWrapper->close(); return $job; } } } $curlWrapper->close(); if (!file_exists($data->destFileLocalPath)) { $this->closeJob($job, KalturaBatchJobErrorTypes::APP, KalturaBatchJobAppErrors::OUTPUT_FILE_DOESNT_EXIST, "Error: output file doesn't exist", KalturaBatchJobStatus::RETRY); return $job; } // check the file size only if its first or second retry // in case it failed few times, taks the file as is if ($fileSize) { clearstatcache(); $actualFileSize = filesize($data->destFileLocalPath); if ($actualFileSize < $fileSize) { $percent = floor($actualFileSize * 100 / $fileSize); $this->updateJob($job, "Downloaded size: {$actualFileSize}({$percent}%)", KalturaBatchJobStatus::PROCESSING, $percent, $data); $this->kClient->batch->resetJobExecutionAttempts($job->id, $this->getExclusiveLockKey(), $job->jobType); // $this->closeJob($job, KalturaBatchJobErrorTypes::APP, KalturaBatchJobAppErrors::OUTPUT_FILE_WRONG_SIZE, "Expected file size[$fileSize] actual file size[$actualFileSize]", KalturaBatchJobStatus::RETRY); return $job; } } $this->updateJob($job, 'File imported, copy to shared folder', KalturaBatchJobStatus::PROCESSED, 90); $job = $this->moveFile($job, $data->destFileLocalPath, $fileSize); } catch (Exception $ex) { $this->closeJob($job, KalturaBatchJobErrorTypes::RUNTIME, $ex->getCode(), "Error: " . $ex->getMessage(), KalturaBatchJobStatus::FAILED); } return $job; }
/** * @param string $srcFileSyncRemoteUrl * @param string $srcFileSyncLocalPath * @param string $errDescription * @return string */ private function fetchFile($srcFileSyncRemoteUrl, $srcFileSyncLocalPath, &$errDescription = null) { KalturaLog::debug("fetchFile({$srcFileSyncRemoteUrl}, {$srcFileSyncLocalPath})"); try { $curlWrapper = new KCurlWrapper($srcFileSyncRemoteUrl); $curlHeaderResponse = $curlWrapper->getHeader(true); if (!$curlHeaderResponse || $curlWrapper->getError()) { $errDescription = "Error: " . $curlWrapper->getError(); return false; } if ($curlHeaderResponse->code != KCurlHeaderResponse::HTTP_STATUS_OK) { $errDescription = "HTTP Error: " . $curlHeaderResponse->code . " " . $curlHeaderResponse->codeName; return false; } $fileSize = null; KalturaLog::debug("Headers:\n" . print_r($curlHeaderResponse->headers, true)); if (isset($curlHeaderResponse->headers['content-length'])) { $fileSize = $curlHeaderResponse->headers['content-length']; } $curlWrapper->close(); if ($fileSize && file_exists($srcFileSyncLocalPath)) { clearstatcache(); $actualFileSize = filesize($srcFileSyncLocalPath); if ($actualFileSize == $fileSize) { KalturaLog::log("File [{$srcFileSyncLocalPath}] already exists with right size[{$fileSize}]"); return true; } KalturaLog::log("File [{$srcFileSyncLocalPath}] already exists with wrong size[{$actualFileSize}] expected size[{$fileSize}]"); KalturaLog::debug("Unlink file[{$srcFileSyncLocalPath}]"); unlink($srcFileSyncLocalPath); } KalturaLog::debug("Executing curl"); $curlWrapper = new KCurlWrapper($srcFileSyncRemoteUrl); $res = $curlWrapper->exec($srcFileSyncLocalPath); KalturaLog::debug("Curl results: {$res}"); if (!$res || $curlWrapper->getError()) { $errDescription = "Error: " . $curlWrapper->getError(); $curlWrapper->close(); return false; } $curlWrapper->close(); if (!file_exists($srcFileSyncLocalPath)) { $errDescription = "Error: output file[{$srcFileSyncLocalPath}] doesn't exist"; return false; } clearstatcache(); $actualFileSize = filesize($srcFileSyncLocalPath); KalturaLog::debug("Fetched file to [{$srcFileSyncLocalPath}] size[{$actualFileSize}]"); if ($fileSize) { if ($actualFileSize != $fileSize) { $errDescription = "Error: output file[{$srcFileSyncLocalPath}] have a wrong size[{$actualFileSize}] expected size[{$fileSize}]"; return false; } } } catch (Exception $ex) { $errDescription = "Error: " . $ex->getMessage(); return false; } return true; }
private static function retrieveCaptionFile($captionObj, $captionUrl, $destFolder) { KalturaLog::debug("Caption object:\n" . print_r($captionObj, 1)); KalturaLog::debug("Executing curl to retrieve caption asset file from - {$captionUrl}"); $curlWrapper = new KCurlWrapper(); $cptFilePath = $destFolder . ".temp." . $captionObj->languageCode . ".srt"; $res = $curlWrapper->exec($captionUrl, $cptFilePath); KalturaLog::debug("Curl results: {$res}"); if (!$res || $curlWrapper->getError()) { $errDescription = "Error: " . $curlWrapper->getError(); $curlWrapper->close(); KalturaLog::err("Failed to curl the caption file url({$captionUrl}). Error ({$errDescription})"); return null; } $curlWrapper->close(); if (!file_exists($cptFilePath)) { KalturaLog::err("Error: output file ({$cptFilePath}) doesn't exist"); return null; } KalturaLog::debug("Finished"); return $cptFilePath; }
/** * This Function can work in two ways if destFilePath is provided it will copy the url to the dest file else will return the file as string as an output * @param string $url - URL to get data from * @param $destFilePath - Optional URL to copy data to * @param int $maxFileSize - Optional max file size allowed for the retrieval action * @throws Exception | file as string | if $destFilePath provide - true or false */ public static function getDataFromFile($url, $destFilePath = null, $maxFileSize = null) { if (!is_null($maxFileSize)) { $curlWrapper = new KCurlWrapper(); $curlHeaderResponse = $curlWrapper->getHeader($url, true); $curlWrapper->close(); if (!$curlHeaderResponse || $curlWrapper->getError()) { throw new Exception("Failed to retrive Curl header response from file path [{$url}] with Error " . $curlWrapper->getError()); } if (!$curlHeaderResponse->isGoodCode()) { throw new Exception("Non Valid Error: {$curlHeaderResponse->code}" . " " . $curlHeaderResponse->codeName); } if (isset($curlHeaderResponse->headers['content-length'])) { $fileSize = $curlHeaderResponse->headers['content-length']; if ($fileSize > $maxFileSize) { throw new Exception("File size [{$fileSize}] Excedded Max Siae Allowed [{$maxFileSize}]"); } KalturaLog::debug("File size [{$fileSize}] validated"); } else { KalturaLog::debug("File size validation skipped"); } } $curlWrapper = new KCurlWrapper(); $res = $curlWrapper->exec($url, $destFilePath); $curlWrapper->close(); return $res; }
/** * * @param $wmData * @param string $destFileSyncLocalPath * @param string $cmdLine * @param string $ffmpegBin * @param string $mediaInfoBin * @return string */ public static function buildWatermarkedCommandLine($wmData, $destFileSyncLocalPath, $cmdLine, $ffmpegBin = "ffmpeg", $mediaInfoBin = "mediainfo") { KalturaLog::log("After:cmdline({$cmdLine})"); if (!isset($mediaInfoBin) || strlen($mediaInfoBin) == 0) { $mediaInfoBin = "mediainfo"; } /* * evaluate WM scale and margins, if any */ KalturaLog::log("Watermark data({$mediaInfoBin}):\n" . print_r($wmData, 1)); if (isset($wmData->scale) && is_string($wmData->scale)) { $wmData->scale = explode("x", $wmData->scale); } if (isset($wmData->margins) && is_string($wmData->margins)) { $wmData->margins = explode("x", $wmData->margins); } KalturaLog::log("Watermark data:\n" . print_r($wmData, 1)); /* * Retrieve watermark image file, * either from image entry or from external url * If both set, prefer image entry */ $imageDownloadUrl = null; $errStr = null; if (isset($wmData->imageEntry)) { $version = null; try { $imgEntry = KBatchBase::$kClient->baseEntry->get($wmData->imageEntry, $version); } catch (Exception $ex) { $imgEntry = null; $errStr = "Exception on retrieval of an image entry({$wmData->imageEntry}),\nexception:" . print_r($ex, 1); } if (isset($imgEntry)) { KalturaLog::log("Watermark entry: {$wmData->imageEntry}"); $imageDownloadUrl = $imgEntry->downloadUrl; } else { if (!isset($errStr)) { $errStr = "Failed to retrieve an image entry({$wmData->imageEntry})"; } } if (!isset($imgEntry)) { KalturaLog::err($errStr); } } if (!isset($imageDownloadUrl)) { if (isset($wmData->url)) { $imageDownloadUrl = $wmData->url; } else { KalturaLog::err("Missing watermark image data, neither via image-entry-id nor via external url."); return null; } } $wmTmpFilepath = $destFileSyncLocalPath . ".wmtmp"; KalturaLog::log("imageDownloadUrl({$imageDownloadUrl}), wmTmpFilepath({$wmTmpFilepath})"); /* * Get the watermark image file */ $curlWrapper = new KCurlWrapper(); $res = $curlWrapper->exec($imageDownloadUrl, $wmTmpFilepath); KalturaLog::debug("Curl results: {$res}"); if (!$res || $curlWrapper->getError()) { $errDescription = "Error: " . $curlWrapper->getError(); $curlWrapper->close(); KalturaLog::err("Failed to curl the caption file url({$imageDownloadUrl}). Error ({$errDescription})"); return null; } $curlWrapper->close(); if (!file_exists($wmTmpFilepath)) { KalturaLog::err("Error: output file ({$wmTmpFilepath}) doesn't exist"); return null; } KalturaLog::log("Successfully retrieved the watermark image file ({$wmTmpFilepath}) "); /* * Query the image file for format and dims */ $output = array(); exec("{$mediaInfoBin} {$wmTmpFilepath} | grep -i \"width\\|height\\|format\"", $ouput, $rv); if ($rv != 0) { KalturaLog::err("Failed to retrieve media data from watermark file ({$wmTmpFilepath}). rv({$rv}). Carry on without watermark."); return null; } foreach ($ouput as $line) { if (!isset($wmData->format) && stristr($line, "format") != false) { $str = stristr($line, ":"); $wmData->format = strtolower(trim($str, ": ")); } else { if (stristr($line, "width") != false) { $str = stristr($line, ":"); $wmData->width = (int) trim($str, ": "); } else { if (stristr($line, "height") != false) { $str = stristr($line, ":"); $wmData->height = (int) trim($str, ": "); } } } } switch ($wmData->format) { case "jpeg": $wmData->format = "jpg"; case "jpg": case "png": default: rename($wmTmpFilepath, "{$wmTmpFilepath}.{$wmData->format}"); $wmTmpFilepath = "{$wmTmpFilepath}.{$wmData->format}"; break; } KalturaLog::log("Updated Watermark data:\n" . print_r($wmData, 1)); /* * Evaluate WM scaling params and scale it accordingly */ $wid = null; $hgt = null; if (!isset($wmData->scale) && ($wmData->width % 2 != 0 || $wmData->height % 2 != 0)) { $wmData->scale = array(); $wmData->width -= $wmData->width % 2; $wmData->height -= $wmData->height % 2; $wmData->scale[0] = $wmData->width; $wmData->scale[1] = $wmData->height; } if (isset($wmData->scale)) { $wid = in_array($wmData->scale[0], array(null, 0, -1)) ? -1 : $wmData->scale[0]; $hgt = in_array($wmData->scale[1], array(null, 0, -1)) ? -1 : $wmData->scale[1]; if ($wid <= 0) { $wid = round($hgt * $wmData->width / $wmData->height); } if ($hgt <= 0) { $hgt = round($wid * $wmData->height / $wmData->width); } if (!($wid > 0 && $hgt > 0)) { $wid = $wmData->width; $hgt = $wmData->height; } if ($wid > 0) { $wid -= $wid % 2; } if ($hgt > 0) { $hgt -= $hgt % 2; } $wmFilePath = $wmTmpFilepath; } else { $wmFilePath = $wmTmpFilepath; $wid = $wmData->width; $hgt = $wmData->height; } /* Samples - "[1]scale=100:100,setsar=100/100[logo];[0:v]crop=100:100:iw-ow-10:300,setsar=100/100[cropped];[cropped][logo]blend=all_expr='if(eq(mod(X,2),mod(Y,2)),A,B)'[blended];[0:v][blended]overlay=main_w-overlay_w-10:300[out]" "[1]scale=100:100,setsar=100/100[logo];[0:v][logo]overlay=main_w-overlay_w-10:300[out]" -map "[out]" */ $cmdLine = str_replace(array(KDLCmdlinePlaceholders::WaterMarkFileName, KDLCmdlinePlaceholders::WaterMarkWidth, KDLCmdlinePlaceholders::WaterMarkHeight), array($wmFilePath, $wid, $hgt), $cmdLine); KalturaLog::log("After:cmdline({$cmdLine})"); return $cmdLine; }
public static function handleImportContent($curlInfo, $importData, $params) { if (!($curlInfo->headers['content-length'] < 16000 && $curlInfo->headers['content-type'] == 'text/html')) { return $importData; } KalturaLog::debug('content-length [' . $curlInfo->headers['content-length'] . '] content-type [' . $curlInfo->headers['content-type'] . ']'); KalturaLog::info('Handle Import data: Webex Plugin'); $matches = null; $recordId = null; if (isset($curlInfo->headers['set-cookie'])) { $recordId = $curlInfo->getCookieValue($curlInfo->headers['set-cookie'], 'recordId'); if ($recordId == null) { throw new Exception('recordId value not found'); } } else { throw new Exception('set-cookie was not found in header'); } $data = file_get_contents($importData->destFileLocalPath); KalturaLog::info("data:\n\n{$data}\n\n"); if (!preg_match("/href='([^']+)';/", $data, $matches)) { throw new Exception('Starting URL not found'); } $url2 = $matches[1]; $curlWrapper = new KCurlWrapper(); $curlWrapper->setOpt(CURLOPT_COOKIE, 'DetectionBrowserStatus=3|1|32|1|11|2;' . $curlInfo->headers["set-cookie"]); $result = $curlWrapper->exec($url2); KalturaLog::info("result:\n\n{$result}\n\n"); if (!preg_match("/var prepareTicket = '([^']+)';/", $result, $matches)) { throw new Exception('prepareTicket parameter not found'); } $prepareTicket = $matches[1]; if (!preg_match('/function (download\\(\\).+prepareTicket;)/s', $result, $matches)) { throw new Exception('download function not found'); } if (!preg_match('/http.+prepareTicket/', $matches[0], $matches)) { throw new Exception('prepareTicket URL not found'); } $url3 = $matches[0]; $url3 = str_replace(array('"', ' ', '+', 'recordId', 'prepareTicket=prepareTicket'), array('', '', '', $recordId, "prepareTicket={$prepareTicket}"), $url3); if (!preg_match("/var downloadUrl = '(http[^']+)' \\+ ticket;/", $result, $matches)) { throw new Exception('Download URL not found'); } $url4 = $matches[1]; $status = null; $iterations = isset($params->webex->iterations) && !is_null($params->webex->iterations) ? intval($params->webex->iterations) : 10; $sleep = isset($params->webex->sleep) && !is_null($params->webex->sleep) ? intval($params->webex->sleep) : 3; for ($i = 0; $i < $iterations; $i++) { $result = $curlWrapper->exec($url3); KalturaLog::info("result ({$i}):\n\n{$result}\n\n"); if (!preg_match("/window\\.parent\\.func_prepare\\('([^']+)','([^']*)','([^']*)'\\);/", $result, $matches)) { KalturaLog::err("Invalid result returned for prepareTicket request - should contain call to the func_prepare method\n {$result}"); throw new Exception('Invalid result: func_prepare function not found'); } $status = $matches[1]; if ($status == 'OKOK') { break; } sleep($sleep); } if ($status != 'OKOK') { KalturaLog::info("Invalid result returned for prepareTicket request. Last result:\n " . $result); throw new kTemporaryException('Invalid result returned for prepareTicket request'); } $ticket = $matches[3]; $url4 .= $ticket; $curlWrapper->setOpt(CURLOPT_RETURNTRANSFER, false); $fileName = pathinfo($importData->destFileLocalPath, PATHINFO_FILENAME); $destFileLocalPath = preg_replace("/{$fileName}\\.[\\w\\d]+/", "{$fileName}.arf", $importData->destFileLocalPath); $importData->destFileLocalPath = $destFileLocalPath; KalturaLog::info('destination: ' . $importData->destFileLocalPath); $result = $curlWrapper->exec($url4, $importData->destFileLocalPath); if (!$result) { $code = $curlWrapper->getErrorNumber(); $message = $curlWrapper->getError(); throw new Exception($message, $code); } $curlWrapper->close(); $importData->fileSize = kFile::fileSize($importData->destFileLocalPath); return $importData; }
private function fetchFile(KalturaBatchJob $job, KalturaImportJobData $data) { $jobSubType = $job->jobSubType; $sshProtocols = array(kFileTransferMgrType::SCP, kFileTransferMgrType::SFTP); if (in_array($jobSubType, $sshProtocols)) { // use SSH file transfer manager for SFTP/SCP return $this->fetchFileSsh($job, $data); } try { $sourceUrl = $data->srcFileUrl; $this->updateJob($job, 'Downloading file header', KalturaBatchJobStatus::QUEUED); $fileSize = null; $resumeOffset = 0; if ($data->destFileLocalPath && file_exists($data->destFileLocalPath)) { $curlWrapper = new KCurlWrapper(self::$taskConfig->params); $useNoBody = $job->executionAttempts > 1; // if the process crashed first time, tries with no body instead of range 0-0 $curlHeaderResponse = $curlWrapper->getHeader($sourceUrl, $useNoBody); if (!$curlHeaderResponse || !count($curlHeaderResponse->headers)) { $this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $curlWrapper->getErrorNumber(), "Couldn't read file. Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::FAILED); return $job; } if ($curlWrapper->getError()) { KalturaLog::err("Headers error: " . $curlWrapper->getError()); KalturaLog::err("Headers error number: " . $curlWrapper->getErrorNumber()); $curlWrapper->close(); $curlWrapper = new KCurlWrapper(self::$taskConfig->params); } if (!$curlHeaderResponse->isGoodCode()) { $this->closeJob($job, KalturaBatchJobErrorTypes::HTTP, $curlHeaderResponse->code, "Failed while reading file. HTTP Error: " . $curlHeaderResponse->code . " " . $curlHeaderResponse->codeName, KalturaBatchJobStatus::FAILED); $curlWrapper->close(); return $job; } if (isset($curlHeaderResponse->headers['content-length'])) { $fileSize = $curlHeaderResponse->headers['content-length']; } $curlWrapper->close(); if ($fileSize) { clearstatcache(); $actualFileSize = kFile::fileSize($data->destFileLocalPath); if ($actualFileSize >= $fileSize) { return $this->moveFile($job, $data->destFileLocalPath, $fileSize); } else { $resumeOffset = $actualFileSize; } } } $curlWrapper = new KCurlWrapper(self::$taskConfig->params); if (is_null($fileSize)) { // Read file size $curlHeaderResponse = $curlWrapper->getHeader($sourceUrl, true); if ($curlHeaderResponse && count($curlHeaderResponse->headers) && !$curlWrapper->getError() && isset($curlHeaderResponse->headers['content-length'])) { $fileSize = $curlHeaderResponse->headers['content-length']; } //Close the curl used to fetch the header and create a new one. //When fetching headers we set curl options that than are not reset once header is fetched. //Not all servers support all the options so we need to remove them from our headers. $curlWrapper->close(); $curlWrapper = new KCurlWrapper(self::$taskConfig->params); } if ($resumeOffset) { $curlWrapper->setResumeOffset($resumeOffset); } else { // creates a temp file path $destFile = $this->getTempFilePath($sourceUrl); KalturaLog::debug("destFile [{$destFile}]"); $data->destFileLocalPath = $destFile; $data->fileSize = is_null($fileSize) ? -1 : $fileSize; $this->updateJob($job, "Downloading file, size: {$fileSize}", KalturaBatchJobStatus::PROCESSING, $data); } $res = $curlWrapper->exec($sourceUrl, $data->destFileLocalPath); KalturaLog::debug("Curl results: {$res}"); if (!$res || $curlWrapper->getError()) { $errNumber = $curlWrapper->getErrorNumber(); if ($errNumber != CURLE_OPERATION_TIMEOUTED) { $this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $errNumber, "Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::RETRY); $curlWrapper->close(); return $job; } else { clearstatcache(); $actualFileSize = kFile::fileSize($data->destFileLocalPath); if ($actualFileSize == $resumeOffset) { $this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $errNumber, "No new information. Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::RETRY); $curlWrapper->close(); return $job; } if (!$fileSize) { $this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $errNumber, "Received timeout, but no filesize available. Completed size [{$actualFileSize}]" . $curlWrapper->getError(), KalturaBatchJobStatus::RETRY); $curlWrapper->close(); return $job; } } } $curlWrapper->close(); if (!file_exists($data->destFileLocalPath)) { $this->closeJob($job, KalturaBatchJobErrorTypes::APP, KalturaBatchJobAppErrors::OUTPUT_FILE_DOESNT_EXIST, "Error: output file doesn't exist", KalturaBatchJobStatus::RETRY); return $job; } // check the file size only if its first or second retry // in case it failed few times, taks the file as is if ($fileSize) { clearstatcache(); $actualFileSize = kFile::fileSize($data->destFileLocalPath); if ($actualFileSize < $fileSize) { $percent = floor($actualFileSize * 100 / $fileSize); $this->updateJob($job, "Downloaded size: {$actualFileSize}({$percent}%)", KalturaBatchJobStatus::PROCESSING, $data); self::$kClient->batch->resetJobExecutionAttempts($job->id, $this->getExclusiveLockKey(), $job->jobType); // $this->closeJob($job, KalturaBatchJobErrorTypes::APP, KalturaBatchJobAppErrors::OUTPUT_FILE_WRONG_SIZE, "Expected file size[$fileSize] actual file size[$actualFileSize]", KalturaBatchJobStatus::RETRY); return $job; } KalturaLog::info("headers " . print_r($curlHeaderResponse, true)); $pluginInstances = KalturaPluginManager::getPluginInstances('IKalturaImportHandler'); foreach ($pluginInstances as $pluginInstance) { /* @var $pluginInstance IKalturaImportHandler */ $data = $pluginInstance->handleImportContent($curlHeaderResponse, $data, KBatchBase::$taskConfig->params); } } $this->updateJob($job, 'File imported, copy to shared folder', KalturaBatchJobStatus::PROCESSED); $job = $this->moveFile($job, $data->destFileLocalPath); } catch (kTemporaryException $tex) { $data->destFileLocalPath = KalturaClient::getKalturaNullValue(); $tex->setData($data); throw $tex; } catch (Exception $ex) { $data->destFileLocalPath = KalturaClient::getKalturaNullValue(); if ($ex->getMessage() == KCurlWrapper::COULD_NOT_CONNECT_TO_HOST_ERROR) { throw new kTemporaryException($ex->getMessage(), $ex->getCode(), $data); } $this->closeJob($job, KalturaBatchJobErrorTypes::RUNTIME, $ex->getCode(), "Error: " . $ex->getMessage(), KalturaBatchJobStatus::FAILED, $data); } return $job; }