/** * @param string $remotePath * @param string $localPath * @param string $errDescription * @return boolean */ private function fetchFile($remotePath, $localPath, &$errDescription) { KalturaLog::debug("Fetch url [{$remotePath}] to file [{$localPath}]"); try { $folder = substr($localPath, 0, strrpos($localPath, '/')); if (!file_exists($folder)) { mkdir($folder, 777, true); } $curlWrapper = new KCurlWrapper(); $curlHeaderResponse = $curlWrapper->getHeader($remotePath, true); $removeServer = isset($curlHeaderResponse->headers['X-Me']) ? $curlHeaderResponse->headers['X-Me'] : "unknown"; if (!$curlHeaderResponse || $curlWrapper->getError()) { $errDescription = "Error: ({$removeServer}) " . $curlWrapper->getError(); return false; } if (!$curlHeaderResponse->isGoodCode()) { $errDescription = "HTTP Error: ({$removeServer}) " . $curlHeaderResponse->code . " " . $curlHeaderResponse->codeName; return false; } $fileSize = null; if (isset($curlHeaderResponse->headers['content-length'])) { $fileSize = $curlHeaderResponse->headers['content-length']; } $curlWrapper->close(); KalturaLog::debug("Executing curl"); // overcome a 32bit issue with curl fetching >=4gb files if (intval("9223372036854775807") == 2147483647 && $fileSize >= 4 * 1024 * 1024 * 1024) { unlink($localPath); $cmd = "curl -s {$remotePath} -o {$localPath}"; KalturaLog::debug($cmd); exec($cmd); } else { $curlWrapper = new KCurlWrapper(); $res = $curlWrapper->exec($remotePath, $localPath); KalturaLog::debug("Curl results: {$res}"); if (!$res || $curlWrapper->getError()) { $errDescription = "Error: ({$removeServer}) " . $curlWrapper->getError(); $curlWrapper->close(); return false; } $curlWrapper->close(); } if (!file_exists($localPath)) { $errDescription = "Error: output file doesn't exist"; return false; } if ($fileSize) { clearstatcache(); if (kFile::fileSize($localPath) != $fileSize) { $errDescription = "Error: output file have a wrong size"; return false; } } } catch (Exception $ex) { $errDescription = "Error: " . $ex->getMessage(); return false; } return true; }
/** * @param string $remotePath * @param string $localPath * @param string $errDescription * @return boolean */ private function fetchFile($remotePath, $localPath, &$errDescription) { KalturaLog::debug("Fetch url [{$remotePath}] to file [{$localPath}]"); try { $folder = substr($localPath, 0, strrpos($localPath, '/')); if (!file_exists($folder)) { mkdir($folder, 777, true); } $curlWrapper = new KCurlWrapper($remotePath); $curlHeaderResponse = $curlWrapper->getHeader(true); if (!$curlHeaderResponse || $curlWrapper->getError()) { $errDescription = "Error: " . $curlWrapper->getError(); return false; } if (!$curlHeaderResponse->isGoodCode()) { $errDescription = "HTTP Error: " . $curlHeaderResponse->code . " " . $curlHeaderResponse->codeName; return false; } $fileSize = null; if (isset($curlHeaderResponse->headers['content-length'])) { $fileSize = $curlHeaderResponse->headers['content-length']; } $curlWrapper->close(); KalturaLog::debug("Executing curl"); $curlWrapper = new KCurlWrapper($remotePath); $res = $curlWrapper->exec($localPath); KalturaLog::debug("Curl results: {$res}"); if (!$res || $curlWrapper->getError()) { $errDescription = "Error: " . $curlWrapper->getError(); $curlWrapper->close(); return false; } $curlWrapper->close(); if (!file_exists($localPath)) { $errDescription = "Error: output file doesn't exist"; return false; } if ($fileSize) { clearstatcache(); if (filesize($localPath) != $fileSize) { $errDescription = "Error: output file have a wrong size"; return false; } } } catch (Exception $ex) { $errDescription = "Error: " . $ex->getMessage(); return false; } return true; }
/** * @param string $srcFileSyncRemoteUrl * @param string $srcFileSyncLocalPath * @param string $errDescription * @return string */ private function fetchFile($srcFileSyncRemoteUrl, $srcFileSyncLocalPath, &$errDescription = null) { KalturaLog::debug("fetchFile({$srcFileSyncRemoteUrl}, {$srcFileSyncLocalPath})"); try { $curlWrapper = new KCurlWrapper($srcFileSyncRemoteUrl); $curlHeaderResponse = $curlWrapper->getHeader(true); if (!$curlHeaderResponse || $curlWrapper->getError()) { $errDescription = "Error: " . $curlWrapper->getError(); return false; } if ($curlHeaderResponse->code != KCurlHeaderResponse::HTTP_STATUS_OK) { $errDescription = "HTTP Error: " . $curlHeaderResponse->code . " " . $curlHeaderResponse->codeName; return false; } $fileSize = null; if (isset($curlHeaderResponse->headers['content-length'])) { $fileSize = $curlHeaderResponse->headers['content-length']; } $curlWrapper->close(); KalturaLog::debug("Executing curl"); $curlWrapper = new KCurlWrapper($srcFileSyncRemoteUrl); $res = $curlWrapper->exec($srcFileSyncLocalPath); KalturaLog::debug("Curl results: {$res}"); if (!$res || $curlWrapper->getError()) { $errDescription = "Error: " . $curlWrapper->getError(); $curlWrapper->close(); return false; } $curlWrapper->close(); if (!file_exists($srcFileSyncLocalPath)) { $errDescription = "Error: output file doesn't exist"; return false; } if ($fileSize) { clearstatcache(); if (filesize($srcFileSyncLocalPath) != $fileSize) { $errDescription = "Error: output file have a wrong size"; return false; } } } catch (Exception $ex) { $errDescription = "Error: " . $ex->getMessage(); return false; } return true; }
private function fetchFile(KalturaBatchJob $job, KalturaImportJobData $data) { KalturaLog::debug("fetchFile({$job->id})"); $jobSubType = $job->jobSubType; if (in_array($jobSubType, array(kFileTransferMgrType::SCP, kFileTransferMgrType::SFTP))) { // use SSH file transfer manager for SFTP/SCP return $this->fetchFileSsh($job, $data); } try { $sourceUrl = $data->srcFileUrl; KalturaLog::debug("sourceUrl [{$sourceUrl}]"); $this->updateJob($job, 'Downloading file header', KalturaBatchJobStatus::QUEUED, 1); $fileSize = null; $resumeOffset = 0; if ($data->destFileLocalPath && file_exists($data->destFileLocalPath)) { $curlWrapper = new KCurlWrapper($sourceUrl, $this->taskConfig->params->curlVerbose); $useNoBody = $job->executionAttempts > 1; // if the process crashed first time, tries with no body instead of range 0-0 $curlHeaderResponse = $curlWrapper->getHeader($useNoBody); if (!$curlHeaderResponse || !count($curlHeaderResponse->headers)) { $this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $curlWrapper->getErrorNumber(), "Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::FAILED); return $job; } if ($curlWrapper->getError()) { KalturaLog::err("Headers error: " . $curlWrapper->getError()); KalturaLog::err("Headers error number: " . $curlWrapper->getErrorNumber()); $curlWrapper->close(); $curlWrapper = new KCurlWrapper($sourceUrl, $this->taskConfig->params->curlVerbose); } if (!$curlHeaderResponse->isGoodCode()) { $this->closeJob($job, KalturaBatchJobErrorTypes::HTTP, $curlHeaderResponse->code, "HTTP Error: " . $curlHeaderResponse->code . " " . $curlHeaderResponse->codeName, KalturaBatchJobStatus::FAILED); return $job; } if (isset($curlHeaderResponse->headers['content-length'])) { $fileSize = $curlHeaderResponse->headers['content-length']; } $curlWrapper->close(); if ($fileSize) { clearstatcache(); $actualFileSize = kFile::fileSize($data->destFileLocalPath); if ($actualFileSize >= $fileSize) { return $this->moveFile($job, $data->destFileLocalPath, $fileSize); } else { $resumeOffset = $actualFileSize; } } } $curlWrapper = new KCurlWrapper($sourceUrl, $this->taskConfig->params->curlVerbose); $curlWrapper->setTimeout($this->taskConfig->params->curlTimeout); if ($resumeOffset) { $curlWrapper->setResumeOffset($resumeOffset); } else { // creates a temp file path $destFile = $this->getTempFilePath($sourceUrl); KalturaLog::debug("destFile [{$destFile}]"); $data->destFileLocalPath = $destFile; $this->updateJob($job, "Downloading file, size: {$fileSize}", KalturaBatchJobStatus::PROCESSING, 2, $data); } KalturaLog::debug("Executing curl"); $res = $curlWrapper->exec($data->destFileLocalPath); KalturaLog::debug("Curl results: {$res}"); if (!$res || $curlWrapper->getError()) { $errNumber = $curlWrapper->getErrorNumber(); if ($errNumber != CURLE_OPERATION_TIMEOUTED) { $this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $errNumber, "Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::RETRY); $curlWrapper->close(); return $job; } else { clearstatcache(); $actualFileSize = kFile::fileSize($data->destFileLocalPath); if ($actualFileSize == $resumeOffset) { $this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $errNumber, "Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::RETRY); $curlWrapper->close(); return $job; } } } $curlWrapper->close(); if (!file_exists($data->destFileLocalPath)) { $this->closeJob($job, KalturaBatchJobErrorTypes::APP, KalturaBatchJobAppErrors::OUTPUT_FILE_DOESNT_EXIST, "Error: output file doesn't exist", KalturaBatchJobStatus::RETRY); return $job; } // check the file size only if its first or second retry // in case it failed few times, taks the file as is if ($fileSize) { clearstatcache(); $actualFileSize = kFile::fileSize($data->destFileLocalPath); if ($actualFileSize < $fileSize) { $percent = floor($actualFileSize * 100 / $fileSize); $this->updateJob($job, "Downloaded size: {$actualFileSize}({$percent}%)", KalturaBatchJobStatus::PROCESSING, $percent, $data); $this->kClient->batch->resetJobExecutionAttempts($job->id, $this->getExclusiveLockKey(), $job->jobType); // $this->closeJob($job, KalturaBatchJobErrorTypes::APP, KalturaBatchJobAppErrors::OUTPUT_FILE_WRONG_SIZE, "Expected file size[$fileSize] actual file size[$actualFileSize]", KalturaBatchJobStatus::RETRY); return $job; } } $this->updateJob($job, 'File imported, copy to shared folder', KalturaBatchJobStatus::PROCESSED, 90); $job = $this->moveFile($job, $data->destFileLocalPath, $fileSize); } catch (Exception $ex) { $this->closeJob($job, KalturaBatchJobErrorTypes::RUNTIME, $ex->getCode(), "Error: " . $ex->getMessage(), KalturaBatchJobStatus::FAILED); } return $job; }
/** * Fetches the header for the given $url and closes the job on any errors * @param KalturaBatchJob $job * @param string $url * @return false|KCurlHeaderResponse */ private function fetchHeader(KalturaBatchJob &$job, $url) { KalturaLog::debug('Fetching header for [' . $url . ']'); $this->updateJob($job, 'Downloading header for [' . $url . ']', KalturaBatchJobStatus::PROCESSING); // fetch the http headers $curlWrapper = new KCurlWrapper($url); $curlHeaderResponse = $curlWrapper->getHeader(); $curlError = $curlWrapper->getError(); $curlWrapper->close(); if (!$curlHeaderResponse || $curlError) { // error fetching headers $msg = 'Error: ' . $curlWrapper->getError(); KalturaLog::err($msg); $this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $curlWrapper->getErrorNumber(), $msg, KalturaBatchJobStatus::FAILED); return false; } if (!$curlHeaderResponse->isGoodCode()) { // some error exists in the response $msg = 'HTTP Error: ' . $curlHeaderResponse->code . ' ' . $curlHeaderResponse->codeName; KalturaLog::err($msg); $this->closeJob($job, KalturaBatchJobErrorTypes::HTTP, $curlHeaderResponse->code, $msg, KalturaBatchJobStatus::FAILED); return false; } // header fetched successfully - return it return $curlHeaderResponse; }
private function fetchFile(KalturaBatchJob $job, KalturaImportJobData $data) { KalturaLog::debug("fetchFile({$job->id})"); try { $sourceUrl = $data->srcFileUrl; KalturaLog::debug("sourceUrl [{$sourceUrl}]"); $this->updateJob($job, 'Downloading file header', KalturaBatchJobStatus::QUEUED, 1); $curlWrapper = new KCurlWrapper($sourceUrl); $useNoBody = $job->executionAttempts > 1; // if the process crashed first time, tries with no body instead of range 0-0 $curlHeaderResponse = $curlWrapper->getHeader($useNoBody); if (!$curlHeaderResponse || $curlWrapper->getError()) { $this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $curlWrapper->getErrorNumber(), "Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::FAILED); return $job; } if (!$curlHeaderResponse->isGoodCode()) { $this->closeJob($job, KalturaBatchJobErrorTypes::HTTP, $curlHeaderResponse->code, "HTTP Error: " . $curlHeaderResponse->code . " " . $curlHeaderResponse->codeName, KalturaBatchJobStatus::FAILED); return $job; } $fileSize = null; if (isset($curlHeaderResponse->headers['content-length'])) { $fileSize = $curlHeaderResponse->headers['content-length']; } $curlWrapper->close(); $resumeOffset = 0; if ($fileSize && $data->destFileLocalPath && file_exists($data->destFileLocalPath)) { clearstatcache(); $actualFileSize = filesize($data->destFileLocalPath); if ($actualFileSize >= $fileSize) { return $this->moveFile($job, $data->destFileLocalPath, $fileSize); } else { $resumeOffset = $actualFileSize; } } $curlWrapper = new KCurlWrapper($sourceUrl); $curlWrapper->setTimeout($this->taskConfig->params->curlTimeout); if ($resumeOffset) { $curlWrapper->setResumeOffset($resumeOffset); } else { // creates a temp file path $rootPath = $this->taskConfig->params->localTempPath; $res = self::createDir($rootPath); if (!$res) { KalturaLog::err("Cannot continue import without temp directory"); die; } $uniqid = uniqid('import_'); $destFile = realpath($rootPath) . "/{$uniqid}"; KalturaLog::debug("destFile [{$destFile}]"); // in case the url has added arguments, remove them (and reveal the URL path) // in order to find the file extension $urlPathEndIndex = strpos($sourceUrl, "?"); if ($urlPathEndIndex !== false) { $sourceUrlPath = substr($sourceUrl, 0, $urlPathEndIndex); } else { $sourceUrlPath = $sourceUrl; } $ext = pathinfo($sourceUrlPath, PATHINFO_EXTENSION); if (strlen($ext)) { $destFile .= ".{$ext}"; } $data->destFileLocalPath = $destFile; $this->updateJob($job, "Downloading file, size: {$fileSize}", KalturaBatchJobStatus::PROCESSING, 2, $data); } KalturaLog::debug("Executing curl"); $res = $curlWrapper->exec($data->destFileLocalPath); KalturaLog::debug("Curl results: {$res}"); if (!$res || $curlWrapper->getError()) { $errNumber = $curlWrapper->getErrorNumber(); if ($errNumber != CURLE_OPERATION_TIMEOUTED) { $this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $errNumber, "Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::RETRY); $curlWrapper->close(); return $job; } else { clearstatcache(); $actualFileSize = filesize($data->destFileLocalPath); if ($actualFileSize == $resumeOffset) { $this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $errNumber, "Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::RETRY); $curlWrapper->close(); return $job; } } } $curlWrapper->close(); if (!file_exists($data->destFileLocalPath)) { $this->closeJob($job, KalturaBatchJobErrorTypes::APP, KalturaBatchJobAppErrors::OUTPUT_FILE_DOESNT_EXIST, "Error: output file doesn't exist", KalturaBatchJobStatus::RETRY); return $job; } // check the file size only if its first or second retry // in case it failed few times, taks the file as is if ($fileSize) { clearstatcache(); $actualFileSize = filesize($data->destFileLocalPath); if ($actualFileSize < $fileSize) { $percent = floor($actualFileSize * 100 / $fileSize); $this->updateJob($job, "Downloaded size: {$actualFileSize}({$percent}%)", KalturaBatchJobStatus::PROCESSING, $percent, $data); $this->kClient->batch->resetJobExecutionAttempts($job->id, $this->getExclusiveLockKey(), $job->jobType); // $this->closeJob($job, KalturaBatchJobErrorTypes::APP, KalturaBatchJobAppErrors::OUTPUT_FILE_WRONG_SIZE, "Expected file size[$fileSize] actual file size[$actualFileSize]", KalturaBatchJobStatus::RETRY); return $job; } } $this->updateJob($job, 'File imported, copy to shared folder', KalturaBatchJobStatus::PROCESSED, 90); $job = $this->moveFile($job, $data->destFileLocalPath, $fileSize); } catch (Exception $ex) { $this->closeJob($job, KalturaBatchJobErrorTypes::RUNTIME, $ex->getCode(), "Error: " . $ex->getMessage(), KalturaBatchJobStatus::FAILED); } return $job; }
/** * @param string $srcFileSyncRemoteUrl * @param string $srcFileSyncLocalPath * @param string $errDescription * @return string */ private function fetchFile($srcFileSyncRemoteUrl, $srcFileSyncLocalPath, &$errDescription = null) { KalturaLog::debug("fetchFile({$srcFileSyncRemoteUrl}, {$srcFileSyncLocalPath})"); try { $curlWrapper = new KCurlWrapper($srcFileSyncRemoteUrl); $curlHeaderResponse = $curlWrapper->getHeader(true); if (!$curlHeaderResponse || $curlWrapper->getError()) { $errDescription = "Error: " . $curlWrapper->getError(); return false; } if ($curlHeaderResponse->code != KCurlHeaderResponse::HTTP_STATUS_OK) { $errDescription = "HTTP Error: " . $curlHeaderResponse->code . " " . $curlHeaderResponse->codeName; return false; } $fileSize = null; KalturaLog::debug("Headers:\n" . print_r($curlHeaderResponse->headers, true)); if (isset($curlHeaderResponse->headers['content-length'])) { $fileSize = $curlHeaderResponse->headers['content-length']; } $curlWrapper->close(); if ($fileSize && file_exists($srcFileSyncLocalPath)) { clearstatcache(); $actualFileSize = filesize($srcFileSyncLocalPath); if ($actualFileSize == $fileSize) { KalturaLog::log("File [{$srcFileSyncLocalPath}] already exists with right size[{$fileSize}]"); return true; } KalturaLog::log("File [{$srcFileSyncLocalPath}] already exists with wrong size[{$actualFileSize}] expected size[{$fileSize}]"); KalturaLog::debug("Unlink file[{$srcFileSyncLocalPath}]"); unlink($srcFileSyncLocalPath); } KalturaLog::debug("Executing curl"); $curlWrapper = new KCurlWrapper($srcFileSyncRemoteUrl); $res = $curlWrapper->exec($srcFileSyncLocalPath); KalturaLog::debug("Curl results: {$res}"); if (!$res || $curlWrapper->getError()) { $errDescription = "Error: " . $curlWrapper->getError(); $curlWrapper->close(); return false; } $curlWrapper->close(); if (!file_exists($srcFileSyncLocalPath)) { $errDescription = "Error: output file[{$srcFileSyncLocalPath}] doesn't exist"; return false; } clearstatcache(); $actualFileSize = filesize($srcFileSyncLocalPath); KalturaLog::debug("Fetched file to [{$srcFileSyncLocalPath}] size[{$actualFileSize}]"); if ($fileSize) { if ($actualFileSize != $fileSize) { $errDescription = "Error: output file[{$srcFileSyncLocalPath}] have a wrong size[{$actualFileSize}] expected size[{$fileSize}]"; return false; } } } catch (Exception $ex) { $errDescription = "Error: " . $ex->getMessage(); return false; } return true; }
/** * This Function can work in two ways if destFilePath is provided it will copy the url to the dest file else will return the file as string as an output * @param string $url - URL to get data from * @param $destFilePath - Optional URL to copy data to * @param int $maxFileSize - Optional max file size allowed for the retrieval action * @throws Exception | file as string | if $destFilePath provide - true or false */ public static function getDataFromFile($url, $destFilePath = null, $maxFileSize = null) { if (!is_null($maxFileSize)) { $curlWrapper = new KCurlWrapper(); $curlHeaderResponse = $curlWrapper->getHeader($url, true); $curlWrapper->close(); if (!$curlHeaderResponse || $curlWrapper->getError()) { throw new Exception("Failed to retrive Curl header response from file path [{$url}] with Error " . $curlWrapper->getError()); } if (!$curlHeaderResponse->isGoodCode()) { throw new Exception("Non Valid Error: {$curlHeaderResponse->code}" . " " . $curlHeaderResponse->codeName); } if (isset($curlHeaderResponse->headers['content-length'])) { $fileSize = $curlHeaderResponse->headers['content-length']; if ($fileSize > $maxFileSize) { throw new Exception("File size [{$fileSize}] Excedded Max Siae Allowed [{$maxFileSize}]"); } KalturaLog::debug("File size [{$fileSize}] validated"); } else { KalturaLog::debug("File size validation skipped"); } } $curlWrapper = new KCurlWrapper(); $res = $curlWrapper->exec($url, $destFilePath); $curlWrapper->close(); return $res; }
private function fetchFile(KalturaBatchJob $job, KalturaImportJobData $data) { $jobSubType = $job->jobSubType; $sshProtocols = array(kFileTransferMgrType::SCP, kFileTransferMgrType::SFTP); if (in_array($jobSubType, $sshProtocols)) { // use SSH file transfer manager for SFTP/SCP return $this->fetchFileSsh($job, $data); } try { $sourceUrl = $data->srcFileUrl; $this->updateJob($job, 'Downloading file header', KalturaBatchJobStatus::QUEUED); $fileSize = null; $resumeOffset = 0; if ($data->destFileLocalPath && file_exists($data->destFileLocalPath)) { $curlWrapper = new KCurlWrapper(self::$taskConfig->params); $useNoBody = $job->executionAttempts > 1; // if the process crashed first time, tries with no body instead of range 0-0 $curlHeaderResponse = $curlWrapper->getHeader($sourceUrl, $useNoBody); if (!$curlHeaderResponse || !count($curlHeaderResponse->headers)) { $this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $curlWrapper->getErrorNumber(), "Couldn't read file. Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::FAILED); return $job; } if ($curlWrapper->getError()) { KalturaLog::err("Headers error: " . $curlWrapper->getError()); KalturaLog::err("Headers error number: " . $curlWrapper->getErrorNumber()); $curlWrapper->close(); $curlWrapper = new KCurlWrapper(self::$taskConfig->params); } if (!$curlHeaderResponse->isGoodCode()) { $this->closeJob($job, KalturaBatchJobErrorTypes::HTTP, $curlHeaderResponse->code, "Failed while reading file. HTTP Error: " . $curlHeaderResponse->code . " " . $curlHeaderResponse->codeName, KalturaBatchJobStatus::FAILED); $curlWrapper->close(); return $job; } if (isset($curlHeaderResponse->headers['content-length'])) { $fileSize = $curlHeaderResponse->headers['content-length']; } $curlWrapper->close(); if ($fileSize) { clearstatcache(); $actualFileSize = kFile::fileSize($data->destFileLocalPath); if ($actualFileSize >= $fileSize) { return $this->moveFile($job, $data->destFileLocalPath, $fileSize); } else { $resumeOffset = $actualFileSize; } } } $curlWrapper = new KCurlWrapper(self::$taskConfig->params); if (is_null($fileSize)) { // Read file size $curlHeaderResponse = $curlWrapper->getHeader($sourceUrl, true); if ($curlHeaderResponse && count($curlHeaderResponse->headers) && !$curlWrapper->getError() && isset($curlHeaderResponse->headers['content-length'])) { $fileSize = $curlHeaderResponse->headers['content-length']; } //Close the curl used to fetch the header and create a new one. //When fetching headers we set curl options that than are not reset once header is fetched. //Not all servers support all the options so we need to remove them from our headers. $curlWrapper->close(); $curlWrapper = new KCurlWrapper(self::$taskConfig->params); } if ($resumeOffset) { $curlWrapper->setResumeOffset($resumeOffset); } else { // creates a temp file path $destFile = $this->getTempFilePath($sourceUrl); KalturaLog::debug("destFile [{$destFile}]"); $data->destFileLocalPath = $destFile; $data->fileSize = is_null($fileSize) ? -1 : $fileSize; $this->updateJob($job, "Downloading file, size: {$fileSize}", KalturaBatchJobStatus::PROCESSING, $data); } $res = $curlWrapper->exec($sourceUrl, $data->destFileLocalPath); KalturaLog::debug("Curl results: {$res}"); if (!$res || $curlWrapper->getError()) { $errNumber = $curlWrapper->getErrorNumber(); if ($errNumber != CURLE_OPERATION_TIMEOUTED) { $this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $errNumber, "Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::RETRY); $curlWrapper->close(); return $job; } else { clearstatcache(); $actualFileSize = kFile::fileSize($data->destFileLocalPath); if ($actualFileSize == $resumeOffset) { $this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $errNumber, "No new information. Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::RETRY); $curlWrapper->close(); return $job; } if (!$fileSize) { $this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $errNumber, "Received timeout, but no filesize available. Completed size [{$actualFileSize}]" . $curlWrapper->getError(), KalturaBatchJobStatus::RETRY); $curlWrapper->close(); return $job; } } } $curlWrapper->close(); if (!file_exists($data->destFileLocalPath)) { $this->closeJob($job, KalturaBatchJobErrorTypes::APP, KalturaBatchJobAppErrors::OUTPUT_FILE_DOESNT_EXIST, "Error: output file doesn't exist", KalturaBatchJobStatus::RETRY); return $job; } // check the file size only if its first or second retry // in case it failed few times, taks the file as is if ($fileSize) { clearstatcache(); $actualFileSize = kFile::fileSize($data->destFileLocalPath); if ($actualFileSize < $fileSize) { $percent = floor($actualFileSize * 100 / $fileSize); $this->updateJob($job, "Downloaded size: {$actualFileSize}({$percent}%)", KalturaBatchJobStatus::PROCESSING, $data); self::$kClient->batch->resetJobExecutionAttempts($job->id, $this->getExclusiveLockKey(), $job->jobType); // $this->closeJob($job, KalturaBatchJobErrorTypes::APP, KalturaBatchJobAppErrors::OUTPUT_FILE_WRONG_SIZE, "Expected file size[$fileSize] actual file size[$actualFileSize]", KalturaBatchJobStatus::RETRY); return $job; } KalturaLog::info("headers " . print_r($curlHeaderResponse, true)); $pluginInstances = KalturaPluginManager::getPluginInstances('IKalturaImportHandler'); foreach ($pluginInstances as $pluginInstance) { /* @var $pluginInstance IKalturaImportHandler */ $data = $pluginInstance->handleImportContent($curlHeaderResponse, $data, KBatchBase::$taskConfig->params); } } $this->updateJob($job, 'File imported, copy to shared folder', KalturaBatchJobStatus::PROCESSED); $job = $this->moveFile($job, $data->destFileLocalPath); } catch (kTemporaryException $tex) { $data->destFileLocalPath = KalturaClient::getKalturaNullValue(); $tex->setData($data); throw $tex; } catch (Exception $ex) { $data->destFileLocalPath = KalturaClient::getKalturaNullValue(); if ($ex->getMessage() == KCurlWrapper::COULD_NOT_CONNECT_TO_HOST_ERROR) { throw new kTemporaryException($ex->getMessage(), $ex->getCode(), $data); } $this->closeJob($job, KalturaBatchJobErrorTypes::RUNTIME, $ex->getCode(), "Error: " . $ex->getMessage(), KalturaBatchJobStatus::FAILED, $data); } return $job; }
public function testRedirectFileSize() { $curlWrapper = new KCurlWrapper('http://scoregolf.com/Web-Media/Video/2008/May/pumafinal.flv'); $curlHeaderResponse = $curlWrapper->getHeader(); var_dump($curlHeaderResponse); }