function parseInputObject(SimpleXMLElement $input = null) { global $inFile; if (is_null($input)) { return null; } if (isset($input['null']) && $input['null']) { return KalturaClient::getKalturaNullValue(); } $type = 'string'; if (isset($input['objectType'])) { $type = strval($input['objectType']); } $value = strval($input); $matches = null; if (preg_match('/\\{prompt:([^}]+)\\}/', $value, $matches)) { $userInput = askForUserParameter($matches[1]); $value = preg_replace('/\\{prompt:[^}]+\\}/', $userInput, $value); } if (preg_match('/\\{php:([^}]+)\\}/', $value, $matches)) { $value = eval($matches[1]); } if (preg_match('/\\{variable:([^}]+)\\}/', $value, $matches)) { global $variables; $value = isset($variables[$matches[1]]) ? $variables[$matches[1]] : null; } if (isset($input['path'])) { $path = $input['path']; if (!file_exists($path)) { $path = dirname($inFile) . '/' . $path; if (!file_exists($path)) { echo "File [{$path}] could not be found\n"; exit(-1); } } $value = file_get_contents($path); } switch ($type) { case 'string': return $value; case 'int': return intval($value); case 'bool': return (bool) $value; case 'file': if (file_exists($value)) { return realpath($value); } $value = dirname($inFile) . '/' . $value; if (file_exists($value)) { return realpath($value); } echo "File [{$value}] could not be found\n"; exit(-1); case 'array': return parseInputArray($input->item); } if (!class_exists($type)) { echo "Type [{$type}] could not be found\n"; exit(-1); } $object = new $type(); $properties = $input->children(); foreach ($properties as $property) { /* @var $property SimpleXMLElement */ $propertyName = $property->getName(); $object->{$propertyName} = parseInputObject($property); } return $object; }
private function fetchFile(KalturaBatchJob $job, KalturaImportJobData $data) { $jobSubType = $job->jobSubType; $sshProtocols = array(kFileTransferMgrType::SCP, kFileTransferMgrType::SFTP); if (in_array($jobSubType, $sshProtocols)) { // use SSH file transfer manager for SFTP/SCP return $this->fetchFileSsh($job, $data); } try { $sourceUrl = $data->srcFileUrl; $this->updateJob($job, 'Downloading file header', KalturaBatchJobStatus::QUEUED); $fileSize = null; $resumeOffset = 0; if ($data->destFileLocalPath && file_exists($data->destFileLocalPath)) { $curlWrapper = new KCurlWrapper(self::$taskConfig->params); $useNoBody = $job->executionAttempts > 1; // if the process crashed first time, tries with no body instead of range 0-0 $curlHeaderResponse = $curlWrapper->getHeader($sourceUrl, $useNoBody); if (!$curlHeaderResponse || !count($curlHeaderResponse->headers)) { $this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $curlWrapper->getErrorNumber(), "Couldn't read file. Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::FAILED); return $job; } if ($curlWrapper->getError()) { KalturaLog::err("Headers error: " . $curlWrapper->getError()); KalturaLog::err("Headers error number: " . $curlWrapper->getErrorNumber()); $curlWrapper->close(); $curlWrapper = new KCurlWrapper(self::$taskConfig->params); } if (!$curlHeaderResponse->isGoodCode()) { $this->closeJob($job, KalturaBatchJobErrorTypes::HTTP, $curlHeaderResponse->code, "Failed while reading file. HTTP Error: " . $curlHeaderResponse->code . " " . $curlHeaderResponse->codeName, KalturaBatchJobStatus::FAILED); $curlWrapper->close(); return $job; } if (isset($curlHeaderResponse->headers['content-length'])) { $fileSize = $curlHeaderResponse->headers['content-length']; } $curlWrapper->close(); if ($fileSize) { clearstatcache(); $actualFileSize = kFile::fileSize($data->destFileLocalPath); if ($actualFileSize >= $fileSize) { return $this->moveFile($job, $data->destFileLocalPath, $fileSize); } else { $resumeOffset = $actualFileSize; } } } $curlWrapper = new KCurlWrapper(self::$taskConfig->params); if (is_null($fileSize)) { // Read file size $curlHeaderResponse = $curlWrapper->getHeader($sourceUrl, true); if ($curlHeaderResponse && count($curlHeaderResponse->headers) && !$curlWrapper->getError() && isset($curlHeaderResponse->headers['content-length'])) { $fileSize = $curlHeaderResponse->headers['content-length']; } //Close the curl used to fetch the header and create a new one. //When fetching headers we set curl options that than are not reset once header is fetched. //Not all servers support all the options so we need to remove them from our headers. $curlWrapper->close(); $curlWrapper = new KCurlWrapper(self::$taskConfig->params); } if ($resumeOffset) { $curlWrapper->setResumeOffset($resumeOffset); } else { // creates a temp file path $destFile = $this->getTempFilePath($sourceUrl); KalturaLog::debug("destFile [{$destFile}]"); $data->destFileLocalPath = $destFile; $data->fileSize = is_null($fileSize) ? -1 : $fileSize; $this->updateJob($job, "Downloading file, size: {$fileSize}", KalturaBatchJobStatus::PROCESSING, $data); } $res = $curlWrapper->exec($sourceUrl, $data->destFileLocalPath); KalturaLog::debug("Curl results: {$res}"); if (!$res || $curlWrapper->getError()) { $errNumber = $curlWrapper->getErrorNumber(); if ($errNumber != CURLE_OPERATION_TIMEOUTED) { $this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $errNumber, "Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::RETRY); $curlWrapper->close(); return $job; } else { clearstatcache(); $actualFileSize = kFile::fileSize($data->destFileLocalPath); if ($actualFileSize == $resumeOffset) { $this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $errNumber, "No new information. Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::RETRY); $curlWrapper->close(); return $job; } if (!$fileSize) { $this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $errNumber, "Received timeout, but no filesize available. Completed size [{$actualFileSize}]" . $curlWrapper->getError(), KalturaBatchJobStatus::RETRY); $curlWrapper->close(); return $job; } } } $curlWrapper->close(); if (!file_exists($data->destFileLocalPath)) { $this->closeJob($job, KalturaBatchJobErrorTypes::APP, KalturaBatchJobAppErrors::OUTPUT_FILE_DOESNT_EXIST, "Error: output file doesn't exist", KalturaBatchJobStatus::RETRY); return $job; } // check the file size only if its first or second retry // in case it failed few times, taks the file as is if ($fileSize) { clearstatcache(); $actualFileSize = kFile::fileSize($data->destFileLocalPath); if ($actualFileSize < $fileSize) { $percent = floor($actualFileSize * 100 / $fileSize); $this->updateJob($job, "Downloaded size: {$actualFileSize}({$percent}%)", KalturaBatchJobStatus::PROCESSING, $data); self::$kClient->batch->resetJobExecutionAttempts($job->id, $this->getExclusiveLockKey(), $job->jobType); // $this->closeJob($job, KalturaBatchJobErrorTypes::APP, KalturaBatchJobAppErrors::OUTPUT_FILE_WRONG_SIZE, "Expected file size[$fileSize] actual file size[$actualFileSize]", KalturaBatchJobStatus::RETRY); return $job; } KalturaLog::info("headers " . print_r($curlHeaderResponse, true)); $pluginInstances = KalturaPluginManager::getPluginInstances('IKalturaImportHandler'); foreach ($pluginInstances as $pluginInstance) { /* @var $pluginInstance IKalturaImportHandler */ $data = $pluginInstance->handleImportContent($curlHeaderResponse, $data, KBatchBase::$taskConfig->params); } } $this->updateJob($job, 'File imported, copy to shared folder', KalturaBatchJobStatus::PROCESSED); $job = $this->moveFile($job, $data->destFileLocalPath); } catch (kTemporaryException $tex) { $data->destFileLocalPath = KalturaClient::getKalturaNullValue(); $tex->setData($data); throw $tex; } catch (Exception $ex) { $data->destFileLocalPath = KalturaClient::getKalturaNullValue(); if ($ex->getMessage() == KCurlWrapper::COULD_NOT_CONNECT_TO_HOST_ERROR) { throw new kTemporaryException($ex->getMessage(), $ex->getCode(), $data); } $this->closeJob($job, KalturaBatchJobErrorTypes::RUNTIME, $ex->getCode(), "Error: " . $ex->getMessage(), KalturaBatchJobStatus::FAILED, $data); } return $job; }