/**
* @param string $remotePath
* @param string $localPath
* @param string $errDescription
* @return boolean
*/
private function fetchFile($remotePath, $localPath, &$errDescription)
{
KalturaLog::debug("Fetch url [{$remotePath}] to file [{$localPath}]");
try {
$folder = substr($localPath, 0, strrpos($localPath, '/'));
if (!file_exists($folder)) {
mkdir($folder, 777, true);
}
$curlWrapper = new KCurlWrapper($remotePath);
$curlHeaderResponse = $curlWrapper->getHeader(true);
if (!$curlHeaderResponse || $curlWrapper->getError()) {
$errDescription = "Error: " . $curlWrapper->getError();
return false;
}
if (!$curlHeaderResponse->isGoodCode()) {
$errDescription = "HTTP Error: " . $curlHeaderResponse->code . " " . $curlHeaderResponse->codeName;
return false;
}
$fileSize = null;
if (isset($curlHeaderResponse->headers['content-length'])) {
$fileSize = $curlHeaderResponse->headers['content-length'];
}
$curlWrapper->close();
KalturaLog::debug("Executing curl");
// overcome a 32bit issue with curl fetching >=4gb files
if (intval("9223372036854775807") == 2147483647 && $fileSize >= 4 * 1024 * 1024 * 1024) {
unlink($localPath);
$cmd = "curl -s {$remotePath} -o {$localPath}";
KalturaLog::debug($cmd);
exec($cmd);
} else {
$curlWrapper = new KCurlWrapper($remotePath);
$res = $curlWrapper->exec($localPath);
KalturaLog::debug("Curl results: {$res}");
if (!$res || $curlWrapper->getError()) {
$errDescription = "Error: " . $curlWrapper->getError();
$curlWrapper->close();
return false;
}
$curlWrapper->close();
}
if (!file_exists($localPath)) {
$errDescription = "Error: output file doesn't exist";
return false;
}
if ($fileSize) {
clearstatcache();
if (kFile::fileSize($localPath) != $fileSize) {
$errDescription = "Error: output file have a wrong size";
return false;
}
}
} catch (Exception $ex) {
$errDescription = "Error: " . $ex->getMessage();
return false;
}
return true;
}
/**
* @param CaptionAsset $captionAsset
* @param BatchJob $parentJob
* @throws kCoreException FILE_NOT_FOUND
* @return BatchJob
*/
public function addParseCaptionAssetJob(CaptionAsset $captionAsset, BatchJob $parentJob = null)
{
$syncKey = $captionAsset->getSyncKey(asset::FILE_SYNC_ASSET_SUB_TYPE_ASSET);
$fileSync = kFileSyncUtils::getReadyInternalFileSyncForKey($syncKey);
if (!$fileSync) {
if (!PermissionPeer::isValidForPartner(CaptionPermissionName::IMPORT_REMOTE_CAPTION_FOR_INDEXING, $captionAsset->getPartnerId())) {
throw new kCoreException("File sync not found: {$syncKey}", kCoreException::FILE_NOT_FOUND);
}
$fileSync = kFileSyncUtils::getReadyExternalFileSyncForKey($syncKey);
if (!$fileSync) {
throw new kCoreException("File sync not found: {$syncKey}", kCoreException::FILE_NOT_FOUND);
}
$fullPath = myContentStorage::getFSUploadsPath() . '/' . $captionAsset->getId() . '.tmp';
if (!KCurlWrapper::getDataFromFile($fileSync->getExternalUrl($captionAsset->getEntryId()), $fullPath)) {
throw new kCoreException("File sync not found: {$syncKey}", kCoreException::FILE_NOT_FOUND);
}
kFileSyncUtils::moveFromFile($fullPath, $syncKey, true, false, true);
}
$jobData = new kParseCaptionAssetJobData();
$jobData->setCaptionAssetId($captionAsset->getId());
$batchJobType = CaptionSearchPlugin::getBatchJobTypeCoreValue(CaptionSearchBatchJobType::PARSE_CAPTION_ASSET);
$batchJob = null;
if ($parentJob) {
$batchJob = $parentJob->createChild($batchJobType);
} else {
$batchJob = new BatchJob();
$batchJob->setEntryId($captionAsset->getEntryId());
$batchJob->setPartnerId($captionAsset->getPartnerId());
}
$batchJob->setObjectId($captionAsset->getId());
$batchJob->setObjectType(BatchJobObjectType::ASSET);
return kJobsManager::addJob($batchJob, $jobData, $batchJobType);
}
public static function createThumbnailAssetFromFile(entry $entry, $filePath)
{
$fileLocation = tempnam(sys_get_temp_dir(), $entry->getId());
$res = KCurlWrapper::getDataFromFile($filePath, $fileLocation, kConf::get('thumb_size_limit'));
if (!$res) {
throw new Exception("thumbnail cannot be created from {$filePath} " . error_get_last());
}
$thumbAsset = new thumbAsset();
$thumbAsset->setPartnerId($entry->getPartnerId());
$thumbAsset->setEntryId($entry->getId());
$thumbAsset->setStatus(thumbAsset::ASSET_STATUS_QUEUED);
$thumbAsset->incrementVersion();
$thumbAsset->save();
$fileSyncKey = $thumbAsset->getSyncKey(asset::FILE_SYNC_ASSET_SUB_TYPE_ASSET);
kFileSyncUtils::moveFromFile($fileLocation, $fileSyncKey);
$finalPath = kFileSyncUtils::getLocalFilePathForKey($fileSyncKey);
$ext = pathinfo($finalPath, PATHINFO_EXTENSION);
$thumbAsset->setFileExt($ext);
list($width, $height, $type, $attr) = getimagesize($finalPath);
$thumbAsset->setWidth($width);
$thumbAsset->setHeight($height);
$thumbAsset->setSize(filesize($finalPath));
$thumbAsset->setStatus(thumbAsset::ASSET_STATUS_READY);
$thumbAsset->save();
kBusinessConvertDL::setAsDefaultThumbAsset($thumbAsset);
myNotificationMgr::createNotification(kNotificationJobData::NOTIFICATION_TYPE_ENTRY_UPDATE_THUMBNAIL, $entry);
}
/**
* @param string $remotePath
* @param string $localPath
* @param string $errDescription
* @return boolean
*/
private function fetchFile($remotePath, $localPath, &$errDescription)
{
KalturaLog::debug("Fetch url [{$remotePath}] to file [{$localPath}]");
try {
$folder = substr($localPath, 0, strrpos($localPath, '/'));
if (!file_exists($folder)) {
mkdir($folder, 777, true);
}
$curlWrapper = new KCurlWrapper($remotePath);
$curlHeaderResponse = $curlWrapper->getHeader(true);
if (!$curlHeaderResponse || $curlWrapper->getError()) {
$errDescription = "Error: " . $curlWrapper->getError();
return false;
}
if (!$curlHeaderResponse->isGoodCode()) {
$errDescription = "HTTP Error: " . $curlHeaderResponse->code . " " . $curlHeaderResponse->codeName;
return false;
}
$fileSize = null;
if (isset($curlHeaderResponse->headers['content-length'])) {
$fileSize = $curlHeaderResponse->headers['content-length'];
}
$curlWrapper->close();
KalturaLog::debug("Executing curl");
$curlWrapper = new KCurlWrapper($remotePath);
$res = $curlWrapper->exec($localPath);
KalturaLog::debug("Curl results: {$res}");
if (!$res || $curlWrapper->getError()) {
$errDescription = "Error: " . $curlWrapper->getError();
$curlWrapper->close();
return false;
}
$curlWrapper->close();
if (!file_exists($localPath)) {
$errDescription = "Error: output file doesn't exist";
return false;
}
if ($fileSize) {
clearstatcache();
if (filesize($localPath) != $fileSize) {
$errDescription = "Error: output file have a wrong size";
return false;
}
}
} catch (Exception $ex) {
$errDescription = "Error: " . $ex->getMessage();
return false;
}
return true;
}
/**
* Fetch the manifest and build all flavors array
* @param string $url
*/
private function buildF4mFlavors($url, array &$flavors, array &$bootstrapInfos)
{
$manifest = KCurlWrapper::getContent($url);
if (!$manifest) {
return;
}
$manifest = preg_replace('/xmlns="[^"]+"/', '', $manifest);
$xml = new SimpleXMLElement($manifest);
$mediaElements = $xml->xpath('/manifest/media');
foreach ($mediaElements as $mediaElement) {
/* @var $mediaElement SimpleXMLElement */
$flavor = array('urlPrefix' => '');
$playlistUrl = null;
foreach ($mediaElement->attributes() as $attr => $attrValue) {
$attrValue = "{$attrValue}";
if ($attr === 'url') {
$attrValue = requestUtils::resolve($attrValue, $url);
}
if ($attr === 'bootstrapInfoId') {
$bootstrapInfoElements = $xml->xpath("/manifest/bootstrapInfo[@id='{$attrValue}']");
if (count($bootstrapInfoElements)) {
$bootstrapInfoElement = reset($bootstrapInfoElements);
/* @var $bootstrapInfoElement SimpleXMLElement */
$playlistUrl = requestUtils::resolve(strval($bootstrapInfoElement['url']), $url);
}
}
$flavor["{$attr}"] = $attrValue;
}
if ($playlistUrl) {
$playlistId = md5($playlistUrl);
$bootstrapInfo = array('id' => $playlistId, 'url' => $playlistUrl);
$bootstrapInfos[$playlistId] = $bootstrapInfo;
$flavor['bootstrapInfoId'] = $playlistId;
}
$flavors[] = $flavor;
}
}
private static function retrieveCaptionFile($captionObj, $captionUrl, $destFolder)
{
KalturaLog::debug("Caption object:\n" . print_r($captionObj, 1));
KalturaLog::debug("Executing curl to retrieve caption asset file from - {$captionUrl}");
$curlWrapper = new KCurlWrapper();
$cptFilePath = $destFolder . ".temp." . $captionObj->languageCode . ".srt";
$res = $curlWrapper->exec($captionUrl, $cptFilePath);
KalturaLog::debug("Curl results: {$res}");
if (!$res || $curlWrapper->getError()) {
$errDescription = "Error: " . $curlWrapper->getError();
$curlWrapper->close();
KalturaLog::err("Failed to curl the caption file url({$captionUrl}). Error ({$errDescription})");
return null;
}
$curlWrapper->close();
if (!file_exists($cptFilePath)) {
KalturaLog::err("Error: output file ({$cptFilePath}) doesn't exist");
return null;
}
KalturaLog::debug("Finished");
return $cptFilePath;
}
/**
* @param string $srcFileSyncRemoteUrl
* @param string $srcFileSyncLocalPath
* @param string $errDescription
* @return string
*/
private function fetchFile($srcFileSyncRemoteUrl, $srcFileSyncLocalPath, &$errDescription = null)
{
KalturaLog::debug("fetchFile({$srcFileSyncRemoteUrl}, {$srcFileSyncLocalPath})");
try {
$curlWrapper = new KCurlWrapper($srcFileSyncRemoteUrl);
$curlHeaderResponse = $curlWrapper->getHeader(true);
if (!$curlHeaderResponse || $curlWrapper->getError()) {
$errDescription = "Error: " . $curlWrapper->getError();
return false;
}
if ($curlHeaderResponse->code != KCurlHeaderResponse::HTTP_STATUS_OK) {
$errDescription = "HTTP Error: " . $curlHeaderResponse->code . " " . $curlHeaderResponse->codeName;
return false;
}
$fileSize = null;
KalturaLog::debug("Headers:\n" . print_r($curlHeaderResponse->headers, true));
if (isset($curlHeaderResponse->headers['content-length'])) {
$fileSize = $curlHeaderResponse->headers['content-length'];
}
$curlWrapper->close();
if ($fileSize && file_exists($srcFileSyncLocalPath)) {
clearstatcache();
$actualFileSize = filesize($srcFileSyncLocalPath);
if ($actualFileSize == $fileSize) {
KalturaLog::log("File [{$srcFileSyncLocalPath}] already exists with right size[{$fileSize}]");
return true;
}
KalturaLog::log("File [{$srcFileSyncLocalPath}] already exists with wrong size[{$actualFileSize}] expected size[{$fileSize}]");
KalturaLog::debug("Unlink file[{$srcFileSyncLocalPath}]");
unlink($srcFileSyncLocalPath);
}
KalturaLog::debug("Executing curl");
$curlWrapper = new KCurlWrapper($srcFileSyncRemoteUrl);
$res = $curlWrapper->exec($srcFileSyncLocalPath);
KalturaLog::debug("Curl results: {$res}");
if (!$res || $curlWrapper->getError()) {
$errDescription = "Error: " . $curlWrapper->getError();
$curlWrapper->close();
return false;
}
$curlWrapper->close();
if (!file_exists($srcFileSyncLocalPath)) {
$errDescription = "Error: output file[{$srcFileSyncLocalPath}] doesn't exist";
return false;
}
clearstatcache();
$actualFileSize = filesize($srcFileSyncLocalPath);
KalturaLog::debug("Fetched file to [{$srcFileSyncLocalPath}] size[{$actualFileSize}]");
if ($fileSize) {
if ($actualFileSize != $fileSize) {
$errDescription = "Error: output file[{$srcFileSyncLocalPath}] have a wrong size[{$actualFileSize}] expected size[{$fileSize}]";
return false;
}
}
} catch (Exception $ex) {
$errDescription = "Error: " . $ex->getMessage();
return false;
}
return true;
}
private function handleEntry($onlyExtractThumb, $prefix, $type, $entry_id, $name = null, $tags = null, $entry = null)
{
$this->clear($prefix, $entry_id);
$kuser_id = $this->kuser_id;
$entry_data_prefix = $kuser_id . '_' . ($prefix == '' ? 'data' : rtrim($prefix, '_'));
$uploads = myContentStorage::getFSUploadsPath();
$content = myContentStorage::getFSContentRootPath();
$media_source = $this->getParam('entry_media_source');
$media_type = $this->getParam('entry_media_type');
$entry_url = $this->getParam('entry_url');
$entry_source_link = $this->getParam('entry_source_link');
$entry_fileName = $this->getParam('entry_data');
$entry_thumbNum = $this->getParam('entry_thumb_num', 0);
$entry_thumbUrl = $this->getParam('entry_thumb_url', '');
$entry_from_time = $this->getParam('entry_from_time', 0);
$entry_to_time = $this->getParam('entry_to_time', 0);
$should_copy = $this->getParam('should_copy', false);
$skip_conversion = $this->getParam('skip_conversion', false);
$webcam_suffix = $this->getParam('webcam_suffix', '');
$entry_fullPath = "";
$ext = null;
$duration = null;
$entry = null;
if ($entry_id) {
$entry = entryPeer::retrieveByPK($entry_id);
} else {
$entry = new entry();
}
$this->entry = $entry;
$entry_status = $entry->getStatus();
if (is_null($entry_status)) {
$entry_status = entryStatus::READY;
}
// by the end of this block of code $entry_fullPath will point to the location of the entry
// the entry status will be set (IMPORT / PRECONVERT / READY)
// a background image is always previewed by the user no matter what source he used
// so the entry is already in the /uploads directory
// continue tracking the file upload
$te = new TrackEntry();
$te->setEntryId($entry_id);
$te->setTrackEventTypeId(TrackEntry::TRACK_ENTRY_EVENT_TYPE_ADD_ENTRY);
KalturaLog::debug("handleEntry: media_source: {$media_source}, prefix: {$prefix}");
if ($media_source == entry::ENTRY_MEDIA_SOURCE_FILE || $prefix == 'bg_') {
$full_path = $this->getParam('entry_full_path');
if ($full_path) {
$entry_fullPath = $full_path;
} else {
$entry_fullPath = $uploads . $entry_data_prefix . strrchr($entry_fileName, '.');
}
if ($media_type == entry::ENTRY_MEDIA_TYPE_VIDEO || $media_type == entry::ENTRY_MEDIA_TYPE_AUDIO) {
$entry_status = entryStatus::PRECONVERT;
}
$te->setParam3Str($entry_fullPath);
$te->setDescription(__METHOD__ . ":" . __LINE__ . "::ENTRY_MEDIA_SOURCE_FILE");
} else {
if ($media_source == entry::ENTRY_MEDIA_SOURCE_WEBCAM) {
// set $entry_fileName to webcam output file and flag that conversion is not needed
$webcam_basePath = $content . '/content/webcam/' . ($webcam_suffix ? $webcam_suffix : 'my_recorded_stream_' . $kuser_id);
$entry_fullPath = $webcam_basePath . '.flv';
$entry_fullPathF4v = $webcam_basePath . '.f4v';
$entry_fullPathMp4 = $webcam_basePath . '.f4v.mp4';
if (file_exists($entry_fullPath)) {
// webcam should be preconvert until REALLY ready
$entry_status = entryStatus::READY;
$ext = "flv";
//echo "myInsertEtryHelper:: [$entry_fullPath]";
// for webcams that might have problmes with the metada - run the clipping even if $entry_from_time and $entry_to_time are null
if ($entry_to_time == 0) {
$entry_to_time = null;
}
// this will cause the clipper to reach the end of the file
// clip the webcam to some new file
$entry_fixedFullPath = $webcam_basePath . '_fixed.flv';
myFlvStaticHandler::fixRed5WebcamFlv($entry_fullPath, $entry_fixedFullPath);
$entry_newFullPath = $webcam_basePath . '_clipped.flv';
myFlvStaticHandler::clipToNewFile($entry_fixedFullPath, $entry_newFullPath, $entry_from_time, $entry_to_time);
$entry_fullPath = $entry_newFullPath;
// continue tracking the webcam
$te->setParam3Str($entry_fullPath);
$te->setDescription(__METHOD__ . ":" . __LINE__ . "::ENTRY_MEDIA_SOURCE_WEBCAM");
$duration = myFlvStaticHandler::getLastTimestamp($entry_fullPath);
} else {
if (file_exists($entry_fullPathF4v)) {
$entry_status = entryStatus::PRECONVERT;
$ext = 'f4v';
$entry_fullPath = $entry_fullPathF4v;
// continue tracking the webcam
$te->setParam3Str($entry_fullPath);
$te->setDescription(__METHOD__ . ":" . __LINE__ . "::ENTRY_MEDIA_SOURCE_WEBCAM");
} else {
if (file_exists($entry_fullPathMp4)) {
$entry_status = entryStatus::PRECONVERT;
$ext = 'mp4';
$entry_fullPath = $entry_fullPathMp4;
// continue tracking the webcam
$te->setParam3Str($entry_fullPath);
$te->setDescription(__METHOD__ . ":" . __LINE__ . "::ENTRY_MEDIA_SOURCE_WEBCAM");
} else {
KalturaLog::err("File [{$entry_fullPath}] does not exist");
$entry_status = entryStatus::ERROR_IMPORTING;
}
}
}
} else {
// if the url ends with .ext, we'll extract it this way
$urlext = strrchr($entry_url, '.');
// TODO: fix this patch
if (strlen($urlext) > 4) {
$urlext = '.jpg';
}
// if we got something wierd, assume we're downloading a jpg
$entry_fileName = $entry_data_prefix . $urlext;
KalturaLog::debug("handleEntry: media_type: {$media_type}");
if ($media_type == entry::ENTRY_MEDIA_TYPE_IMAGE) {
$duration = 0;
$entry_fullPath = $uploads . $entry_fileName;
if (!KCurlWrapper::getDataFromFile($entry_url, $entry_fullPath)) {
KalturaLog::debug("Failed downloading file[{$entry_url}]");
$entry_status = entryStatus::ERROR_IMPORTING;
}
// track images
$te->setParam3Str($entry_fullPath);
$te->setDescription(__METHOD__ . ":" . __LINE__ . "::ENTRY_MEDIA_SOURCE_URL:ENTRY_MEDIA_TYPE_IMAGE");
} else {
if ($media_type == entry::ENTRY_MEDIA_TYPE_VIDEO) {
//fixme - we can extract during import
$ext = "flv";
} else {
$ext = "mp3";
}
$entry_status = entryStatus::IMPORT;
// track images
$te->setParam3Str($ext);
$te->setDescription(__METHOD__ . ":" . __LINE__ . "::ENTRY_MEDIA_SOURCE_URL:ENTRY_MEDIA_TYPE_VIDEO");
}
}
}
if ($ext == null) {
$entry_fullPathTmp = $entry_fullPath;
$qpos = strpos($entry_fullPathTmp, "?");
if ($qpos !== false) {
$entry_fullPathTmp = substr($entry_fullPathTmp, 0, $qpos);
}
$ext = strtolower(pathinfo($entry_fullPathTmp, PATHINFO_EXTENSION));
}
// save the Trackentry
TrackEntry::addTrackEntry($te);
KalturaLog::debug("handleEntry: ext: {$ext}");
// We don't want to reject entries based on file extentions anumore
// Remarked by Tan-Tan
//
// if ($entry_status == entryStatus::PRECONVERT && !myContentStorage::fileExtNeedConversion($ext))
// {
//
// $this->errorMsg = "insertEntryAction Error - PRECONVERT file type not acceptable ($ext)";
// KalturaLog::debug("handleEntry: err: $this->errorMsg");
// if(is_null($entry) && $this->entry_id)
// {
// $entry = entryPeer::retrieveByPK($this->entry_id);
// }
// if($entry)
// {
// $entry->setStatus(entryStatus::ERROR_CONVERTING);
// $entry->save();
// }
// return false;
// }
$media_date = null;
// We don't want to reject entries based on file extentions anumore
// Remarked by Tan-Tan
//
// // if entry is ready, validate file type (webcam is an exception since we control the file type - flv)
// if ($entry_status == entryStatus::READY &&
// $media_source != entry::ENTRY_MEDIA_SOURCE_WEBCAM && !myContentStorage::fileExtAccepted($ext))
// {
// $this->errorMsg = "insertEntryAction Error - READY file type not acceptable ($ext)";
// KalturaLog::debug("handleEntry: err: $this->errorMsg");
// if(is_null($entry) && $this->entry_id)
// {
// $entry = entryPeer::retrieveByPK($this->entry_id);
// }
// if($entry)
// {
// $entry->setStatus(entryStatus::ERROR_CONVERTING);
// $entry->save();
// }
// return false;
// }
if ($entry_status == entryStatus::ERROR_IMPORTING) {
$need_thumb = false;
// we wont be needing a thumb for an errornous entry
KalturaLog::log("handleEntry: error importing, thumb not needed");
} else {
// thumbs are created by one of the following ways:
// 1. Image - images are already on disk for every selection method, so we can just create a thumb
// 2. Audio - no thumb is needed
// 3. Video -
// a. uploaded (file / webcam) - file is on disk and the user already selected a thumb
// b. imported - the source site had a thumbnail and we'll use it
$thumbTempPrefix = $uploads . $entry_data_prefix . '_thumbnail_';
$thumbBigFullPath = null;
$need_thumb = $type == entryType::MEDIA_CLIP;
KalturaLog::debug("handleEntry: handling media {$media_type}");
if ($media_type == entry::ENTRY_MEDIA_TYPE_IMAGE) {
// fetch media creation date
$exif_image_type = @exif_imagetype($entry_fullPath);
if ($exif_image_type == IMAGETYPE_JPEG || $exif_image_type == IMAGETYPE_TIFF_II || $exif_image_type == IMAGETYPE_TIFF_MM || $exif_image_type == IMAGETYPE_IFF || $exif_image_type == IMAGETYPE_PNG) {
$exif_data = @exif_read_data($entry_fullPath);
if ($exif_data && isset($exif_data["DateTimeOriginal"]) && $exif_data["DateTimeOriginal"]) {
$media_date = $exif_data["DateTimeOriginal"];
$ts = strtotime($media_date);
// handle invalid dates either due to bad format or out of range
if ($ts === -1 || $ts === false || $ts < strtotime('2000-01-01') || $ts > strtotime('2015-01-01')) {
$media_date = null;
}
}
}
// create thumb
$thumbFullPath = $thumbTempPrefix . '1.jpg';
$entry_thumbNum = 1;
$need_thumb = true;
//copy($entry_fullPath, $thumbFullPath);
myFileConverter::createImageThumbnail($entry_fullPath, $thumbFullPath, "image2");
//$thumbBigFullPath = $thumbFullPath; // no filesync for thumbnail of image
} else {
if ($media_type == entry::ENTRY_MEDIA_TYPE_VIDEO) {
if ($entry_status == entryStatus::IMPORT || $media_source == entry::ENTRY_MEDIA_SOURCE_URL) {
// import thumb and convert to our size
$thumbFullPath = $thumbTempPrefix . '1.jpg';
$entry_thumbNum = 1;
$importedThumbPath = $uploads . $entry_data_prefix . '_temp_thumb' . strrchr($entry_thumbUrl, '.');
if (KCurlWrapper::getDataFromFile($entry_thumbUrl, $importedThumbPath)) {
myFileConverter::createImageThumbnail($importedThumbPath, $thumbFullPath, "image2");
// set thumb as big thumb so fileSync will be created.
$thumbBigFullPath = $thumbFullPath;
} else {
$need_thumb = false;
}
} else {
if ($entry_thumbNum == 0) {
$entry_thumbNum = 1;
$thumbTime = 3;
if ($duration && $duration < $thumbTime * 1000) {
$thumbTime = floor($duration / 1000);
}
// for videos - thumbail should be created in post convert
// otherwise this code will fail if the thumbanil wasn't created successfully (roman)
//myFileConverter::autoCaptureFrame($entry_fullPath, $thumbTempPrefix."big_", $thumbTime, -1, -1);
$need_thumb = false;
$thumbBigFullPath = $thumbTempPrefix . "big_" . $entry_thumbNum . '.jpg';
}
}
//else select existing thumb ($entry_thumbNum already points to the right thumbnail)
}
}
$thumbFullPath = $thumbTempPrefix . $entry_thumbNum . '.jpg';
// if we arrived here both entry and thumbnail are valid we can now update the db
// in order to have the final entry_id and move its data to its final destination
if ($onlyExtractThumb) {
return $thumbFullPath;
}
}
$entry->setkshowId($this->kshow_id);
$entry->setKuserId($kuser_id);
$entry->setCreatorKuserId($kuser_id);
if ($this->partner_id != null) {
$entry->setPartnerId($this->partner_id);
$entry->setSubpId($this->subp_id);
}
$entry->setName($name ? $name : $this->getParam('entry_name'));
// $entry->setDescription('');//$this->getParam('entry_description'));
$entry->setType($type);
$entry->setMediaType($media_type);
$entry->setTags($tags ? $tags : $this->getParam('entry_tags'));
$entry->setSource($media_source);
$entry->setSourceId($this->getParam('entry_media_id'));
if ($media_date) {
$entry->setMediaDate($media_date);
}
// if source_link wasnt given use the entry_url HOWEVER, use it only if id doesnt contain @ which suggests the use of a password
$entry->setSourceLink($entry_source_link ? $entry_source_link : (strstr($entry_url, '@') ? "" : $entry_url));
if ($media_source == entry::ENTRY_MEDIA_SOURCE_FILE) {
$entry->setSourceLink("file:{$entry_fullPath}");
}
$entry->setLicenseType($this->getParam('entry_license'));
$entry->setCredit($this->getParam('entry_credit'));
$entry->setStatus($entry_status);
if ($entry->getCalculateDuration()) {
$entry->setLengthInMsecs($duration);
}
if ($this->entry_id == 0) {
$entry->save();
$this->entry_id = $entry->getId();
}
// move thumb to final destination and set db entry
if ($media_type != entry::ENTRY_MEDIA_TYPE_AUDIO && $entry_thumbNum && $need_thumb) {
KalturaLog::debug("handleEntry: saving none audio thumb [{$thumbBigFullPath}]");
$entry->setThumbnail('.jpg');
if ($thumbBigFullPath) {
if ($media_type != entry::ENTRY_MEDIA_TYPE_IMAGE) {
myFileConverter::convertImage($thumbBigFullPath, $thumbFullPath);
}
/*$thumbBigFinalPath = $content.$entry->getBigThumbnailPath();
myContentStorage::moveFile($thumbBigFullPath, $thumbBigFinalPath, true , $should_copy );
*/
$entryThumbKey = $entry->getSyncKey(entry::FILE_SYNC_ENTRY_SUB_TYPE_THUMB);
try {
if (!$should_copy) {
kFileSyncUtils::moveFromFile($thumbBigFullPath, $entryThumbKey);
} else {
kFileSyncUtils::copyFromFile($thumbBigFullPath, $entryThumbKey);
}
} catch (Exception $e) {
$entry->setStatus(entryStatus::ERROR_CONVERTING);
$entry->save();
throw $e;
}
}
}
// after extracting the thumb we can move the entry to its next destination
KalturaLog::debug("handleEntry: current status [" . $entry->getStatus() . "]");
// if needed a job will be submitted for importing external media sources
if ($entry->getStatus() == entryStatus::IMPORT) {
// changed by Tan-Tan, Nov 09 to support the new batch mechanism
kJobsManager::addImportJob(null, $this->entry_id, $this->partner_id, $entry_url);
// remarked by Tan-Tan
// $entry_fullPath = $content.'/content/imports/data/'.$this->entry_id.".".$ext;
// myContentStorage::fullMkdir($entry_fullPath);
//
// $batchClient = new myBatchUrlImportClient();
// $batchClient->addJob($this->entry_id, $entry_url, $entry_fullPath);
} else {
if ($entry->getStatus() == entryStatus::PRECONVERT) {
if (!$skip_conversion) {
// changed by Tan-Tan, Dec 09 to support the new batch mechanism
$flavorAsset = kFlowHelper::createOriginalFlavorAsset($this->partner_id, $this->entry_id);
if ($flavorAsset) {
$flavorAsset->setFileExt($ext);
$flavorAsset->save();
$syncKey = $flavorAsset->getSyncKey(flavorAsset::FILE_SYNC_FLAVOR_ASSET_SUB_TYPE_ASSET);
try {
kFileSyncUtils::moveFromFile($entry_fullPath, $syncKey);
} catch (Exception $e) {
$entry->setStatus(entryStatus::ERROR_CONVERTING);
$flavorAsset->setStatus(flavorAsset::FLAVOR_ASSET_STATUS_ERROR);
$entry->save();
$flavorAsset->save();
throw $e;
}
kEventsManager::raiseEvent(new kObjectAddedEvent($flavorAsset));
} else {
$entry->setStatus(entryStatus::ERROR_CONVERTING);
}
// Remarked by Tan-Tan
// $targetFileName = $this->entry_id.".".$ext;
// if ( false /* old conversion */)
// {
// // if we need to convert move entry to conversion directory
// $preConvPath = $content.'/content/preconvert/';
// myContentStorage::moveFile($entry_fullPath, $preConvPath."data/".$targetFileName, true , $should_copy );
//
// $signalFilePath = $preConvPath."files/".$targetFileName;
// myContentStorage::fullMkdir($signalFilePath);
// touch($signalFilePath);
// }
// else
// {
// $preConvPath = myContentStorage::getFSContentRootPath (). "/content/new_preconvert";
// $to_data = $preConvPath . "/$targetFileName" ;
// myContentStorage::moveFile($entry_fullPath, $to_data , true);
// touch ( $to_data . ".indicator" );
// }
}
} else {
if ($entry->getStatus() == entryStatus::PENDING || $media_source == entry::ENTRY_MEDIA_SOURCE_WEBCAM) {
$entry->setData($entry_fullPath);
$entry->save();
if ($media_type == entry::ENTRY_MEDIA_TYPE_VIDEO || $media_type == entry::ENTRY_MEDIA_TYPE_AUDIO) {
$flavorAsset = kFlowHelper::createOriginalFlavorAsset($this->partner_id, $this->entry_id);
if ($flavorAsset) {
$ext = pathinfo($entry_fullPath, PATHINFO_EXTENSION);
$flavorAsset->setFileExt($ext);
$flavorAsset->save();
$syncKey = $flavorAsset->getSyncKey(flavorAsset::FILE_SYNC_FLAVOR_ASSET_SUB_TYPE_ASSET);
try {
if (!$should_copy) {
kFileSyncUtils::moveFromFile($entry_fullPath, $syncKey);
} else {
// copy & create file sync from $entry_fullPath
kFileSyncUtils::copyFromFile($entry_fullPath, $syncKey);
}
} catch (Exception $e) {
$entry->setStatus(entryStatus::ERROR_CONVERTING);
$flavorAsset->setStatus(flavorAsset::FLAVOR_ASSET_STATUS_ERROR);
$entry->save();
$flavorAsset->save();
throw $e;
}
// // bypass to conversion
// kBusinessPreConvertDL::bypassConversion($flavorAsset, $entry);
/**
* if this is webcam entry, create mediaInfo for the source flavor asset synchronously
* since entry is ready right at the beginning
*/
if ($media_source == entry::ENTRY_MEDIA_SOURCE_WEBCAM) {
require_once SF_ROOT_DIR . DIRECTORY_SEPARATOR . ".." . DIRECTORY_SEPARATOR . "api_v3" . DIRECTORY_SEPARATOR . "bootstrap.php";
// extract file path
$sourceFileKey = $flavorAsset->getSyncKey(flavorAsset::FILE_SYNC_FLAVOR_ASSET_SUB_TYPE_ASSET);
$sourceFilePath = kFileSyncUtils::getLocalFilePathForKey($sourceFileKey);
// call mediaInfo for file
$mediaInfo = null;
try {
$mediaInfoParser = new KMediaInfoMediaParser($sourceFilePath, kConf::get('bin_path_mediainfo'));
$KalturaMediaInfo = $mediaInfoParser->getMediaInfo();
if ($KalturaMediaInfo) {
$mediaInfo = $KalturaMediaInfo->toInsertableObject($mediaInfo);
$mediaInfo->setFlavorAssetId($flavorAsset->getId());
$mediaInfo->save();
}
} catch (Exception $e) {
KalturaLog::err("Getting media info: " . $e->getMessage());
$mediaInfo = null;
}
// fix flavor asset according to mediainfo
if ($mediaInfo) {
KDLWrap::ConvertMediainfoCdl2FlavorAsset($mediaInfo, $flavorAsset);
$flavorTags = KDLWrap::CDLMediaInfo2Tags($mediaInfo, array(flavorParams::TAG_WEB));
$flavorAsset->setTags(implode(',', $flavorTags));
}
$flavorAsset->save();
}
kEventsManager::raiseEvent(new kObjectAddedEvent($flavorAsset));
$flavorAsset->setStatusLocalReady();
$flavorAsset->save();
} else {
$entry->setStatus(entryStatus::ERROR_IMPORTING);
}
} else {
if ($entry->getType() == entryType::DOCUMENT) {
//TODO: document should be handled by the plugin manager)
$flavorAsset = kFlowHelper::createOriginalFlavorAsset($this->partner_id, $this->entry_id);
if ($flavorAsset) {
$ext = pathinfo($entry_fullPath, PATHINFO_EXTENSION);
$flavorAsset->setFileExt($ext);
$flavorAsset->save();
$syncKey = $flavorAsset->getSyncKey(flavorAsset::FILE_SYNC_FLAVOR_ASSET_SUB_TYPE_ASSET);
try {
if (!$should_copy) {
kFileSyncUtils::moveFromFile($entry_fullPath, $syncKey);
} else {
// copy & create file sync from $entry_fullPath
kFileSyncUtils::copyFromFile($entry_fullPath, $syncKey);
}
} catch (Exception $e) {
$entry->setStatus(entryStatus::ERROR_CONVERTING);
$flavorAsset->setStatus(flavorAsset::FLAVOR_ASSET_STATUS_ERROR);
$entry->save();
$flavorAsset->save();
throw $e;
}
kEventsManager::raiseEvent(new kObjectAddedEvent($flavorAsset));
}
} else {
KalturaLog::debug("handleEntry: creating data file sync for file [{$entry_fullPath}]");
$entryDataKey = $entry->getSyncKey(entry::FILE_SYNC_ENTRY_SUB_TYPE_DATA);
if (!kFileSyncUtils::file_exists($entryDataKey)) {
try {
if (!$should_copy) {
kFileSyncUtils::moveFromFile($entry_fullPath, $entryDataKey);
} else {
// copy & create file sync from $entry_fullPath
kFileSyncUtils::copyFromFile($entry_fullPath, $entryDataKey);
}
} catch (Exception $e) {
$entry->setStatus(entryStatus::ERROR_CONVERTING);
$entry->save();
throw $e;
}
}
$entry->setStatus(entryStatus::READY);
$entry->save();
}
}
// Remarked by Tan-Tan, the flavor asset should be synced instead of the entry
//
// $entryDataKey = $entry->getSyncKey(entry::FILE_SYNC_ENTRY_SUB_TYPE_DATA);
// if(!$should_copy)
// {
// kFileSyncUtils::moveFromFile($entry_fullPath, $entryDataKey);
// }
// else
// {
// // copy & create file sync from $entry_fullPath
// kFileSyncUtils::copyFromFile($entry_fullPath, $entryDataKey);
// }
}
}
}
if ($entry->getStatus() == entryStatus::READY) {
$entry->updateDimensions();
}
$entry->save();
return true;
}
/**
* Server business-process case diagram
*
* @action serveDiagram
* @param KalturaEventNotificationEventObjectType $objectType
* @param string $objectId
* @param int $businessProcessStartNotificationTemplateId
* @return file
*
* @throws KalturaEventNotificationErrors::EVENT_NOTIFICATION_TEMPLATE_NOT_FOUND
* @throws KalturaBusinessProcessNotificationErrors::BUSINESS_PROCESS_CASE_NOT_FOUND
* @throws KalturaBusinessProcessNotificationErrors::BUSINESS_PROCESS_SERVER_NOT_FOUND
*/
public function serveDiagramAction($objectType, $objectId, $businessProcessStartNotificationTemplateId)
{
$dbObject = kEventNotificationFlowManager::getObject($objectType, $objectId);
if (!$dbObject) {
throw new KalturaAPIException(KalturaErrors::OBJECT_NOT_FOUND);
}
$dbTemplate = EventNotificationTemplatePeer::retrieveByPK($businessProcessStartNotificationTemplateId);
if (!$dbTemplate || !$dbTemplate instanceof BusinessProcessStartNotificationTemplate) {
throw new KalturaAPIException(KalturaEventNotificationErrors::EVENT_NOTIFICATION_TEMPLATE_NOT_FOUND, $businessProcessStartNotificationTemplateId);
}
$caseIds = $dbTemplate->getCaseIds($dbObject, false);
if (!count($caseIds)) {
throw new KalturaAPIException(KalturaBusinessProcessNotificationErrors::BUSINESS_PROCESS_CASE_NOT_FOUND);
}
$dbBusinessProcessServer = BusinessProcessServerPeer::retrieveByPK($dbTemplate->getServerId());
if (!$dbBusinessProcessServer) {
throw new KalturaAPIException(KalturaBusinessProcessNotificationErrors::BUSINESS_PROCESS_SERVER_NOT_FOUND, $dbTemplate->getServerId());
}
$businessProcessServer = KalturaBusinessProcessServer::getInstanceByType($dbBusinessProcessServer->getType());
$businessProcessServer->fromObject($dbBusinessProcessServer);
$provider = kBusinessProcessProvider::get($businessProcessServer);
$caseId = end($caseIds);
$filename = myContentStorage::getFSCacheRootPath() . 'bpm_diagram/bpm_';
$filename .= $objectId . '_';
$filename .= $businessProcessStartNotificationTemplateId . '_';
$filename .= $caseId . '.jpg';
$url = $provider->getCaseDiagram($caseId, $filename);
KCurlWrapper::getDataFromFile($url, $filename);
$mimeType = kFile::mimeType($filename);
return $this->dumpFile($filename, $mimeType);
}
/**
* @param string $str
* @return boolean
*/
protected function isUrl($str)
{
$str = KCurlWrapper::encodeUrl($str);
$strRegex = "^(?P<protocol>(https?)|(ftp)|(sftp)|(scp)):\\/\\/?" . "(?P<credentials>([ 0-9a-zA-Z_!~*'().&=+\$%-\\[\\]]+:)?[ 0-9a-zA-Z_!~*'().&=+\$%-\\[\\]]+@)?" . "(?P<domain>([0-9]{1,3}\\.){3}[0-9]{1,3}" . "|" . "(?P<tertiary_domain>[0-9a-zA-Z_!~*'()-]+[.])*" . "(?P<second_level_domain>([0-9a-zA-Z][0-9a-zA-Z-]{0,61})?[0-9a-zA-Z])\\." . "(?P<first_level_domain>[a-zA-Z]{2,6}))" . "([.](?P<additional_level_domain>[a-zA-Z]{2,6}))*" . "(?P<port>:[0-9]{1,4})?" . "(?P<path>(\\/?)|" . "(\\/[ 0-9a-zA-Z_!~*'().;?:@&=+\$,%#-\\[\\]]+)+)\$";
return preg_match("/{$strRegex}/i", $str);
}
/**
* @return false|KCurlHeaderResponse
*/
public function getHeader($sourceUrl, $noBody = false)
{
curl_setopt($this->ch, CURLOPT_HEADER, true);
curl_setopt($this->ch, CURLOPT_BINARYTRANSFER, true);
curl_setopt($this->ch, CURLOPT_HEADERFUNCTION, 'KCurlWrapper::read_header');
curl_setopt($this->ch, CURLOPT_WRITEFUNCTION, 'KCurlWrapper::read_body');
$this->setSourceUrlAndprotocol($sourceUrl);
if ($this->protocol == self::HTTP_PROTOCOL_FTP) {
$noBody = true;
}
if ($noBody) {
curl_setopt($this->ch, CURLOPT_NOBODY, true);
} else {
curl_setopt($this->ch, CURLOPT_RANGE, '0-0');
}
self::$headers = "";
self::$lastHeader = false;
curl_exec($this->ch);
//Added to support multiple curl executions using the same curl. Wince this is the same curl re-used we need to reset the range option before continuing forward
if (!$noBody) {
curl_setopt($this->ch, CURLOPT_RANGE, '0-');
}
if (!self::$headers) {
return false;
}
self::$headers = explode("\r\n", self::$headers);
$curlHeaderResponse = new KCurlHeaderResponse();
if ($this->protocol == self::HTTP_PROTOCOL_HTTP) {
$header = reset(self::$headers);
// this line is true if the protocol is HTTP (or HTTPS);
$matches = null;
if (preg_match('/HTTP\\/?[\\d.]{0,3} ([\\d]{3}) ([^\\n\\r]+)/', $header, $matches)) {
$curlHeaderResponse->code = $matches[1];
$curlHeaderResponse->codeName = $matches[2];
}
foreach (self::$headers as $header) {
if (!strstr($header, ':')) {
if (preg_match('/HTTP\\/?[\\d.]{0,3} ([\\d]{3}) (.+)/', $header, $matches)) {
$curlHeaderResponse->code = $matches[1];
$curlHeaderResponse->codeName = $matches[2];
}
continue;
}
list($name, $value) = explode(':', $header, 2);
$curlHeaderResponse->headers[trim(strtolower($name))] = trim($value);
}
if (!$noBody) {
$matches = null;
if (isset($curlHeaderResponse->headers['content-range']) && preg_match('/0-0\\/([\\d]+)$/', $curlHeaderResponse->headers['content-range'], $matches)) {
$curlHeaderResponse->headers['content-length'] = $matches[1];
} else {
return $this->getHeader($sourceUrl, true);
}
}
} else {
// for now - assume FTP
foreach (self::$headers as $header) {
$headerParts = explode(':', $header, 2);
if (count($headerParts) < 2) {
continue;
}
list($name, $value) = $headerParts;
$curlHeaderResponse->headers[trim(strtolower($name))] = trim($value);
}
// if this is a good ftp url - there will be a content-length header
$length = @$curlHeaderResponse->headers["content-length"];
if ($length > 0) {
// this is equivalent to a good HTTP request
$curlHeaderResponse->code = KCurlHeaderResponse::HTTP_STATUS_OK;
$curlHeaderResponse->codeName = "OK";
} else {
if (isset($curlHeaderResponse->headers["curl"])) {
// example: curl: (10) the username and/or the password are incorrect
// in this case set the error code to unknown error and use the whole string as the description
$curlHeaderResponse->code = -1;
// unknown error
$curlHeaderResponse->codeName = "curl: " . $curlHeaderResponse->headers["curl"];
} else {
// example: curl: (10) the username and/or the password are incorrect
// in this case set the error code to unknown error and use the whole string as the description
$curlHeaderResponse->code = -1;
// unknown error
$curlHeaderResponse->codeName = "Unknown FTP error";
}
}
}
curl_setopt($this->ch, CURLOPT_HEADERFUNCTION, 'KCurlWrapper::read_header_do_nothing');
curl_setopt($this->ch, CURLOPT_WRITEFUNCTION, 'KCurlWrapper::read_body_do_nothing');
return $curlHeaderResponse;
}
/**
* Fetches the header for the given $url and closes the job on any errors
* @param KalturaBatchJob $job
* @param string $url
* @return false|KCurlHeaderResponse
*/
private function fetchHeader(KalturaBatchJob &$job, $url)
{
KalturaLog::debug('Fetching header for [' . $url . ']');
$this->updateJob($job, 'Downloading header for [' . $url . ']', KalturaBatchJobStatus::PROCESSING);
// fetch the http headers
$curlWrapper = new KCurlWrapper($url);
$curlHeaderResponse = $curlWrapper->getHeader();
$curlError = $curlWrapper->getError();
$curlWrapper->close();
if (!$curlHeaderResponse || $curlError) {
// error fetching headers
$msg = 'Error: ' . $curlWrapper->getError();
KalturaLog::err($msg);
$this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $curlWrapper->getErrorNumber(), $msg, KalturaBatchJobStatus::FAILED);
return false;
}
if (!$curlHeaderResponse->isGoodCode()) {
// some error exists in the response
$msg = 'HTTP Error: ' . $curlHeaderResponse->code . ' ' . $curlHeaderResponse->codeName;
KalturaLog::err($msg);
$this->closeJob($job, KalturaBatchJobErrorTypes::HTTP, $curlHeaderResponse->code, $msg, KalturaBatchJobStatus::FAILED);
return false;
}
// header fetched successfully - return it
return $curlHeaderResponse;
}
/**
* Validates that the xml is valid using the XSD
*@return bool - if the validation is ok
*/
protected function validate()
{
if (!file_exists($this->data->filePath)) {
throw new KalturaBatchException("File doesn't exist [{$this->data->filePath}]", KalturaBatchJobAppErrors::BULK_FILE_NOT_FOUND);
}
libxml_use_internal_errors(true);
$this->loadXslt();
$xdoc = new KDOMDocument();
$this->xslTransformedContent = $this->xslTransform($this->data->filePath);
KalturaLog::info("Tranformed content: " . $this->xslTransformedContent);
libxml_clear_errors();
if (!$xdoc->loadXML($this->xslTransformedContent)) {
$errorMessage = kXml::getLibXmlErrorDescription($this->xslTransformedContent);
KalturaLog::debug("Could not load xml");
throw new KalturaBatchException("Could not load xml [{$this->job->id}], {$errorMessage}", KalturaBatchJobAppErrors::BULK_VALIDATION_FAILED);
}
//Validate the XML file against the schema
libxml_clear_errors();
$xsdURL = KBatchBase::$kClient->schema->serve($this->getSchemaType());
if (KBatchBase::$taskConfig->params->xmlSchemaVersion) {
$xsdURL .= "&version=" . KBatchBase::$taskConfig->params->xmlSchemaVersion;
}
$xsd = KCurlWrapper::getContent($xsdURL);
if (!$xdoc->schemaValidateSource($xsd)) {
$errorMessage = kXml::getLibXmlErrorDescription($this->xslTransformedContent);
KalturaLog::debug("XML is invalid:\n{$errorMessage}");
throw new KalturaBatchException("Validate files failed on job [{$this->job->id}], {$errorMessage}", KalturaBatchJobAppErrors::BULK_VALIDATION_FAILED);
}
return true;
}
private function fetchFile(KalturaBatchJob $job, KalturaImportJobData $data)
{
KalturaLog::debug("fetchFile({$job->id})");
try {
$sourceUrl = $data->srcFileUrl;
KalturaLog::debug("sourceUrl [{$sourceUrl}]");
$this->updateJob($job, 'Downloading file header', KalturaBatchJobStatus::QUEUED, 1);
$curlWrapper = new KCurlWrapper($sourceUrl);
$useNoBody = $job->executionAttempts > 1;
// if the process crashed first time, tries with no body instead of range 0-0
$curlHeaderResponse = $curlWrapper->getHeader($useNoBody);
if (!$curlHeaderResponse || $curlWrapper->getError()) {
$this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $curlWrapper->getErrorNumber(), "Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::FAILED);
return $job;
}
if (!$curlHeaderResponse->isGoodCode()) {
$this->closeJob($job, KalturaBatchJobErrorTypes::HTTP, $curlHeaderResponse->code, "HTTP Error: " . $curlHeaderResponse->code . " " . $curlHeaderResponse->codeName, KalturaBatchJobStatus::FAILED);
return $job;
}
$fileSize = null;
if (isset($curlHeaderResponse->headers['content-length'])) {
$fileSize = $curlHeaderResponse->headers['content-length'];
}
$curlWrapper->close();
$resumeOffset = 0;
if ($fileSize && $data->destFileLocalPath && file_exists($data->destFileLocalPath)) {
clearstatcache();
$actualFileSize = filesize($data->destFileLocalPath);
if ($actualFileSize >= $fileSize) {
return $this->moveFile($job, $data->destFileLocalPath, $fileSize);
} else {
$resumeOffset = $actualFileSize;
}
}
$curlWrapper = new KCurlWrapper($sourceUrl);
$curlWrapper->setTimeout($this->taskConfig->params->curlTimeout);
if ($resumeOffset) {
$curlWrapper->setResumeOffset($resumeOffset);
} else {
// creates a temp file path
$rootPath = $this->taskConfig->params->localTempPath;
$res = self::createDir($rootPath);
if (!$res) {
KalturaLog::err("Cannot continue import without temp directory");
die;
}
$uniqid = uniqid('import_');
$destFile = realpath($rootPath) . "/{$uniqid}";
KalturaLog::debug("destFile [{$destFile}]");
// in case the url has added arguments, remove them (and reveal the URL path)
// in order to find the file extension
$urlPathEndIndex = strpos($sourceUrl, "?");
if ($urlPathEndIndex !== false) {
$sourceUrlPath = substr($sourceUrl, 0, $urlPathEndIndex);
} else {
$sourceUrlPath = $sourceUrl;
}
$ext = pathinfo($sourceUrlPath, PATHINFO_EXTENSION);
if (strlen($ext)) {
$destFile .= ".{$ext}";
}
$data->destFileLocalPath = $destFile;
$this->updateJob($job, "Downloading file, size: {$fileSize}", KalturaBatchJobStatus::PROCESSING, 2, $data);
}
KalturaLog::debug("Executing curl");
$res = $curlWrapper->exec($data->destFileLocalPath);
KalturaLog::debug("Curl results: {$res}");
if (!$res || $curlWrapper->getError()) {
$errNumber = $curlWrapper->getErrorNumber();
if ($errNumber != CURLE_OPERATION_TIMEOUTED) {
$this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $errNumber, "Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::RETRY);
$curlWrapper->close();
return $job;
} else {
clearstatcache();
$actualFileSize = filesize($data->destFileLocalPath);
if ($actualFileSize == $resumeOffset) {
$this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $errNumber, "Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::RETRY);
$curlWrapper->close();
return $job;
}
}
}
$curlWrapper->close();
if (!file_exists($data->destFileLocalPath)) {
$this->closeJob($job, KalturaBatchJobErrorTypes::APP, KalturaBatchJobAppErrors::OUTPUT_FILE_DOESNT_EXIST, "Error: output file doesn't exist", KalturaBatchJobStatus::RETRY);
return $job;
}
// check the file size only if its first or second retry
// in case it failed few times, taks the file as is
if ($fileSize) {
clearstatcache();
$actualFileSize = filesize($data->destFileLocalPath);
if ($actualFileSize < $fileSize) {
$percent = floor($actualFileSize * 100 / $fileSize);
$this->updateJob($job, "Downloaded size: {$actualFileSize}({$percent}%)", KalturaBatchJobStatus::PROCESSING, $percent, $data);
$this->kClient->batch->resetJobExecutionAttempts($job->id, $this->getExclusiveLockKey(), $job->jobType);
// $this->closeJob($job, KalturaBatchJobErrorTypes::APP, KalturaBatchJobAppErrors::OUTPUT_FILE_WRONG_SIZE, "Expected file size[$fileSize] actual file size[$actualFileSize]", KalturaBatchJobStatus::RETRY);
return $job;
}
}
$this->updateJob($job, 'File imported, copy to shared folder', KalturaBatchJobStatus::PROCESSED, 90);
$job = $this->moveFile($job, $data->destFileLocalPath, $fileSize);
} catch (Exception $ex) {
$this->closeJob($job, KalturaBatchJobErrorTypes::RUNTIME, $ex->getCode(), "Error: " . $ex->getMessage(), KalturaBatchJobStatus::FAILED);
}
return $job;
}
/**
* @param string $str
* @return boolean
*/
protected function isUrl($str)
{
$str = KCurlWrapper::encodeUrl($str);
$redundant_url_chars = array("_");
$str = str_replace($redundant_url_chars, "", $str);
return filter_var($str, FILTER_VALIDATE_URL);
}
/**
* @param thumbAsset $thumbAsset
* @param string $url
*/
protected function attachUrl(thumbAsset $thumbAsset, $url)
{
$fullPath = myContentStorage::getFSUploadsPath() . '/' . $thumbAsset->getId() . '.jpg';
if (KCurlWrapper::getDataFromFile($url, $fullPath)) {
return $this->attachFile($thumbAsset, $fullPath);
}
if ($thumbAsset->getStatus() == thumbAsset::ASSET_STATUS_QUEUED || $thumbAsset->getStatus() == thumbAsset::ASSET_STATUS_NOT_APPLICABLE) {
$thumbAsset->setDescription("Failed downloading file[{$url}]");
$thumbAsset->setStatus(thumbAsset::ASSET_STATUS_ERROR);
$thumbAsset->save();
}
throw new KalturaAPIException(KalturaErrors::THUMB_ASSET_DOWNLOAD_FAILED, $url);
}
public static function handleImportContent($curlInfo, $importData, $params)
{
if (!($curlInfo->headers['content-length'] < 16000 && $curlInfo->headers['content-type'] == 'text/html')) {
return $importData;
}
KalturaLog::debug('content-length [' . $curlInfo->headers['content-length'] . '] content-type [' . $curlInfo->headers['content-type'] . ']');
KalturaLog::info('Handle Import data: Webex Plugin');
$matches = null;
$recordId = null;
if (isset($curlInfo->headers['set-cookie'])) {
$recordId = $curlInfo->getCookieValue($curlInfo->headers['set-cookie'], 'recordId');
if ($recordId == null) {
throw new Exception('recordId value not found');
}
} else {
throw new Exception('set-cookie was not found in header');
}
$data = file_get_contents($importData->destFileLocalPath);
KalturaLog::info("data:\n\n{$data}\n\n");
if (!preg_match("/href='([^']+)';/", $data, $matches)) {
throw new Exception('Starting URL not found');
}
$url2 = $matches[1];
$curlWrapper = new KCurlWrapper();
$curlWrapper->setOpt(CURLOPT_COOKIE, 'DetectionBrowserStatus=3|1|32|1|11|2;' . $curlInfo->headers["set-cookie"]);
$result = $curlWrapper->exec($url2);
KalturaLog::info("result:\n\n{$result}\n\n");
if (!preg_match("/var prepareTicket = '([^']+)';/", $result, $matches)) {
throw new Exception('prepareTicket parameter not found');
}
$prepareTicket = $matches[1];
if (!preg_match('/function (download\\(\\).+prepareTicket;)/s', $result, $matches)) {
throw new Exception('download function not found');
}
if (!preg_match('/http.+prepareTicket/', $matches[0], $matches)) {
throw new Exception('prepareTicket URL not found');
}
$url3 = $matches[0];
$url3 = str_replace(array('"', ' ', '+', 'recordId', 'prepareTicket=prepareTicket'), array('', '', '', $recordId, "prepareTicket={$prepareTicket}"), $url3);
if (!preg_match("/var downloadUrl = '(http[^']+)' \\+ ticket;/", $result, $matches)) {
throw new Exception('Download URL not found');
}
$url4 = $matches[1];
$status = null;
$iterations = isset($params->webex->iterations) && !is_null($params->webex->iterations) ? intval($params->webex->iterations) : 10;
$sleep = isset($params->webex->sleep) && !is_null($params->webex->sleep) ? intval($params->webex->sleep) : 3;
for ($i = 0; $i < $iterations; $i++) {
$result = $curlWrapper->exec($url3);
KalturaLog::info("result ({$i}):\n\n{$result}\n\n");
if (!preg_match("/window\\.parent\\.func_prepare\\('([^']+)','([^']*)','([^']*)'\\);/", $result, $matches)) {
KalturaLog::err("Invalid result returned for prepareTicket request - should contain call to the func_prepare method\n {$result}");
throw new Exception('Invalid result: func_prepare function not found');
}
$status = $matches[1];
if ($status == 'OKOK') {
break;
}
sleep($sleep);
}
if ($status != 'OKOK') {
KalturaLog::info("Invalid result returned for prepareTicket request. Last result:\n " . $result);
throw new kTemporaryException('Invalid result returned for prepareTicket request');
}
$ticket = $matches[3];
$url4 .= $ticket;
$curlWrapper->setOpt(CURLOPT_RETURNTRANSFER, false);
$fileName = pathinfo($importData->destFileLocalPath, PATHINFO_FILENAME);
$destFileLocalPath = preg_replace("/{$fileName}\\.[\\w\\d]+/", "{$fileName}.arf", $importData->destFileLocalPath);
$importData->destFileLocalPath = $destFileLocalPath;
KalturaLog::info('destination: ' . $importData->destFileLocalPath);
$result = $curlWrapper->exec($url4, $importData->destFileLocalPath);
if (!$result) {
$code = $curlWrapper->getErrorNumber();
$message = $curlWrapper->getError();
throw new Exception($message, $code);
}
$curlWrapper->close();
$importData->fileSize = kFile::fileSize($importData->destFileLocalPath);
return $importData;
}
public function testRedirectFileSize()
{
$curlWrapper = new KCurlWrapper('http://scoregolf.com/Web-Media/Video/2008/May/pumafinal.flv');
$curlHeaderResponse = $curlWrapper->getHeader();
var_dump($curlHeaderResponse);
}
/**
* Fetch the manifest and build all flavors array
* @param string $url
*/
private function buildM3u8Flavors($url, array &$flavors)
{
$this->finalizeUrls($url, $flavors);
$manifest = KCurlWrapper::getContent($url);
if (!$manifest) {
return;
}
$manifestLines = explode("\n", $manifest);
$manifestLine = reset($manifestLines);
while ($manifestLine) {
$lineParts = explode(':', $manifestLine, 2);
if ($lineParts[0] === '#EXT-X-STREAM-INF') {
// passing the url as urlPrefix so that only the path will be tokenized
$flavor = array('url' => '', 'urlPrefix' => requestUtils::resolve(next($manifestLines), $url), 'ext' => 'm3u8');
$attributes = explode(',', $lineParts[1]);
foreach ($attributes as $attribute) {
$attributeParts = explode('=', $attribute, 2);
switch ($attributeParts[0]) {
case 'BANDWIDTH':
$flavor['bitrate'] = $attributeParts[1] / 1024;
break;
case 'RESOLUTION':
list($flavor['width'], $flavor['height']) = explode('x', $attributeParts[1], 2);
break;
}
}
$flavors[] = $flavor;
}
$manifestLine = next($manifestLines);
}
}
private function fetchFile(KalturaBatchJob $job, KalturaImportJobData $data)
{
$jobSubType = $job->jobSubType;
$sshProtocols = array(kFileTransferMgrType::SCP, kFileTransferMgrType::SFTP);
if (in_array($jobSubType, $sshProtocols)) {
// use SSH file transfer manager for SFTP/SCP
return $this->fetchFileSsh($job, $data);
}
try {
$sourceUrl = $data->srcFileUrl;
$this->updateJob($job, 'Downloading file header', KalturaBatchJobStatus::QUEUED);
$fileSize = null;
$resumeOffset = 0;
if ($data->destFileLocalPath && file_exists($data->destFileLocalPath)) {
$curlWrapper = new KCurlWrapper(self::$taskConfig->params);
$useNoBody = $job->executionAttempts > 1;
// if the process crashed first time, tries with no body instead of range 0-0
$curlHeaderResponse = $curlWrapper->getHeader($sourceUrl, $useNoBody);
if (!$curlHeaderResponse || !count($curlHeaderResponse->headers)) {
$this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $curlWrapper->getErrorNumber(), "Couldn't read file. Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::FAILED);
return $job;
}
if ($curlWrapper->getError()) {
KalturaLog::err("Headers error: " . $curlWrapper->getError());
KalturaLog::err("Headers error number: " . $curlWrapper->getErrorNumber());
$curlWrapper->close();
$curlWrapper = new KCurlWrapper(self::$taskConfig->params);
}
if (!$curlHeaderResponse->isGoodCode()) {
$this->closeJob($job, KalturaBatchJobErrorTypes::HTTP, $curlHeaderResponse->code, "Failed while reading file. HTTP Error: " . $curlHeaderResponse->code . " " . $curlHeaderResponse->codeName, KalturaBatchJobStatus::FAILED);
$curlWrapper->close();
return $job;
}
if (isset($curlHeaderResponse->headers['content-length'])) {
$fileSize = $curlHeaderResponse->headers['content-length'];
}
$curlWrapper->close();
if ($fileSize) {
clearstatcache();
$actualFileSize = kFile::fileSize($data->destFileLocalPath);
if ($actualFileSize >= $fileSize) {
return $this->moveFile($job, $data->destFileLocalPath, $fileSize);
} else {
$resumeOffset = $actualFileSize;
}
}
}
$curlWrapper = new KCurlWrapper(self::$taskConfig->params);
if (is_null($fileSize)) {
// Read file size
$curlHeaderResponse = $curlWrapper->getHeader($sourceUrl, true);
if ($curlHeaderResponse && count($curlHeaderResponse->headers) && !$curlWrapper->getError() && isset($curlHeaderResponse->headers['content-length'])) {
$fileSize = $curlHeaderResponse->headers['content-length'];
}
//Close the curl used to fetch the header and create a new one.
//When fetching headers we set curl options that than are not reset once header is fetched.
//Not all servers support all the options so we need to remove them from our headers.
$curlWrapper->close();
$curlWrapper = new KCurlWrapper(self::$taskConfig->params);
}
if ($resumeOffset) {
$curlWrapper->setResumeOffset($resumeOffset);
} else {
// creates a temp file path
$destFile = $this->getTempFilePath($sourceUrl);
KalturaLog::debug("destFile [{$destFile}]");
$data->destFileLocalPath = $destFile;
$data->fileSize = is_null($fileSize) ? -1 : $fileSize;
$this->updateJob($job, "Downloading file, size: {$fileSize}", KalturaBatchJobStatus::PROCESSING, $data);
}
$res = $curlWrapper->exec($sourceUrl, $data->destFileLocalPath);
KalturaLog::debug("Curl results: {$res}");
if (!$res || $curlWrapper->getError()) {
$errNumber = $curlWrapper->getErrorNumber();
if ($errNumber != CURLE_OPERATION_TIMEOUTED) {
$this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $errNumber, "Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::RETRY);
$curlWrapper->close();
return $job;
} else {
clearstatcache();
$actualFileSize = kFile::fileSize($data->destFileLocalPath);
if ($actualFileSize == $resumeOffset) {
$this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $errNumber, "No new information. Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::RETRY);
$curlWrapper->close();
return $job;
}
if (!$fileSize) {
$this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $errNumber, "Received timeout, but no filesize available. Completed size [{$actualFileSize}]" . $curlWrapper->getError(), KalturaBatchJobStatus::RETRY);
$curlWrapper->close();
return $job;
}
}
}
$curlWrapper->close();
if (!file_exists($data->destFileLocalPath)) {
$this->closeJob($job, KalturaBatchJobErrorTypes::APP, KalturaBatchJobAppErrors::OUTPUT_FILE_DOESNT_EXIST, "Error: output file doesn't exist", KalturaBatchJobStatus::RETRY);
return $job;
}
// check the file size only if its first or second retry
// in case it failed few times, taks the file as is
if ($fileSize) {
clearstatcache();
$actualFileSize = kFile::fileSize($data->destFileLocalPath);
if ($actualFileSize < $fileSize) {
$percent = floor($actualFileSize * 100 / $fileSize);
$this->updateJob($job, "Downloaded size: {$actualFileSize}({$percent}%)", KalturaBatchJobStatus::PROCESSING, $data);
self::$kClient->batch->resetJobExecutionAttempts($job->id, $this->getExclusiveLockKey(), $job->jobType);
// $this->closeJob($job, KalturaBatchJobErrorTypes::APP, KalturaBatchJobAppErrors::OUTPUT_FILE_WRONG_SIZE, "Expected file size[$fileSize] actual file size[$actualFileSize]", KalturaBatchJobStatus::RETRY);
return $job;
}
KalturaLog::info("headers " . print_r($curlHeaderResponse, true));
$pluginInstances = KalturaPluginManager::getPluginInstances('IKalturaImportHandler');
foreach ($pluginInstances as $pluginInstance) {
/* @var $pluginInstance IKalturaImportHandler */
$data = $pluginInstance->handleImportContent($curlHeaderResponse, $data, KBatchBase::$taskConfig->params);
}
}
$this->updateJob($job, 'File imported, copy to shared folder', KalturaBatchJobStatus::PROCESSED);
$job = $this->moveFile($job, $data->destFileLocalPath);
} catch (kTemporaryException $tex) {
$data->destFileLocalPath = KalturaClient::getKalturaNullValue();
$tex->setData($data);
throw $tex;
} catch (Exception $ex) {
$data->destFileLocalPath = KalturaClient::getKalturaNullValue();
if ($ex->getMessage() == KCurlWrapper::COULD_NOT_CONNECT_TO_HOST_ERROR) {
throw new kTemporaryException($ex->getMessage(), $ex->getCode(), $data);
}
$this->closeJob($job, KalturaBatchJobErrorTypes::RUNTIME, $ex->getCode(), "Error: " . $ex->getMessage(), KalturaBatchJobStatus::FAILED, $data);
}
return $job;
}
/**
* @param BatchJob $parentJob
* @param int $partnerId
* @param string $entryId
* @param string $thumbAssetId
* @param FileSyncKey $srcSyncKey
* @param string $srcAssetId
* @param int $srcAssetType enum of assetType
* @param thumbParamsOutput $thumbParams
* @return BatchJob
*/
public static function addCapturaThumbJob(BatchJob $parentJob = null, $partnerId, $entryId, $thumbAssetId, FileSyncKey $srcSyncKey, $srcAssetId, $srcAssetType, thumbParamsOutput $thumbParams = null)
{
$thumbAsset = assetPeer::retrieveById($thumbAssetId);
if (!$thumbAsset) {
KalturaLog::err("No thumbnail asset found for id [{$thumbAssetId}]");
return null;
}
$partner = PartnerPeer::retrieveByPK($thumbAsset->getPartnerId());
list($fileSync, $local) = kFileSyncUtils::getReadyFileSyncForKey($srcSyncKey, true, false);
if (!$fileSync) {
$thumbAsset->setStatus(asset::ASSET_STATUS_ERROR);
$thumbAsset->setDescription("Source file sync not found: {$srcSyncKey}");
$thumbAsset->save();
KalturaLog::err("Source file sync not found: {$srcSyncKey}");
return null;
}
if (!$local) {
if ($fileSync->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_URL && $partner && $partner->getImportRemoteSourceForConvert()) {
$url = $fileSync->getExternalUrl($entryId);
$originalAsset = kFileSyncUtils::retrieveObjectForSyncKey($srcSyncKey);
if ($originalAsset instanceof flavorAsset) {
KalturaLog::debug("Creates import job for remote file sync [{$url}]");
if ($thumbParams) {
$thumbParams->setSourceParamsId($originalAsset->getFlavorParamsId());
$thumbParams->save();
}
$thumbAsset->setStatus(asset::ASSET_STATUS_WAIT_FOR_CONVERT);
$thumbAsset->setDescription("Source file sync is importing: {$srcSyncKey}");
$thumbAsset->save();
return kJobsManager::addImportJob($parentJob, $thumbAsset->getEntryId(), $partner->getId(), $url, $originalAsset, null, null, true);
}
KalturaLog::debug("Downloading remote file sync [{$url}]");
$downloadPath = myContentStorage::getFSUploadsPath() . '/' . $thumbAsset->getId() . '.jpg';
if (KCurlWrapper::getDataFromFile($url, $downloadPath)) {
kFileSyncUtils::moveFromFile($downloadPath, $srcSyncKey);
list($fileSync, $local) = kFileSyncUtils::getReadyFileSyncForKey($srcSyncKey, false, false);
if (!$fileSync) {
throw new kCoreException("Source file not found for thumbnail capture [{$thumbAssetId}]", kCoreException::SOURCE_FILE_NOT_FOUND);
}
}
} else {
throw new kCoreException("Source file not found for thumbnail capture [{$thumbAssetId}]", kCoreException::SOURCE_FILE_NOT_FOUND);
}
}
$localPath = $fileSync->getFullPath();
$remoteUrl = $fileSync->getExternalUrl($entryId);
// creates convert data
$data = new kCaptureThumbJobData();
$data->setThumbAssetId($thumbAssetId);
$data->setSrcAssetId($srcAssetId);
$data->setSrcAssetType($srcAssetType);
$data->setSrcFileSyncLocalPath($localPath);
$data->setSrcFileSyncRemoteUrl($remoteUrl);
$data->setThumbParamsOutputId($thumbParams->getId());
$batchJob = null;
if ($parentJob) {
$batchJob = $parentJob->createChild(BatchJobType::CAPTURE_THUMB);
} else {
$batchJob = new BatchJob();
$batchJob->setEntryId($entryId);
$batchJob->setPartnerId($partnerId);
}
$batchJob->setObjectId($thumbAssetId);
$batchJob->setObjectType(BatchJobObjectType::ASSET);
return kJobsManager::addJob($batchJob, $data, BatchJobType::CAPTURE_THUMB);
}
public function __construct($script_name)
{
$this->script_name = $script_name;
$this->register($script_name);
SET_CONTEXT("FS");
$MAX_ITERATIONS_DUE_TO_PROPEL_MEMORY_LEAK = 10000000;
self::initDb();
list($sleep_between_cycles, $number_of_times_to_skip_writing_sleeping) = self::getSleepParams('app_flatten_');
$last_worker_count = 0;
$iteration = 0;
$c = new Criteria();
$currentDc = kDataCenterMgr::getCurrentDc();
$c->add(BatchJobPeer::DC, kDataCenterMgr::getCurrentDcId());
$c->add(BatchJobPeer::JOB_TYPE, BatchJobType::FLATTEN);
$c->add(BatchJobPeer::STATUS, BatchJob::BATCHJOB_STATUS_PROCESSED);
$temp_count = 0;
while (1) {
self::exitIfDone();
try {
sleep($sleep_between_cycles);
$jobs = BatchJobPeer::doSelect($c);
foreach ($jobs as $job) {
$data = json_decode($job->getData(true), true);
$entry_id = $data['entryId'];
$entry_int_id = $data['entryIntId'];
$entry_version = $data['entryVersion'];
$file_format = $data['fileFormat'];
$entry = entryPeer::retrieveByPK($entry_id);
if (!$entry) {
// entry is probably deleted if it is not returned from retrieveByPK
// close job as failed
$job->setStatus(BatchJob::BATCHJOB_STATUS_FAILED);
$job->setDescription("could not retrieve entry, probably deleted");
KalturaLog::debug("could not retrieve entry {$entry_id} , probably deleted");
$job->save();
continue;
}
$fileSyncKey = $entry->getSyncKey(entry::FILE_SYNC_ENTRY_SUB_TYPE_DOWNLOAD, $file_format);
$fullFinalPath = kFileSyncUtils::getLocalFilePathForKey($fileSyncKey);
$finalPathNoExt = substr($fullFinalPath, 0, strlen($fullFinalPath) - strlen($file_format));
kFile::fullMkdir($fullFinalPath);
$wildcardFinalPath = $finalPathNoExt . "*";
$older_files = glob($wildcardFinalPath);
foreach ($older_files as $older_file) {
KalturaLog::debug("removing old file: [{$older_file}]");
@unlink($older_file);
}
KalturaLog::debug("Downloading: {$fullFinalPath}");
KCurlWrapper::getDataFromFile($data["serverUrl"], $fullFinalPath);
if (!file_exists($fullFinalPath)) {
KalturaLog::debug("file doesnt exist: " . $data["serverUrl"]);
$job->setDescription("file doesnt exist: " . $data["serverUrl"]);
$job->setStatus(BatchJob::BATCHJOB_STATUS_FAILED);
} else {
if (filesize($fullFinalPath) < 100000) {
@unlink($fullFinalPath);
KalturaLog::debug("file too small: " . $data["serverUrl"]);
$job->setDescription("file too small: " . $data["serverUrl"]);
$job->setStatus(BatchJob::BATCHJOB_STATUS_FAILED);
} else {
if ($data['email']) {
$downloadLink = $entry->getDownloadUrl() . '/format/' . $file_format;
kJobsManager::addMailJob(null, $entry_id, $entry->getPartnerId(), self::KALTURAS_FLATTEN_READY, kMailJobData::MAIL_PRIORITY_NORMAL, kConf::get("batch_flatten_video_sender_email"), kConf::get("batch_flatten_video_sender_name"), $data['email'], array($data['email'], $downloadLink));
}
KalturaLog::debug("Deleting: " . $data["deleteUrl"]);
kFile::downloadUrlToString($data["deleteUrl"]);
myNotificationMgr::createNotification(kNotificationJobData::NOTIFICATION_TYPE_ENTRY_UPDATE, $entry);
$job->setStatus(BatchJob::BATCHJOB_STATUS_FINISHED);
list($rootPath, $filePath) = kFileSyncUtils::getLocalFilePathArrForKey($fileSyncKey);
$fileFullPath = $rootPath . $filePath;
if (file_exists($fileFullPath)) {
try {
kFileSyncUtils::createSyncFileForKey($rootPath, $filePath, $fileSyncKey);
} catch (Exception $ex) {
KalturaLog::debug("ignore ERROR: " . $ex->getMessage());
}
} else {
KalturaLog::debug("The file [{$fileFullPath}] doesn't exists, not creating FileSync");
}
}
}
$job->save();
}
} catch (Exception $ex) {
KalturaLog::debug("ERROR: " . $ex->getMessage());
self::initDb(true);
self::failed();
}
if ($temp_count == 0) {
KalturaLog::debug("Ended conversion. sleeping for a while (" . $sleep_between_cycles . " seconds). Will write to the log in (" . $sleep_between_cycles * $number_of_times_to_skip_writing_sleeping . ") seconds");
}
$temp_count++;
if ($temp_count >= $number_of_times_to_skip_writing_sleeping) {
$temp_count = 0;
}
}
}
/**
* @param kUrlResource $resource
* @param entry $dbEntry
* @param asset $dbAsset
* @return asset
*/
protected function attachUrlResource(kUrlResource $resource, entry $dbEntry, asset $dbAsset = null)
{
if ($dbAsset instanceof flavorAsset) {
$dbEntry->setSource(KalturaSourceType::URL);
$dbEntry->save();
}
$url = $resource->getUrl();
if (!$resource->getForceAsyncDownload()) {
// TODO - move image handling to media service
if ($dbEntry->getMediaType() == KalturaMediaType::IMAGE) {
$entryFullPath = myContentStorage::getFSUploadsPath() . '/' . $dbEntry->getId() . '.jpg';
if (KCurlWrapper::getDataFromFile($url, $entryFullPath)) {
return $this->attachFile($entryFullPath, $dbEntry, $dbAsset);
}
KalturaLog::err("Failed downloading file[{$url}]");
$dbEntry->setStatus(entryStatus::ERROR_IMPORTING);
$dbEntry->save();
return null;
}
if ($dbAsset && !$dbAsset instanceof flavorAsset) {
$ext = pathinfo($url, PATHINFO_EXTENSION);
$entryFullPath = myContentStorage::getFSUploadsPath() . '/' . $dbEntry->getId() . '.' . $ext;
if (KCurlWrapper::getDataFromFile($url, $entryFullPath)) {
$dbAsset = $this->attachFile($entryFullPath, $dbEntry, $dbAsset);
return $dbAsset;
}
KalturaLog::err("Failed downloading file[{$url}]");
$dbAsset->setStatus(asset::FLAVOR_ASSET_STATUS_ERROR);
$dbAsset->save();
return null;
}
}
kJobsManager::addImportJob(null, $dbEntry->getId(), $this->getPartnerId(), $url, $dbAsset, null, $resource->getImportJobData());
return $dbAsset;
}
/**
* @param AttachmentAsset $attachmentAsset
* @param string $url
*/
protected function attachUrl(AttachmentAsset $attachmentAsset, $url)
{
$fullPath = myContentStorage::getFSUploadsPath() . '/' . basename($url);
if (KCurlWrapper::getDataFromFile($url, $fullPath)) {
return $this->attachFile($attachmentAsset, $fullPath);
}
if ($attachmentAsset->getStatus() == AttachmentAsset::ASSET_STATUS_QUEUED || $attachmentAsset->getStatus() == AttachmentAsset::ASSET_STATUS_NOT_APPLICABLE) {
$attachmentAsset->setDescription("Failed downloading file[{$url}]");
$attachmentAsset->setStatus(AttachmentAsset::ASSET_STATUS_ERROR);
$attachmentAsset->save();
}
throw new KalturaAPIException(KalturaAttachmentErrors::ATTACHMENT_ASSET_DOWNLOAD_FAILED, $url);
}
private function fetchFile(KalturaBatchJob $job, KalturaImportJobData $data)
{
KalturaLog::debug("fetchFile({$job->id})");
$jobSubType = $job->jobSubType;
if (in_array($jobSubType, array(kFileTransferMgrType::SCP, kFileTransferMgrType::SFTP))) {
// use SSH file transfer manager for SFTP/SCP
return $this->fetchFileSsh($job, $data);
}
try {
$sourceUrl = $data->srcFileUrl;
KalturaLog::debug("sourceUrl [{$sourceUrl}]");
$this->updateJob($job, 'Downloading file header', KalturaBatchJobStatus::QUEUED, 1);
$fileSize = null;
$resumeOffset = 0;
if ($data->destFileLocalPath && file_exists($data->destFileLocalPath)) {
$curlWrapper = new KCurlWrapper($sourceUrl, $this->taskConfig->params->curlVerbose);
$useNoBody = $job->executionAttempts > 1;
// if the process crashed first time, tries with no body instead of range 0-0
$curlHeaderResponse = $curlWrapper->getHeader($useNoBody);
if (!$curlHeaderResponse || !count($curlHeaderResponse->headers)) {
$this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $curlWrapper->getErrorNumber(), "Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::FAILED);
return $job;
}
if ($curlWrapper->getError()) {
KalturaLog::err("Headers error: " . $curlWrapper->getError());
KalturaLog::err("Headers error number: " . $curlWrapper->getErrorNumber());
$curlWrapper->close();
$curlWrapper = new KCurlWrapper($sourceUrl, $this->taskConfig->params->curlVerbose);
}
if (!$curlHeaderResponse->isGoodCode()) {
$this->closeJob($job, KalturaBatchJobErrorTypes::HTTP, $curlHeaderResponse->code, "HTTP Error: " . $curlHeaderResponse->code . " " . $curlHeaderResponse->codeName, KalturaBatchJobStatus::FAILED);
return $job;
}
if (isset($curlHeaderResponse->headers['content-length'])) {
$fileSize = $curlHeaderResponse->headers['content-length'];
}
$curlWrapper->close();
if ($fileSize) {
clearstatcache();
$actualFileSize = kFile::fileSize($data->destFileLocalPath);
if ($actualFileSize >= $fileSize) {
return $this->moveFile($job, $data->destFileLocalPath, $fileSize);
} else {
$resumeOffset = $actualFileSize;
}
}
}
$curlWrapper = new KCurlWrapper($sourceUrl, $this->taskConfig->params->curlVerbose);
$curlWrapper->setTimeout($this->taskConfig->params->curlTimeout);
if ($resumeOffset) {
$curlWrapper->setResumeOffset($resumeOffset);
} else {
// creates a temp file path
$destFile = $this->getTempFilePath($sourceUrl);
KalturaLog::debug("destFile [{$destFile}]");
$data->destFileLocalPath = $destFile;
$this->updateJob($job, "Downloading file, size: {$fileSize}", KalturaBatchJobStatus::PROCESSING, 2, $data);
}
KalturaLog::debug("Executing curl");
$res = $curlWrapper->exec($data->destFileLocalPath);
KalturaLog::debug("Curl results: {$res}");
if (!$res || $curlWrapper->getError()) {
$errNumber = $curlWrapper->getErrorNumber();
if ($errNumber != CURLE_OPERATION_TIMEOUTED) {
$this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $errNumber, "Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::RETRY);
$curlWrapper->close();
return $job;
} else {
clearstatcache();
$actualFileSize = kFile::fileSize($data->destFileLocalPath);
if ($actualFileSize == $resumeOffset) {
$this->closeJob($job, KalturaBatchJobErrorTypes::CURL, $errNumber, "Error: " . $curlWrapper->getError(), KalturaBatchJobStatus::RETRY);
$curlWrapper->close();
return $job;
}
}
}
$curlWrapper->close();
if (!file_exists($data->destFileLocalPath)) {
$this->closeJob($job, KalturaBatchJobErrorTypes::APP, KalturaBatchJobAppErrors::OUTPUT_FILE_DOESNT_EXIST, "Error: output file doesn't exist", KalturaBatchJobStatus::RETRY);
return $job;
}
// check the file size only if its first or second retry
// in case it failed few times, taks the file as is
if ($fileSize) {
clearstatcache();
$actualFileSize = kFile::fileSize($data->destFileLocalPath);
if ($actualFileSize < $fileSize) {
$percent = floor($actualFileSize * 100 / $fileSize);
$this->updateJob($job, "Downloaded size: {$actualFileSize}({$percent}%)", KalturaBatchJobStatus::PROCESSING, $percent, $data);
$this->kClient->batch->resetJobExecutionAttempts($job->id, $this->getExclusiveLockKey(), $job->jobType);
// $this->closeJob($job, KalturaBatchJobErrorTypes::APP, KalturaBatchJobAppErrors::OUTPUT_FILE_WRONG_SIZE, "Expected file size[$fileSize] actual file size[$actualFileSize]", KalturaBatchJobStatus::RETRY);
return $job;
}
}
$this->updateJob($job, 'File imported, copy to shared folder', KalturaBatchJobStatus::PROCESSED, 90);
$job = $this->moveFile($job, $data->destFileLocalPath, $fileSize);
} catch (Exception $ex) {
$this->closeJob($job, KalturaBatchJobErrorTypes::RUNTIME, $ex->getCode(), "Error: " . $ex->getMessage(), KalturaBatchJobStatus::FAILED);
}
return $job;
}
/**
*
* @param $wmData
* @param string $destFileSyncLocalPath
* @param string $cmdLine
* @param string $ffmpegBin
* @param string $mediaInfoBin
* @return string
*/
public static function buildWatermarkedCommandLine($wmData, $destFileSyncLocalPath, $cmdLine, $ffmpegBin = "ffmpeg", $mediaInfoBin = "mediainfo")
{
KalturaLog::log("After:cmdline({$cmdLine})");
if (!isset($mediaInfoBin) || strlen($mediaInfoBin) == 0) {
$mediaInfoBin = "mediainfo";
}
/*
* evaluate WM scale and margins, if any
*/
KalturaLog::log("Watermark data({$mediaInfoBin}):\n" . print_r($wmData, 1));
if (isset($wmData->scale) && is_string($wmData->scale)) {
$wmData->scale = explode("x", $wmData->scale);
}
if (isset($wmData->margins) && is_string($wmData->margins)) {
$wmData->margins = explode("x", $wmData->margins);
}
KalturaLog::log("Watermark data:\n" . print_r($wmData, 1));
/*
* Retrieve watermark image file,
* either from image entry or from external url
* If both set, prefer image entry
*/
$imageDownloadUrl = null;
$errStr = null;
if (isset($wmData->imageEntry)) {
$version = null;
try {
$imgEntry = KBatchBase::$kClient->baseEntry->get($wmData->imageEntry, $version);
} catch (Exception $ex) {
$imgEntry = null;
$errStr = "Exception on retrieval of an image entry({$wmData->imageEntry}),\nexception:" . print_r($ex, 1);
}
if (isset($imgEntry)) {
KalturaLog::log("Watermark entry: {$wmData->imageEntry}");
$imageDownloadUrl = $imgEntry->downloadUrl;
} else {
if (!isset($errStr)) {
$errStr = "Failed to retrieve an image entry({$wmData->imageEntry})";
}
}
if (!isset($imgEntry)) {
KalturaLog::err($errStr);
}
}
if (!isset($imageDownloadUrl)) {
if (isset($wmData->url)) {
$imageDownloadUrl = $wmData->url;
} else {
KalturaLog::err("Missing watermark image data, neither via image-entry-id nor via external url.");
return null;
}
}
$wmTmpFilepath = $destFileSyncLocalPath . ".wmtmp";
KalturaLog::log("imageDownloadUrl({$imageDownloadUrl}), wmTmpFilepath({$wmTmpFilepath})");
/*
* Get the watermark image file
*/
$curlWrapper = new KCurlWrapper();
$res = $curlWrapper->exec($imageDownloadUrl, $wmTmpFilepath);
KalturaLog::debug("Curl results: {$res}");
if (!$res || $curlWrapper->getError()) {
$errDescription = "Error: " . $curlWrapper->getError();
$curlWrapper->close();
KalturaLog::err("Failed to curl the caption file url({$imageDownloadUrl}). Error ({$errDescription})");
return null;
}
$curlWrapper->close();
if (!file_exists($wmTmpFilepath)) {
KalturaLog::err("Error: output file ({$wmTmpFilepath}) doesn't exist");
return null;
}
KalturaLog::log("Successfully retrieved the watermark image file ({$wmTmpFilepath}) ");
/*
* Query the image file for format and dims
*/
$output = array();
exec("{$mediaInfoBin} {$wmTmpFilepath} | grep -i \"width\\|height\\|format\"", $ouput, $rv);
if ($rv != 0) {
KalturaLog::err("Failed to retrieve media data from watermark file ({$wmTmpFilepath}). rv({$rv}). Carry on without watermark.");
return null;
}
foreach ($ouput as $line) {
if (!isset($wmData->format) && stristr($line, "format") != false) {
$str = stristr($line, ":");
$wmData->format = strtolower(trim($str, ": "));
} else {
if (stristr($line, "width") != false) {
$str = stristr($line, ":");
$wmData->width = (int) trim($str, ": ");
} else {
if (stristr($line, "height") != false) {
$str = stristr($line, ":");
$wmData->height = (int) trim($str, ": ");
}
}
}
}
switch ($wmData->format) {
case "jpeg":
$wmData->format = "jpg";
case "jpg":
case "png":
default:
rename($wmTmpFilepath, "{$wmTmpFilepath}.{$wmData->format}");
$wmTmpFilepath = "{$wmTmpFilepath}.{$wmData->format}";
break;
}
KalturaLog::log("Updated Watermark data:\n" . print_r($wmData, 1));
/*
* Evaluate WM scaling params and scale it accordingly
*/
$wid = null;
$hgt = null;
if (!isset($wmData->scale) && ($wmData->width % 2 != 0 || $wmData->height % 2 != 0)) {
$wmData->scale = array();
$wmData->width -= $wmData->width % 2;
$wmData->height -= $wmData->height % 2;
$wmData->scale[0] = $wmData->width;
$wmData->scale[1] = $wmData->height;
}
if (isset($wmData->scale)) {
$wid = in_array($wmData->scale[0], array(null, 0, -1)) ? -1 : $wmData->scale[0];
$hgt = in_array($wmData->scale[1], array(null, 0, -1)) ? -1 : $wmData->scale[1];
if ($wid <= 0) {
$wid = round($hgt * $wmData->width / $wmData->height);
}
if ($hgt <= 0) {
$hgt = round($wid * $wmData->height / $wmData->width);
}
if (!($wid > 0 && $hgt > 0)) {
$wid = $wmData->width;
$hgt = $wmData->height;
}
if ($wid > 0) {
$wid -= $wid % 2;
}
if ($hgt > 0) {
$hgt -= $hgt % 2;
}
$wmFilePath = $wmTmpFilepath;
} else {
$wmFilePath = $wmTmpFilepath;
$wid = $wmData->width;
$hgt = $wmData->height;
}
/* Samples -
"[1]scale=100:100,setsar=100/100[logo];[0:v]crop=100:100:iw-ow-10:300,setsar=100/100[cropped];[cropped][logo]blend=all_expr='if(eq(mod(X,2),mod(Y,2)),A,B)'[blended];[0:v][blended]overlay=main_w-overlay_w-10:300[out]"
"[1]scale=100:100,setsar=100/100[logo];[0:v][logo]overlay=main_w-overlay_w-10:300[out]" -map "[out]"
*/
$cmdLine = str_replace(array(KDLCmdlinePlaceholders::WaterMarkFileName, KDLCmdlinePlaceholders::WaterMarkWidth, KDLCmdlinePlaceholders::WaterMarkHeight), array($wmFilePath, $wid, $hgt), $cmdLine);
KalturaLog::log("After:cmdline({$cmdLine})");
return $cmdLine;
}
/**
* @param string $srcFileSyncRemoteUrl
* @param string $srcFileSyncLocalPath
* @param string $errDescription
* @return string
*/
private function fetchFile($srcFileSyncRemoteUrl, $srcFileSyncLocalPath, &$errDescription = null)
{
KalturaLog::debug("fetchFile({$srcFileSyncRemoteUrl}, {$srcFileSyncLocalPath})");
try {
$curlWrapper = new KCurlWrapper($srcFileSyncRemoteUrl);
$curlHeaderResponse = $curlWrapper->getHeader(true);
if (!$curlHeaderResponse || $curlWrapper->getError()) {
$errDescription = "Error: " . $curlWrapper->getError();
return false;
}
if ($curlHeaderResponse->code != KCurlHeaderResponse::HTTP_STATUS_OK) {
$errDescription = "HTTP Error: " . $curlHeaderResponse->code . " " . $curlHeaderResponse->codeName;
return false;
}
$fileSize = null;
if (isset($curlHeaderResponse->headers['content-length'])) {
$fileSize = $curlHeaderResponse->headers['content-length'];
}
$curlWrapper->close();
KalturaLog::debug("Executing curl");
$curlWrapper = new KCurlWrapper($srcFileSyncRemoteUrl);
$res = $curlWrapper->exec($srcFileSyncLocalPath);
KalturaLog::debug("Curl results: {$res}");
if (!$res || $curlWrapper->getError()) {
$errDescription = "Error: " . $curlWrapper->getError();
$curlWrapper->close();
return false;
}
$curlWrapper->close();
if (!file_exists($srcFileSyncLocalPath)) {
$errDescription = "Error: output file doesn't exist";
return false;
}
if ($fileSize) {
clearstatcache();
if (filesize($srcFileSyncLocalPath) != $fileSize) {
$errDescription = "Error: output file have a wrong size";
return false;
}
}
} catch (Exception $ex) {
$errDescription = "Error: " . $ex->getMessage();
return false;
}
return true;
}
/**
* @param string $str
* @return boolean
*/
private function isUrl($str)
{
KalturaLog::debug("isUrl({$str})");
$str = KCurlWrapper::encodeUrl($str);
$strRegex = "^((https?)|(ftp)):\\/\\/" . "?(([0-9a-z_!~*'().&=+\$%-]+:)?[0-9a-z_!~*'().&=+\$%-]+@)?" . "(([0-9]{1,3}\\.){3}[0-9]{1,3}" . "|" . "([0-9a-z_!~*'()-]+\\.)*" . "([0-9a-z][0-9a-z-]{0,61})?[0-9a-z]\\." . "[a-z]{2,6})" . "(:[0-9]{1,4})?" . "((\\/?)|" . "(\\/[0-9a-z_!~*'().;?:@&=+\$,%#-]+)+)\$";
return preg_match("/{$strRegex}/i", $str);
}