public function validateEntry(entry $dbEntry)
{
parent::validateEntry($dbEntry);
$this->validatePropertyNotNull('entryId');
$srcEntry = entryPeer::retrieveByPK($this->entryId);
if (!$srcEntry) {
throw new KalturaAPIException(KalturaErrors::ENTRY_ID_NOT_FOUND, $this->entryId);
}
if ($srcEntry->getMediaType() == KalturaMediaType::IMAGE) {
return parent::validateEntry($dbEntry);
}
$srcFlavorAsset = null;
if (is_null($this->flavorParamsId)) {
$srcFlavorAsset = assetPeer::retrieveOriginalByEntryId($this->entryId);
if (!$srcFlavorAsset) {
throw new KalturaAPIException(KalturaErrors::ORIGINAL_FLAVOR_ASSET_IS_MISSING);
}
} else {
$srcFlavorAsset = assetPeer::retrieveByEntryIdAndParams($this->entryId, $this->flavorParamsId);
if (!$srcFlavorAsset) {
throw new KalturaAPIException(KalturaErrors::FLAVOR_ASSET_ID_NOT_FOUND, $this->assetId);
}
}
$key = $srcFlavorAsset->getSyncKey(asset::FILE_SYNC_ASSET_SUB_TYPE_ASSET);
$c = FileSyncPeer::getCriteriaForFileSyncKey($key);
$c->addAnd(FileSyncPeer::FILE_TYPE, array(FileSync::FILE_SYNC_FILE_TYPE_FILE, FileSync::FILE_SYNC_FILE_TYPE_LINK), Criteria::IN);
$fileSyncs = FileSyncPeer::doSelect($c);
foreach ($fileSyncs as $fileSync) {
$fileSync = kFileSyncUtils::resolve($fileSync);
if ($fileSync->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_FILE) {
return;
}
}
throw new KalturaAPIException(KalturaErrors::FILE_DOESNT_EXIST);
}
/**
* @param BaseObject $object
* @return bool true if should continue to the next consumer
*/
public function objectAdded(BaseObject $object)
{
if (!$object instanceof FileSync || $object->getStatus() != FileSync::FILE_SYNC_STATUS_PENDING || $object->getFileType() != FileSync::FILE_SYNC_FILE_TYPE_FILE || $object->getDc() == kDataCenterMgr::getCurrentDcId()) {
return true;
}
$c = new Criteria();
$c->addAnd(FileSyncPeer::OBJECT_ID, $object->getObjectId());
$c->addAnd(FileSyncPeer::VERSION, $object->getVersion());
$c->addAnd(FileSyncPeer::OBJECT_TYPE, $object->getObjectType());
$c->addAnd(FileSyncPeer::OBJECT_SUB_TYPE, $object->getObjectSubType());
$c->addAnd(FileSyncPeer::ORIGINAL, '1');
$original_filesync = FileSyncPeer::doSelectOne($c);
if (!$original_filesync) {
KalturaLog::err('Original filesync not found for object_id[' . $object->getObjectId() . '] version[' . $object->getVersion() . '] type[' . $object->getObjectType() . '] subtype[' . $object->getObjectSubType() . ']');
return true;
}
$sourceFileUrl = $original_filesync->getExternalUrl();
if (!$sourceFileUrl) {
KalturaLog::err('External URL not found for filesync id [' . $object->getId() . ']');
return true;
}
$job = kMultiCentersManager::addFileSyncImportJob($this->getEntryId($object), $object->getPartnerId(), $object->getId(), $sourceFileUrl);
$job->setDc($object->getDc());
$job->save();
return true;
}
/**
* @action sync
* @param int $fileSyncId
* @param file $fileData
* @return KalturaFileSync
*/
function syncAction($fileSyncId, $fileData)
{
$dbFileSync = FileSyncPeer::retrieveByPK($fileSyncId);
if (!$dbFileSync) {
throw new APIException(APIErrors::INVALID_FILE_SYNC_ID, $fileSyncId);
}
$key = kFileSyncUtils::getKeyForFileSync($dbFileSync);
kFileSyncUtils::moveFromFile($fileData['tmp_name'], $key, false);
list($file_root, $real_path) = kPathManager::getFilePathArr($key);
$full_path = $file_root . $real_path;
chmod($full_path, 0644);
if (file_exists($full_path)) {
$dbFileSync->setFileRoot($file_root);
$dbFileSync->setFilePath($real_path);
$dbFileSync->setFileSizeFromPath($full_path);
$dbFileSync->setStatus(FileSync::FILE_SYNC_STATUS_READY);
} else {
$dbFileSync->setFileSize(-1);
$dbFileSync->setStatus(FileSync::FILE_SYNC_STATUS_ERROR);
}
$dbFileSync->save();
$fileSync = new KalturaFileSync();
$fileSync->fromObject($dbFileSync);
return $fileSync;
}
public function clearMemory()
{
accessControlPeer::clearInstancePool();
BatchJobPeer::clearInstancePool();
BulkUploadResultPeer::clearInstancePool();
categoryPeer::clearInstancePool();
EmailIngestionProfilePeer::clearInstancePool();
entryPeer::clearInstancePool();
FileSyncPeer::clearInstancePool();
flavorAssetPeer::clearInstancePool();
flavorParamsConversionProfilePeer::clearInstancePool();
flavorParamsOutputPeer::clearInstancePool();
flavorParamsPeer::clearInstancePool();
kshowPeer::clearInstancePool();
mediaInfoPeer::clearInstancePool();
moderationFlagPeer::clearInstancePool();
moderationPeer::clearInstancePool();
notificationPeer::clearInstancePool();
roughcutEntryPeer::clearInstancePool();
SchedulerConfigPeer::clearInstancePool();
SchedulerPeer::clearInstancePool();
SchedulerStatusPeer::clearInstancePool();
SchedulerWorkerPeer::clearInstancePool();
StorageProfilePeer::clearInstancePool();
syndicationFeedPeer::clearInstancePool();
TrackEntryPeer::clearInstancePool();
uiConfPeer::clearInstancePool();
UploadTokenPeer::clearInstancePool();
// TODO clear default filters
// TODO call all memory cleaner plugins
if (function_exists('gc_collect_cycles')) {
// php 5.3 and above
gc_collect_cycles();
}
}
public static function setDefaultCriteriaFilter()
{
if (self::$s_criteria_filter == null) {
self::$s_criteria_filter = new criteriaFilter();
}
$c = new Criteria();
$c->add(self::STATUS, array(FileSync::FILE_SYNC_STATUS_DELETED, FileSync::FILE_SYNC_STATUS_PURGED), Criteria::NOT_IN);
self::$s_criteria_filter->setFilter($c);
}
/**
* Serves multiple files for synchronization between datacenters
*/
public function execute()
{
$fileSyncIds = $this->getRequestParameter("ids");
$hash = $this->getRequestParameter("hash");
// validate hash
$currentDc = kDataCenterMgr::getCurrentDc();
$currentDcId = $currentDc["id"];
$expectedHash = md5($currentDc["secret"] . $fileSyncIds);
if ($hash !== $expectedHash) {
$error = "Invalid hash - ids [{$fileSyncIds}] got [{$hash}] expected [{$expectedHash}]";
KalturaLog::err($error);
KExternalErrors::dieError(KExternalErrors::INVALID_TOKEN);
}
// load file syncs
$fileSyncs = FileSyncPeer::retrieveByPks(explode(',', $fileSyncIds));
if ($fileSyncs) {
KalturaMonitorClient::initApiMonitor(false, 'extwidget.serveMultiFile', $fileSyncs[0]->getPartnerId());
}
// resolve file syncs
$filePaths = array();
foreach ($fileSyncs as $fileSync) {
if ($fileSync->getDc() != $currentDcId) {
$error = "FileSync id [" . $fileSync->getId() . "] does not belong to this DC";
KalturaLog::err($error);
KExternalErrors::dieError(KExternalErrors::BAD_QUERY);
}
// resolve if file_sync is link
$fileSyncResolved = kFileSyncUtils::resolve($fileSync);
// check if file sync path leads to a file or a directory
$resolvedPath = $fileSyncResolved->getFullPath();
if (is_dir($resolvedPath)) {
$error = "FileSync id [" . $fileSync->getId() . "] is a directory";
KalturaLog::err($error);
KExternalErrors::dieError(KExternalErrors::BAD_QUERY);
}
if (!file_exists($resolvedPath)) {
$error = "Path [{$resolvedPath}] for fileSync id [" . $fileSync->getId() . "] does not exist";
KalturaLog::err($error);
continue;
}
$filePaths[$fileSync->getId()] = $resolvedPath;
}
$boundary = md5(uniqid('', true));
header('Content-Type: multipart/form-data; boundary=' . $boundary);
foreach ($filePaths as $id => $filePath) {
echo "--{$boundary}\n";
echo "Content-Type: application/octet-stream\n";
echo "Content-Disposition: form-data; name=\"{$id}\"\n\n";
readfile($filePath);
echo "\n";
}
echo "--{$boundary}--\n";
KExternalErrors::dieGracefully();
}
/**
* @param string $srcWAMSAssetId
* @param string $targetPath
*/
public function __construct($srcWAMSAssetId, $targetPath)
{
KalturaLog::debug("Creation instance of KWAMSThumbnailMaker srcWAMSAssetId = [{$srcWAMSAssetId}] targetPath = [{$targetPath}]");
$this->srcWAMSAssetId = $srcWAMSAssetId;
$this->targetPath = $targetPath;
DbManager::setConfig(kConf::getDB());
DbManager::initialize();
$fileSync = FileSyncPeer::retrieveByWamsAssetId($srcWAMSAssetId);
if (!empty($fileSync)) {
$this->partnerId = $fileSync->getPartnerId();
}
}
/**
* @param string $id
* @param int $type
*/
protected function syncableDeleted($id, $type)
{
$c = new Criteria();
$c->add(FileSyncPeer::OBJECT_ID, $id);
$c->add(FileSyncPeer::OBJECT_TYPE, $type);
$c->add(FileSyncPeer::STATUS, array(FileSync::FILE_SYNC_STATUS_PURGED, FileSync::FILE_SYNC_STATUS_DELETED), Criteria::NOT_IN);
$fileSyncs = FileSyncPeer::doSelect($c);
foreach ($fileSyncs as $fileSync) {
$key = kFileSyncUtils::getKeyForFileSync($fileSync);
kFileSyncUtils::deleteSyncFileForKey($key);
}
}
/**
* Update file sync by id
*
* @action update
* @param int $id
* @param KalturaFileSync $fileSync
* @return KalturaFileSync
*
* @throws FileSyncErrors::FILESYNC_ID_NOT_FOUND
*/
function updateAction($id, KalturaFileSync $fileSync)
{
$dbFileSync = FileSyncPeer::retrieveByPK($id);
if (!$dbFileSync) {
throw new KalturaAPIException(FileSyncErrors::FILESYNC_ID_NOT_FOUND, $id);
}
$fileSync->toUpdatableObject($dbFileSync);
$dbFileSync->save();
$fileSync = new KalturaFileSync();
$fileSync->fromObject($dbFileSync, $this->getResponseProfile());
return $fileSync;
}
private function resumeEvents($flavorAsset)
{
$syncKey = $flavorAsset->getSyncKey(flavorAsset::FILE_SYNC_FLAVOR_ASSET_SUB_TYPE_ASSET);
$c = FileSyncPeer::getCriteriaForFileSyncKey($syncKey);
$fileSyncList = FileSyncPeer::doSelect($c);
foreach ($fileSyncList as $fileSync) {
// resume file sync added event
kEventsManager::continueEvent(new kObjectAddedEvent($fileSync), 'kVirusScanFlowManager');
}
// resume flavor asset added event consumption
kEventsManager::continueEvent(new kObjectAddedEvent($flavorAsset), 'kVirusScanFlowManager');
}
function getAllReadyInternalFileSyncsForKey(FileSyncKey $key)
{
$c = new Criteria();
$c = FileSyncPeer::getCriteriaForFileSyncKey($key);
$c->addAnd(FileSyncPeer::FILE_TYPE, FileSync::FILE_SYNC_FILE_TYPE_FILE);
$c->addAnd(FileSyncPeer::STATUS, FileSync::FILE_SYNC_STATUS_READY);
$results = FileSyncPeer::doSelect($c);
$assocResults = array();
foreach ($results as $curResult) {
$assocResults[$curResult->getDc()] = $curResult;
}
return $assocResults;
}
/**
* batch getExclusiveFileSyncImportJob action allows to get a BatchJob of type FILESYNC_IMPORT
*
* @action getExclusiveFileSyncImportJobs
* @param KalturaExclusiveLockKey $lockKey The unique lock key from the batch-process. Is used for the locking mechanism
* @param int $maxExecutionTime The maximum time in seconds the job reguarly take. Is used for the locking mechanism when determining an unexpected termination of a batch-process.
* @param int $numberOfJobs The maximum number of jobs to return.
* @param KalturaBatchJobFilter $filter Set of rules to fetch only rartial list of jobs
* @param int $maxOffset The maximum offset we accept for the distance from the best result.
* @return KalturaBatchJobArray
*
* TODO remove the destFilePath from the job data and get it later using the api, then delete this method
*/
function getExclusiveFileSyncImportJobsAction(KalturaExclusiveLockKey $lockKey, $maxExecutionTime, $numberOfJobs, KalturaBatchJobFilter $filter = null, $maxOffset = null)
{
$coreJobs = $this->getExclusiveJobs($lockKey, $maxExecutionTime, $numberOfJobs, $filter, BatchJobType::FILESYNC_IMPORT, $maxOffset);
$jobs = KalturaBatchJobArray::fromBatchJobArray($coreJobs);
if ($jobs) {
foreach ($jobs as $index => $job) {
$data = $job->data;
// try to get destination path from file sync
$fileSyncId = $data->filesyncId;
$fileSync = FileSyncPeer::retrieveByPK($fileSyncId);
if (!$fileSync) {
KalturaLog::err("Failed to load file sync [{$fileSyncId}] aborting job [{$job->id}]");
$dbJob = BatchJobPeer::retrieveByPK($job->id);
$dbJob->setMessage("Failed to load file sync [{$fileSyncId}]");
kJobsManager::abortDbBatchJob($dbJob);
unset($jobs[$index]);
continue;
}
$fileSyncRoot = $fileSync->getFileRoot();
$fileSyncPath = $fileSync->getFilePath();
if ($fileSyncRoot && $fileSyncPath) {
// destination path set on filesync
$dest_path = $fileSyncRoot . $fileSyncPath;
} else {
// not set on filesync - get path from path manager
$fileSyncKey = kFileSyncUtils::getKeyForFileSync($fileSync);
list($file_root, $real_path) = kPathManager::getFilePathArr($fileSyncKey);
$dest_path = $file_root . $real_path;
// update filesync on database
$fileSync->setFileRoot($file_root);
$fileSync->setFilePath($real_path);
$fileSync->save();
}
// update job data with destination path if needed
if (!$data->destFilePath) {
$data->destFilePath = $dest_path;
$job->data = $data;
KalturaLog::log('Updating destination path for job id [$job->id]');
$this->updateJob($job);
}
if (!is_dir(dirname($dest_path)) && !@mkdir(dirname($dest_path), 0755, true)) {
KalturaLog::ERR("Cannot create directory [{$dest_path}] - " . error_get_last());
}
}
}
return $jobs;
}
function deleteObject(FileSync $fileSync)
{
$object = kFileSyncUtils::retrieveObjectForFileSync($fileSync);
$key = $object->getSyncKey($fileSync->getObjectSubType());
if ($key->version != $fileSync->getVersion()) {
return;
}
switch ($fileSync->getObjectType()) {
case FileSyncObjectType::UICONF:
$object->setStatus(uiConf::UI_CONF_STATUS_DELETED);
$object->save();
break;
case FileSyncObjectType::ENTRY:
myEntryUtils::deleteEntry($object);
try {
$wrapper = objectWrapperBase::getWrapperClass($object);
$wrapper->removeFromCache("entry", $object->getId());
} catch (Exception $e) {
KalturaLog::err($e);
}
break;
case FileSyncObjectType::ASSET:
$object->setStatus(flavorAsset::FLAVOR_ASSET_STATUS_DELETED);
$object->setDeletedAt(time());
$object->save();
break;
case FileSyncObjectType::METADATA:
$object->setStatus(Metadata::STATUS_DELETED);
$object->save();
break;
default:
return;
}
if ($fileSync->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_LINK) {
return;
}
$criteria = new Criteria();
$criteria->add(FileSyncPeer::DC, $fileSync->getDc());
$criteria->add(FileSyncPeer::FILE_TYPE, FileSync::FILE_SYNC_FILE_TYPE_LINK);
$criteria->add(FileSyncPeer::LINKED_ID, $fileSync->getId());
$links = FileSyncPeer::doSelect($criteria);
foreach ($links as $link) {
deleteObject($link);
}
}
public function updatedJob(BatchJob $dbBatchJob)
{
switch ($dbBatchJob->getStatus()) {
case BatchJob::BATCHJOB_STATUS_FINISHED:
$data = $dbBatchJob->getData();
$kontikiFileSync = FileSyncPeer::retrieveByPK($data->getSrcFileSyncId());
/* @var $data kStorageExportJobData */
$asset = assetPeer::retrieveByFileSync($kontikiFileSync);
$asset->setTags(KontikiPlugin::KONTIKI_ASSET_TAG);
$asset->save();
//Get Kontiki file sync and set the external URL
$kontikiFileSync->setFileRoot("");
$kontikiFileSync->setFilePath($data->getContentMoid());
$kontikiFileSync->save();
break;
}
return true;
}
protected function updatedFileSyncImportFailed(BatchJob $dbBatchJob, kFileSyncImportJobData $data, BatchJob $twinJob = null)
{
// Update relevant filesync as FAILED
$fileSyncId = $data->getFilesyncId();
if (!$fileSyncId) {
KalturaLog::err('File sync ID not found in job data.');
throw new KalturaAPIException(MultiCentersErrors::INVALID_FILESYNC_ID);
}
$fileSync = FileSyncPeer::retrieveByPK($fileSyncId);
if (!$fileSync) {
KalturaLog::err("Invalid filesync record with id [{$fileSyncId}]");
throw new KalturaAPIException(MultiCentersErrors::INVALID_FILESYNC_RECORD, $fileSyncId);
}
$fileSync->setStatus(FileSync::FILE_SYNC_STATUS_ERROR);
$fileSync->setReadyAt(time());
$fileSync->save();
return $dbBatchJob;
}
/**
* Creates instance of class and initializes properties
* @param string $type
* @param string $filePath
* @param KSchedularTaskConfig $taskConfig
*/
public function __construct($type, $filePath, KSchedularTaskConfig $taskConfig, KalturaBatchJob $job, $wamsAssetId)
{
$this->wamsAssetId = $wamsAssetId;
$this->filePath = $filePath;
$this->mediaInfoParser = parent::getParser($type, $filePath, $taskConfig, $job);
$this->partnerId = $job->partnerId;
DbManager::setConfig(kConf::getDB());
DbManager::initialize();
$fileSync = FileSyncPeer::retrieveByWamsAssetId($this->wamsAssetId);
if ($fileSync) {
$flavorAsset = kFileSyncUtils::retrieveObjectForFileSync($fileSync);
if ($flavorAsset instanceof asset) {
$this->originalMediaInfo = mediaInfoPeer::retrieveOriginalByEntryId($flavorAsset->getEntryId());
$entry = $flavorAsset->getentry();
if ($entry) {
$this->mediaType = $entry->getMediaType();
}
}
}
}
/**
* Will forward to the regular swf player according to the widget_id
*/
public function execute()
{
requestUtils::handleConditionalGet();
$file_sync_id = $this->getRequestParameter("id");
$hash = $this->getRequestParameter("hash");
$file_name = $this->getRequestParameter("fileName");
if ($file_name) {
$file_name = base64_decode($file_name);
}
$file_sync = FileSyncPeer::retrieveByPk($file_sync_id);
if (!$file_sync) {
$current_dc_id = kDataCenterMgr::getCurrentDcId();
$error = "DC[{$current_dc_id}]: Cannot find FileSync with id [{$file_sync_id}]";
KalturaLog::err($error);
KExternalErrors::dieError(KExternalErrors::FILE_NOT_FOUND);
}
KalturaMonitorClient::initApiMonitor(false, 'extwidget.serveFile', $file_sync->getPartnerId());
kDataCenterMgr::serveFileToRemoteDataCenter($file_sync, $hash, $file_name);
die;
}
public function validateForUsage($sourceObject, $propertiesToSkip = array())
{
parent::validateForUsage($sourceObject, $propertiesToSkip);
$this->validatePropertyNotNull('assetId');
$srcFlavorAsset = assetPeer::retrieveById($this->assetId);
if (!$srcFlavorAsset) {
throw new KalturaAPIException(KalturaErrors::FLAVOR_ASSET_ID_NOT_FOUND, $resource->assetId);
}
$key = $srcFlavorAsset->getSyncKey(asset::FILE_SYNC_ASSET_SUB_TYPE_ASSET);
$c = FileSyncPeer::getCriteriaForFileSyncKey($key);
$c->addAnd(FileSyncPeer::FILE_TYPE, array(FileSync::FILE_SYNC_FILE_TYPE_FILE, FileSync::FILE_SYNC_FILE_TYPE_LINK), Criteria::IN);
$fileSyncs = FileSyncPeer::doSelect($c);
foreach ($fileSyncs as $fileSync) {
$fileSync = kFileSyncUtils::resolve($fileSync);
if ($fileSync->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_FILE) {
return;
}
}
throw new KalturaAPIException(KalturaErrors::FILE_DOESNT_EXIST);
}
/**
* batch getExclusiveFileSyncImportJob action allows to get a BatchJob of type FILESYNC_IMPORT
*
* @action getExclusiveFileSyncImportJobs
* @param KalturaExclusiveLockKey $lockKey The unique lock key from the batch-process. Is used for the locking mechanism
* @param int $maxExecutionTime The maximum time in seconds the job reguarly take. Is used for the locking mechanism when determining an unexpected termination of a batch-process.
* @param int $numberOfJobs The maximum number of jobs to return.
* @param KalturaBatchJobFilter $filter Set of rules to fetch only rartial list of jobs
* @return KalturaBatchJobArray
*/
function getExclusiveFileSyncImportJobsAction(KalturaExclusiveLockKey $lockKey, $maxExecutionTime, $numberOfJobs, KalturaBatchJobFilter $filter = null)
{
$jobs = $this->getExclusiveJobsAction($lockKey, $maxExecutionTime, $numberOfJobs, $filter, BatchJobType::FILESYNC_IMPORT);
if ($jobs) {
foreach ($jobs as $job) {
$data = $job->data;
// try to get destination path from file sync
$fileSyncId = $data->filesyncId;
$fileSync = FileSyncPeer::retrieveByPK($fileSyncId);
if (!$fileSync) {
throw new KalturaAPIException(MultiCentersErrors::INVALID_FILESYNC_RECORD, $fileSyncId);
}
$fileSyncRoot = $fileSync->getFileRoot();
$fileSyncPath = $fileSync->getFilePath();
if ($fileSyncRoot && $fileSyncPath) {
// destination path set on filesync
$dest_path = $fileSyncRoot . $fileSyncPath;
} else {
// not set on filesync - get path from path manager
$fileSyncKey = kFileSyncUtils::getKeyForFileSync($fileSync);
list($file_root, $real_path) = kPathManager::getFilePathArr($fileSyncKey);
$dest_path = $file_root . $real_path;
// update filesync on database
$fileSync->setFileRoot($file_root);
$fileSync->setFilePath($real_path);
$fileSync->save();
}
// update job data with destination path if needed
if (!$data->destFilePath) {
$data->destFilePath = $dest_path;
$job->data = $data;
KalturaLog::log('Updating destination path for job id [$job->id]');
$this->updateJob($job);
}
if (!is_dir(dirname($dest_path)) && !@mkdir(dirname($dest_path), 0755, true)) {
KalturaLog::ERR("Cannot create directory [{$dest_path}] - " . error_get_last());
}
}
}
return $jobs;
}
/**
* @param BatchJob $dbBatchJob
* @param kStorageExportJobData $data
* @return BatchJob
*/
public static function handleStorageExportFailed(BatchJob $dbBatchJob, kStorageExportJobData $data)
{
KalturaLog::debug("Export to storage failed for sync file[" . $data->getSrcFileSyncId() . "]");
$fileSync = FileSyncPeer::retrieveByPK($data->getSrcFileSyncId());
$fileSync->setStatus(FileSync::FILE_SYNC_STATUS_ERROR);
$fileSync->save();
return $dbBatchJob;
}
public function isPendingExport(asset $asset)
{
$key = $asset->getSyncKey(flavorAsset::FILE_SYNC_FLAVOR_ASSET_SUB_TYPE_ASSET);
$c = FileSyncPeer::getCriteriaForFileSyncKey($key);
$c->addAnd(FileSyncPeer::DC, $this->getId(), Criteria::EQUAL);
$fileSync = FileSyncPeer::doSelectOne($c);
if (!$fileSync) {
return false;
}
return $fileSync->getStatus() == FileSync::FILE_SYNC_STATUS_PENDING;
}
/**
* @return BaseObject
*/
public function getObject()
{
switch ($this->getObjectType()) {
case BatchJobObjectType::ENTRY:
entryPeer::setUseCriteriaFilter(false);
$object = entryPeer::retrieveByPK($this->getObjectId());
entryPeer::setUseCriteriaFilter(true);
return $object;
case BatchJobObjectType::ASSET:
assetPeer::setUseCriteriaFilter(false);
$object = assetPeer::retrieveById($this->getObjectId());
assetPeer::setUseCriteriaFilter(true);
return $object;
case BatchJobObjectType::CATEGORY:
categoryPeer::setUseCriteriaFilter(false);
$object = categoryPeer::retrieveByPK($this->getObjectId());
categoryPeer::setUseCriteriaFilter(true);
return $object;
case BatchJobObjectType::FILE_SYNC:
FileSyncPeer::setUseCriteriaFilter(false);
$object = FileSyncPeer::retrieveByPK($this->getObjectId());
FileSyncPeer::setUseCriteriaFilter(true);
return $object;
default:
// TODO implement IBatchable in relevant plugins
return KalturaPluginManager::loadObject('IBatchable', $this->getObjectId());
}
return $this->aRootJob;
}
/**
* batch lockPendingFileSyncs action locks file syncs for import by the file sync periodic worker
*
* @action lockPendingFileSyncs
* @param KalturaFileSyncFilter $filter
* @param int $workerId The id of the file sync import worker
* @param int $sourceDc The id of the DC from which the file syncs should be pulled
* @param int $maxCount The maximum number of file syncs that should be returned
* @param int $maxSize The maximum total size of file syncs that should be returned, this limit may be exceeded by one file sync
* @return KalturaLockFileSyncsResponse
*/
function lockPendingFileSyncsAction(KalturaFileSyncFilter $filter, $workerId, $sourceDc, $maxCount, $maxSize = null)
{
// need to explicitly disable the cache since this action may not perform any queries
kApiCache::disableConditionalCache();
// for dual dc deployments, if source dc is not specified, set it to the remote dc
if ($sourceDc < 0) {
$sourceDc = 1 - kDataCenterMgr::getCurrentDcId();
}
// get caches
$keysCache = kCacheManager::getSingleLayerCache(kCacheManager::CACHE_TYPE_QUERY_CACHE_KEYS);
if (!$keysCache) {
throw new KalturaAPIException(MultiCentersErrors::GET_KEYS_CACHE_FAILED);
}
$lockCache = kCacheManager::getSingleLayerCache(kCacheManager::CACHE_TYPE_LOCK_KEYS);
if (!$lockCache) {
throw new KalturaAPIException(MultiCentersErrors::GET_LOCK_CACHE_FAILED);
}
// get the max id / last id
$maxId = $keysCache->get(self::MAX_FILESYNC_ID_PREFIX . $sourceDc);
if (!$maxId) {
throw new KalturaAPIException(MultiCentersErrors::GET_MAX_FILESYNC_ID_FAILED, $sourceDc);
}
$initialLastId = $keysCache->get(self::LAST_FILESYNC_ID_PREFIX . $workerId);
KalturaLog::info("got lastId [{$initialLastId}] for worker [{$workerId}]");
$lastId = $initialLastId ? $initialLastId : $maxId;
// created at less than handled explicitly
$createdAtLessThanOrEqual = $filter->createdAtLessThanOrEqual;
$filter->createdAtLessThanOrEqual = null;
// build the criteria
$fileSyncFilter = new FileSyncFilter();
$filter->toObject($fileSyncFilter);
$baseCriteria = new Criteria();
$fileSyncFilter->attachToCriteria($baseCriteria);
$baseCriteria->add(FileSyncPeer::STATUS, FileSync::FILE_SYNC_STATUS_PENDING);
$baseCriteria->add(FileSyncPeer::FILE_TYPE, FileSync::FILE_SYNC_FILE_TYPE_FILE);
$baseCriteria->add(FileSyncPeer::DC, kDataCenterMgr::getCurrentDcId());
$baseCriteria->addAscendingOrderByColumn(FileSyncPeer::ID);
$baseCriteria->setLimit(self::MAX_FILESYNCS_PER_CHUNK);
$lockedFileSyncs = array();
$lockedFileSyncsSize = 0;
$limitReached = false;
$selectCount = 0;
$lastSelectId = 0;
$done = false;
while (!$done && $selectCount < self::MAX_FILESYNC_QUERIES_PER_CALL) {
// make sure last id is always increasing
if ($lastId <= $lastSelectId) {
KalturaLog::info("last id was decremented {$lastId} <= {$lastSelectId}, stopping");
break;
}
$lastSelectId = $lastId;
// clear the instance pool every once in a while (not clearing every time since
// some objects repeat between selects)
$selectCount++;
if ($selectCount % 5 == 0) {
FileSyncPeer::clearInstancePool();
}
// get a chunk of file syncs
// Note: starting slightly before the last id, because the ids may arrive out of order in the mysql replication
$c = clone $baseCriteria;
$idCriterion = $c->getNewCriterion(FileSyncPeer::ID, $lastId - 100, Criteria::GREATER_THAN);
$idCriterion->addAnd($c->getNewCriterion(FileSyncPeer::ID, $maxId, Criteria::LESS_THAN));
$c->addAnd($idCriterion);
// Note: disabling the criteria because it accumulates more and more criterions, and the status was already explicitly added
// once that bug is fixed, this can be removed
FileSyncPeer::setUseCriteriaFilter(false);
$fileSyncs = FileSyncPeer::doSelect($c);
FileSyncPeer::setUseCriteriaFilter(true);
if (!$fileSyncs) {
$lastId = $maxId;
break;
}
// if we got less than the limit no reason to perform any more queries
if (count($fileSyncs) < self::MAX_FILESYNCS_PER_CHUNK) {
$done = true;
}
$lastFileSync = end($fileSyncs);
$lastId = $lastFileSync->getId();
// filter by source dc
foreach ($fileSyncs as $index => $fileSync) {
if ($fileSync->getOriginalDc() != $sourceDc) {
unset($fileSyncs[$index]);
}
}
// filter by object type / sub type
$fileSyncs = array_filter($fileSyncs, array('FileSyncImportBatchService', 'shouldSyncFileObjectType'));
if (!$fileSyncs) {
continue;
}
// filter by created at
if ($createdAtLessThanOrEqual) {
$firstFileSync = reset($fileSyncs);
$prevLastId = $firstFileSync->getId();
foreach ($fileSyncs as $index => $fileSync) {
if ($fileSync->getCreatedAt(null) > $createdAtLessThanOrEqual) {
$done = true;
unset($fileSyncs[$index]);
if (!is_null($prevLastId)) {
$lastId = $prevLastId;
$prevLastId = null;
}
} else {
$prevLastId = $fileSync->getId();
}
}
if (!$fileSyncs) {
break;
}
}
// get locked file syncs with multi get
$lockKeys = array();
foreach ($fileSyncs as $fileSync) {
$lockKeys[] = self::LOCK_KEY_PREFIX . $fileSync->getId();
}
$lockKeys = $lockCache->get($lockKeys);
// try to lock file syncs
foreach ($fileSyncs as $fileSync) {
$curKey = self::LOCK_KEY_PREFIX . $fileSync->getId();
if (isset($lockKeys[$curKey])) {
KalturaLog::info('file sync ' . $fileSync->getId() . ' already locked');
continue;
}
if (!$lockCache->add($curKey, true, self::LOCK_EXPIRY)) {
KalturaLog::info('failed to lock file sync ' . $fileSync->getId());
continue;
}
KalturaLog::info('locked file sync ' . $fileSync->getId());
// get the original id if not set
if (!$fileSync->getOriginalId()) {
$originalFileSync = self::getOriginalFileSync($fileSync);
if (!$originalFileSync) {
KalturaLog::info('failed to get original file sync for ' . $fileSync->getId());
continue;
}
$fileSync->setOriginalId($originalFileSync->getId());
$fileSync->setCustomDataObj();
// update $fileSync->custom_data so that originalId will be set by fromObject
}
// add to the result set
$lockedFileSyncs[] = $fileSync;
$lockedFileSyncsSize += $fileSync->getFileSize();
if (count($lockedFileSyncs) >= $maxCount || $maxSize && $lockedFileSyncsSize >= $maxSize) {
$lastId = $fileSync->getId();
$limitReached = true;
break;
}
}
if ($limitReached) {
break;
}
}
// update the last id
// Note: it is possible that the last id will go back in case of race condition,
// but the only effect of this is that some file syncs will be scanned again
if (!$initialLastId || $lastId > $initialLastId) {
KalturaLog::info("setting lastId to [{$lastId}] for worker [{$workerId}]");
$keysCache->set(self::LAST_FILESYNC_ID_PREFIX . $workerId, $lastId);
}
// make sure all file syncs have a path
foreach ($lockedFileSyncs as $fileSync) {
if ($fileSync->getFileRoot() && $fileSync->getFilePath()) {
continue;
}
$fileSyncKey = kFileSyncUtils::getKeyForFileSync($fileSync);
list($fileRoot, $realPath) = kPathManager::getFilePathArr($fileSyncKey);
$fileSync->setFileRoot($fileRoot);
$fileSync->setFilePath($realPath);
}
// build the response object
$sourceDc = kDataCenterMgr::getDcById($sourceDc);
$result = new KalturaLockFileSyncsResponse();
$result->fileSyncs = KalturaFileSyncArray::fromDbArray($lockedFileSyncs, $this->getResponseProfile());
$result->limitReached = $limitReached;
$result->dcSecret = $sourceDc["secret"];
$result->baseUrl = isset($sourceDc["fileSyncImportUrl"]) ? $sourceDc["fileSyncImportUrl"] : $sourceDc["url"];
return $result;
}
/**
* Retrieve multiple objects by pkey.
*
* @param array $pks List of primary keys
* @param PropelPDO $con the connection to use
* @throws PropelException Any exceptions caught during processing will be
* rethrown wrapped into a PropelException.
*/
public static function retrieveByPKs($pks, PropelPDO $con = null)
{
$objs = null;
if (empty($pks)) {
$objs = array();
} else {
$criteria = new Criteria(FileSyncPeer::DATABASE_NAME);
$criteria->add(FileSyncPeer::ID, $pks, Criteria::IN);
$objs = FileSyncPeer::doSelect($criteria, $con);
}
return $objs;
}
public function execute()
{
$this->forceSystemAuthentication();
myDbHelper::$use_alternative_con = null;
//myDbHelper::DB_HELPER_CONN_PROPEL2
$this->saved = false;
$uiConfId = $this->getRequestParameter("id");
if ($uiConfId) {
$uiConf = uiConfPeer::retrieveByPK($uiConfId);
} else {
$uiConf = new uiConf();
}
if ($this->getRequest()->getMethodName() == "POST") {
$uiConf->setPartnerId($this->getRequestParameter("partnerId"));
$uiConf->setSubpId($uiConf->getPartnerId() * 100);
$uiConf->setObjType($this->getRequestParameter("type"));
$uiConf->setName($this->getRequestParameter("name"));
$uiConf->setWidth($this->getRequestParameter("width"));
$uiConf->setHeight($this->getRequestParameter("height"));
$uiConf->setCreationMode($this->getRequestParameter("creationMode"));
$uiConf->setSwfUrl($this->getRequestParameter("swfUrl"));
$uiConf->setConfVars($this->getRequestParameter("confVars"));
$useCdn = $this->getRequestParameter("useCdn");
$uiConf->setUseCdn($useCdn === "1" ? true : false);
$uiConf->setDisplayInSearch($this->getRequestParameter("displayInSearch"));
$uiConf->setConfVars($this->getRequestParameter("confVars"));
$uiConf->setTags($this->getRequestParameter("tags"));
/* files: */
if ($uiConf->getCreationMode() != uiConf::UI_CONF_CREATION_MODE_MANUAL) {
$confFile = $this->getRequestParameter("uiconf_confFile");
$confFileFeatures = $this->getRequestParameter("uiconf_confFileFeatures");
if ($uiConf->getConfFile() != $confFile || $uiConf->getConfFileFeatures() != $confFileFeatures) {
$uiConf->setConfFile($confFile);
$uiConf->setConfFileFeatures($confFileFeatures);
}
}
$uiConf->save();
$this->saved = true;
}
// so script won't die when uiconf is missing
if (!$uiConf) {
$uiConf = new uiConf();
}
$partner = PartnerPeer::retrieveByPK($uiConf->getPartnerId());
$types = $uiConf->getUiConfTypeMap();
$c = new Criteria();
$c->add(widgetPeer::UI_CONF_ID, $uiConf->getId());
$c->setLimit(1000);
$widgets = widgetPeer::doSelect($c);
$widgetsPerPartner = array();
$this->tooMuchWidgets = false;
if (count($widgets) == 1000) {
$this->tooMuchWidgets = true;
} else {
if ($widgets) {
foreach ($widgets as $widget) {
if (!array_key_exists($widget->getPartnerId(), $widgetsPerPartner)) {
$widgetsPerPartner[$widget->getPartnerId()] = 0;
}
$widgetsPerPartner[$widget->getPartnerId()]++;
}
}
}
// find the FileSync
$fileSyncs = array();
$fileSyncs[] = array("key" => $uiConf->getSyncKey(uiConf::FILE_SYNC_UICONF_SUB_TYPE_DATA));
$fileSyncs[] = array("key" => $uiConf->getSyncKey(uiConf::FILE_SYNC_UICONF_SUB_TYPE_FEATURES));
foreach ($fileSyncs as &$fileSync) {
$fileSync["fileSyncs"] = FileSyncPeer::retrieveAllByFileSyncKey($fileSync["key"]);
}
$this->fileSyncs = $fileSyncs;
$this->widgetsPerPartner = $widgetsPerPartner;
$this->directoryMap = $uiConf->getDirectoryMap();
$this->swfNames = $uiConf->getSwfNames();
$this->types = $types;
$this->uiConf = $uiConf;
$this->partner = $partner;
}
/**
* Get remote storage existing paths for the asset
*
* @action getRemotePaths
* @param string $id
* @return KalturaRemotePathListResponse
* @throws KalturaAttachmentErrors::ATTACHMENT_ASSET_ID_NOT_FOUND
* @throws KalturaAttachmentErrors::ATTACHMENT_ASSET_IS_NOT_READY
*/
public function getRemotePathsAction($id)
{
$assetDb = assetPeer::retrieveById($id);
if (!$assetDb || !$assetDb instanceof AttachmentAsset) {
throw new KalturaAPIException(KalturaAttachmentErrors::ATTACHMENT_ASSET_ID_NOT_FOUND, $id);
}
if ($assetDb->getStatus() != asset::ASSET_STATUS_READY) {
throw new KalturaAPIException(KalturaAttachmentErrors::ATTACHMENT_ASSET_IS_NOT_READY);
}
$c = new Criteria();
$c->add(FileSyncPeer::OBJECT_TYPE, FileSyncObjectType::ASSET);
$c->add(FileSyncPeer::OBJECT_SUB_TYPE, asset::FILE_SYNC_ASSET_SUB_TYPE_ASSET);
$c->add(FileSyncPeer::OBJECT_ID, $id);
$c->add(FileSyncPeer::VERSION, $assetDb->getVersion());
$c->add(FileSyncPeer::PARTNER_ID, $assetDb->getPartnerId());
$c->add(FileSyncPeer::STATUS, FileSync::FILE_SYNC_STATUS_READY);
$c->add(FileSyncPeer::FILE_TYPE, FileSync::FILE_SYNC_FILE_TYPE_URL);
$fileSyncs = FileSyncPeer::doSelect($c);
$listResponse = new KalturaRemotePathListResponse();
$listResponse->objects = KalturaRemotePathArray::fromFileSyncArray($fileSyncs);
$listResponse->totalCount = count($listResponse->objects);
return $listResponse;
}
/**
* @param EntryDistribution $entryDistribution
* @param DistributionProfile $distributionProfile
* @param int $dc
* @return bool true if the job could be created
*/
protected static function prepareDistributionJob(EntryDistribution $entryDistribution, DistributionProfile $distributionProfile, &$dc)
{
// prepare ids list of all the assets
$assetIds = explode(',', implode(',', array($entryDistribution->getThumbAssetIds(), $entryDistribution->getFlavorAssetIds())));
$assets = assetPeer::retrieveByIds($assetIds);
$assetObjects = array();
foreach ($assets as $asset) {
/* @var $asset asset */
$assetObjects[$asset->getId()] = array('asset' => $asset, 'downloadUrl' => null);
}
// lists all files from all assets
$c = new Criteria();
$c->add(FileSyncPeer::OBJECT_TYPE, FileSyncObjectType::ASSET);
$c->add(FileSyncPeer::OBJECT_SUB_TYPE, asset::FILE_SYNC_ASSET_SUB_TYPE_ASSET);
$c->add(FileSyncPeer::OBJECT_ID, $assetIds, Criteria::IN);
$c->add(FileSyncPeer::PARTNER_ID, $entryDistribution->getPartnerId());
$c->add(FileSyncPeer::STATUS, FileSync::FILE_SYNC_STATUS_READY);
$fileSyncs = FileSyncPeer::doSelect($c);
$dcs = array();
foreach ($fileSyncs as $fileSync) {
/* @var $fileSync FileSync */
$assetId = $fileSync->getObjectId();
if (!isset($assetObjects[$assetId])) {
// the object is not in the list of assets
continue;
}
$asset = $assetObjects[$assetId]['asset'];
/* @var $asset asset */
if ($asset->getVersion() != $fileSync->getVersion()) {
// the file sync is not of the current asset version
continue;
}
$fileSync = kFileSyncUtils::resolve($fileSync);
// use the best URL as the source for download in case it will be needed
if ($fileSync->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_URL) {
if (!is_null($assetObjects[$assetId]['downloadUrl']) && $fileSync->getDc() != $distributionProfile->getRecommendedStorageProfileForDownload()) {
continue;
}
$downloadUrl = $fileSync->getExternalUrl($entryDistribution->getEntryId());
if (!$downloadUrl) {
continue;
}
$assetObjects[$assetId]['downloadUrl'] = $downloadUrl;
continue;
}
// populates the list of files in each dc
$fileSyncDc = $fileSync->getDc();
if (!isset($dcs[$fileSyncDc])) {
$dcs[$fileSyncDc] = array();
}
$dcs[$fileSyncDc][$assetId] = $fileSync->getId();
}
if (isset($dcs[$dc]) && count($dcs[$dc]) == count($assets)) {
KalturaLog::debug("All files exist in the preferred dc [{$dc}]");
return true;
}
// check if all files exist on any of the remote dcs
$otherDcs = kDataCenterMgr::getAllDcs(true);
foreach ($otherDcs as $remoteDc) {
$remoteDcId = $remoteDc['id'];
if (!isset($dcs[$remoteDcId]) && count($dcs[$remoteDcId]) != count($assets)) {
continue;
}
$dc = $remoteDcId;
KalturaLog::debug("All files exist in none-preferred dc [{$dc}]");
return true;
}
if ($entryDistribution->getStatus() == EntryDistributionStatus::IMPORT_SUBMITTING || $entryDistribution->getStatus() == EntryDistributionStatus::IMPORT_UPDATING) {
KalturaLog::debug("Entry distribution already importing");
return false;
}
// create all needed import jobs
$destinationDc = $distributionProfile->getRecommendedDcForDownload();
$dcExistingFiles = $dcs[$destinationDc];
foreach ($assetObjects as $assetId => $assetObject) {
if (is_null($assetObject['downloadUrl'])) {
KalturaLog::debug("Download URL not found for asset [{$assetId}]");
continue;
}
$asset = $assetObject['asset'];
/* @var $asset asset */
if (isset($dcExistingFiles[$assetId])) {
continue;
}
$jobData = new kImportJobData();
$jobData->setCacheOnly(true);
self::addImportJob($destinationDc, $assetObject['downloadUrl'], $asset);
}
return false;
}
private function deleteOldFileSyncVersions(FileSync $newFileSync)
{
KalturaLog::debug('Deleting old file_sync versions for [' . $newFileSync->getId() . ']');
if (kConf::hasParam('num_of_old_file_sync_versions_to_keep')) {
$keepCount = kConf::get('num_of_old_file_sync_versions_to_keep');
if (!is_numeric($newFileSync->getVersion())) {
return;
}
$intVersion = intval($newFileSync->getVersion());
$c = new Criteria();
$c->add(FileSyncPeer::OBJECT_ID, $newFileSync->getObjectId());
$c->add(FileSyncPeer::OBJECT_TYPE, $newFileSync->getObjectType());
$c->add(FileSyncPeer::OBJECT_SUB_TYPE, $newFileSync->getObjectSubType());
$c->add(FileSyncPeer::STATUS, array(FileSync::FILE_SYNC_STATUS_PURGED, FileSync::FILE_SYNC_STATUS_DELETED), Criteria::NOT_IN);
$c->setLimit(40);
//we limit the number of files to delete in one run so there will be no out of memory issues
$fileSyncs = FileSyncPeer::doSelect($c);
foreach ($fileSyncs as $fileSync) {
if (is_numeric($fileSync->getVersion())) {
$currentIntVersion = intval($fileSync->getVersion());
if ($intVersion - $keepCount > $currentIntVersion) {
$key = kFileSyncUtils::getKeyForFileSync($fileSync);
self::deleteSyncFileForKey($key);
}
}
}
}
}
protected function getRemotePaths($entryId, $entryType = null)
{
$dbEntry = entryPeer::retrieveByPK($entryId);
if (!$dbEntry || $entryType !== null && $dbEntry->getType() != $entryType) {
throw new KalturaAPIException(KalturaErrors::ENTRY_ID_NOT_FOUND, $entryId);
}
if ($dbEntry->getStatus() != entryStatus::READY) {
throw new KalturaAPIException(KalturaErrors::ENTRY_NOT_READY, $entryId);
}
$c = new Criteria();
$c->add(FileSyncPeer::OBJECT_TYPE, FileSyncObjectType::ENTRY);
$c->add(FileSyncPeer::OBJECT_SUB_TYPE, entry::FILE_SYNC_ENTRY_SUB_TYPE_DATA);
$c->add(FileSyncPeer::OBJECT_ID, $entryId);
$c->add(FileSyncPeer::VERSION, $dbEntry->getVersion());
$c->add(FileSyncPeer::PARTNER_ID, $dbEntry->getPartnerId());
$c->add(FileSyncPeer::STATUS, FileSync::FILE_SYNC_STATUS_READY);
$c->add(FileSyncPeer::FILE_TYPE, FileSync::FILE_SYNC_FILE_TYPE_URL);
$fileSyncs = FileSyncPeer::doSelect($c);
$listResponse = new KalturaRemotePathListResponse();
$listResponse->objects = KalturaRemotePathArray::fromDbArray($fileSyncs, $this->getResponseProfile());
$listResponse->totalCount = count($listResponse->objects);
return $listResponse;
}
public function isPendingExport(FileSyncKey $key)
{
$c = FileSyncPeer::getCriteriaForFileSyncKey($key);
$c->addAnd(FileSyncPeer::DC, $this->getId(), Criteria::EQUAL);
$fileSync = FileSyncPeer::doSelectOne($c);
if (!$fileSync) {
return false;
}
return $fileSync->getStatus() == FileSync::FILE_SYNC_STATUS_PENDING;
}
