/**
* will return a pair of file_root and file_path
* This is the only function that should be extended for building a different path
*
* @param ISyncableFile $object
* @param int $subType
* @param $version
*/
public function generateFilePathArr(ISyncableFile $object, $subType, $version = null, $storageProfileId = null)
{
// currently xsl paths are only used for assets
if (!$object instanceof asset) {
return parent::generateFilePathArr($object, $subType, $version, $storageProfileId);
}
$storageProfile = kPathManager::getStorageProfile($storageProfileId);
$pathXsl = $storageProfile->getPathFormat();
$entry = $object->getEntry();
$xslVariables = $this->getXslVariables($storageProfile, $object, $subType, $version, $entry);
$xslStr = $this->getXsl($pathXsl, $xslVariables);
try {
$path = $this->getPathValue($entry, $xslStr);
} catch (Exception $e) {
KalturaLog::err('Error executing XSL - ' . $e->getMessage());
$path = null;
}
if (empty($path)) {
KalturaLog::log('Empty path recieved - using parent\'s path instead');
return parent::generateFilePathArr($object, $subType, $version, $storageProfileId);
}
$path = trim($path);
KalturaLog::debug('Path value [' . $path . ']');
$root = '/';
return array($root, $path);
}
/**
* @action sync
* @param int $fileSyncId
* @param file $fileData
* @return KalturaFileSync
*/
function syncAction($fileSyncId, $fileData)
{
$dbFileSync = FileSyncPeer::retrieveByPK($fileSyncId);
if (!$dbFileSync) {
throw new APIException(APIErrors::INVALID_FILE_SYNC_ID, $fileSyncId);
}
$key = kFileSyncUtils::getKeyForFileSync($dbFileSync);
kFileSyncUtils::moveFromFile($fileData['tmp_name'], $key, false);
list($file_root, $real_path) = kPathManager::getFilePathArr($key);
$full_path = $file_root . $real_path;
chmod($full_path, 0644);
if (file_exists($full_path)) {
$dbFileSync->setFileRoot($file_root);
$dbFileSync->setFilePath($real_path);
$dbFileSync->setFileSizeFromPath($full_path);
$dbFileSync->setStatus(FileSync::FILE_SYNC_STATUS_READY);
} else {
$dbFileSync->setFileSize(-1);
$dbFileSync->setStatus(FileSync::FILE_SYNC_STATUS_ERROR);
}
$dbFileSync->save();
$fileSync = new KalturaFileSync();
$fileSync->fromObject($dbFileSync);
return $fileSync;
}
/**
* batch getExclusiveFileSyncImportJob action allows to get a BatchJob of type FILESYNC_IMPORT
*
* @action getExclusiveFileSyncImportJobs
* @param KalturaExclusiveLockKey $lockKey The unique lock key from the batch-process. Is used for the locking mechanism
* @param int $maxExecutionTime The maximum time in seconds the job reguarly take. Is used for the locking mechanism when determining an unexpected termination of a batch-process.
* @param int $numberOfJobs The maximum number of jobs to return.
* @param KalturaBatchJobFilter $filter Set of rules to fetch only rartial list of jobs
* @param int $maxOffset The maximum offset we accept for the distance from the best result.
* @return KalturaBatchJobArray
*
* TODO remove the destFilePath from the job data and get it later using the api, then delete this method
*/
function getExclusiveFileSyncImportJobsAction(KalturaExclusiveLockKey $lockKey, $maxExecutionTime, $numberOfJobs, KalturaBatchJobFilter $filter = null, $maxOffset = null)
{
$coreJobs = $this->getExclusiveJobs($lockKey, $maxExecutionTime, $numberOfJobs, $filter, BatchJobType::FILESYNC_IMPORT, $maxOffset);
$jobs = KalturaBatchJobArray::fromBatchJobArray($coreJobs);
if ($jobs) {
foreach ($jobs as $index => $job) {
$data = $job->data;
// try to get destination path from file sync
$fileSyncId = $data->filesyncId;
$fileSync = FileSyncPeer::retrieveByPK($fileSyncId);
if (!$fileSync) {
KalturaLog::err("Failed to load file sync [{$fileSyncId}] aborting job [{$job->id}]");
$dbJob = BatchJobPeer::retrieveByPK($job->id);
$dbJob->setMessage("Failed to load file sync [{$fileSyncId}]");
kJobsManager::abortDbBatchJob($dbJob);
unset($jobs[$index]);
continue;
}
$fileSyncRoot = $fileSync->getFileRoot();
$fileSyncPath = $fileSync->getFilePath();
if ($fileSyncRoot && $fileSyncPath) {
// destination path set on filesync
$dest_path = $fileSyncRoot . $fileSyncPath;
} else {
// not set on filesync - get path from path manager
$fileSyncKey = kFileSyncUtils::getKeyForFileSync($fileSync);
list($file_root, $real_path) = kPathManager::getFilePathArr($fileSyncKey);
$dest_path = $file_root . $real_path;
// update filesync on database
$fileSync->setFileRoot($file_root);
$fileSync->setFilePath($real_path);
$fileSync->save();
}
// update job data with destination path if needed
if (!$data->destFilePath) {
$data->destFilePath = $dest_path;
$job->data = $data;
KalturaLog::log('Updating destination path for job id [$job->id]');
$this->updateJob($job);
}
if (!is_dir(dirname($dest_path)) && !@mkdir(dirname($dest_path), 0755, true)) {
KalturaLog::ERR("Cannot create directory [{$dest_path}] - " . error_get_last());
}
}
}
return $jobs;
}
/**
* will return a pair of file_root and file_path
* This is the only function that should be extended for building a different path
*
* @param ISyncableFile $object
* @param int $subType
* @param $version
*/
public function generateFilePathArr(ISyncableFile $object, $subType, $version = null, $storageProfileId = null)
{
$storageProfile = kPathManager::getStorageProfile($storageProfileId);
$path_format = $storageProfile->getPathFormat();
if (is_null($path_format)) {
$path_format = '{year}{month}{day}/{partnerDir}/{fileName}';
}
$fileName = $object->generateFileName($subType, $version);
$partnerDir = floor($object->getPartnerId() / 1000);
$path = str_replace('{fileName}', $fileName, $path_format);
$path = str_replace('{partnerDir}', $partnerDir, $path);
$path = str_replace('{year}', date("Y"), $path);
$path = str_replace('{month}', date("m"), $path);
$path = str_replace('{day}', date("d"), $path);
$root = '/';
return array($root, $path);
}
/**
* batch getExclusiveFileSyncImportJob action allows to get a BatchJob of type FILESYNC_IMPORT
*
* @action getExclusiveFileSyncImportJobs
* @param KalturaExclusiveLockKey $lockKey The unique lock key from the batch-process. Is used for the locking mechanism
* @param int $maxExecutionTime The maximum time in seconds the job reguarly take. Is used for the locking mechanism when determining an unexpected termination of a batch-process.
* @param int $numberOfJobs The maximum number of jobs to return.
* @param KalturaBatchJobFilter $filter Set of rules to fetch only rartial list of jobs
* @return KalturaBatchJobArray
*/
function getExclusiveFileSyncImportJobsAction(KalturaExclusiveLockKey $lockKey, $maxExecutionTime, $numberOfJobs, KalturaBatchJobFilter $filter = null)
{
$jobs = $this->getExclusiveJobsAction($lockKey, $maxExecutionTime, $numberOfJobs, $filter, BatchJobType::FILESYNC_IMPORT);
if ($jobs) {
foreach ($jobs as $job) {
$data = $job->data;
// try to get destination path from file sync
$fileSyncId = $data->filesyncId;
$fileSync = FileSyncPeer::retrieveByPK($fileSyncId);
if (!$fileSync) {
throw new KalturaAPIException(MultiCentersErrors::INVALID_FILESYNC_RECORD, $fileSyncId);
}
$fileSyncRoot = $fileSync->getFileRoot();
$fileSyncPath = $fileSync->getFilePath();
if ($fileSyncRoot && $fileSyncPath) {
// destination path set on filesync
$dest_path = $fileSyncRoot . $fileSyncPath;
} else {
// not set on filesync - get path from path manager
$fileSyncKey = kFileSyncUtils::getKeyForFileSync($fileSync);
list($file_root, $real_path) = kPathManager::getFilePathArr($fileSyncKey);
$dest_path = $file_root . $real_path;
// update filesync on database
$fileSync->setFileRoot($file_root);
$fileSync->setFilePath($real_path);
$fileSync->save();
}
// update job data with destination path if needed
if (!$data->destFilePath) {
$data->destFilePath = $dest_path;
$job->data = $data;
KalturaLog::log('Updating destination path for job id [$job->id]');
$this->updateJob($job);
}
if (!is_dir(dirname($dest_path)) && !@mkdir(dirname($dest_path), 0755, true)) {
KalturaLog::ERR("Cannot create directory [{$dest_path}] - " . error_get_last());
}
}
}
return $jobs;
}
/**
* batch lockPendingFileSyncs action locks file syncs for import by the file sync periodic worker
*
* @action lockPendingFileSyncs
* @param KalturaFileSyncFilter $filter
* @param int $workerId The id of the file sync import worker
* @param int $sourceDc The id of the DC from which the file syncs should be pulled
* @param int $maxCount The maximum number of file syncs that should be returned
* @param int $maxSize The maximum total size of file syncs that should be returned, this limit may be exceeded by one file sync
* @return KalturaLockFileSyncsResponse
*/
function lockPendingFileSyncsAction(KalturaFileSyncFilter $filter, $workerId, $sourceDc, $maxCount, $maxSize = null)
{
// need to explicitly disable the cache since this action may not perform any queries
kApiCache::disableConditionalCache();
// for dual dc deployments, if source dc is not specified, set it to the remote dc
if ($sourceDc < 0) {
$sourceDc = 1 - kDataCenterMgr::getCurrentDcId();
}
// get caches
$keysCache = kCacheManager::getSingleLayerCache(kCacheManager::CACHE_TYPE_QUERY_CACHE_KEYS);
if (!$keysCache) {
throw new KalturaAPIException(MultiCentersErrors::GET_KEYS_CACHE_FAILED);
}
$lockCache = kCacheManager::getSingleLayerCache(kCacheManager::CACHE_TYPE_LOCK_KEYS);
if (!$lockCache) {
throw new KalturaAPIException(MultiCentersErrors::GET_LOCK_CACHE_FAILED);
}
// get the max id / last id
$maxId = $keysCache->get(self::MAX_FILESYNC_ID_PREFIX . $sourceDc);
if (!$maxId) {
throw new KalturaAPIException(MultiCentersErrors::GET_MAX_FILESYNC_ID_FAILED, $sourceDc);
}
$initialLastId = $keysCache->get(self::LAST_FILESYNC_ID_PREFIX . $workerId);
KalturaLog::info("got lastId [{$initialLastId}] for worker [{$workerId}]");
$lastId = $initialLastId ? $initialLastId : $maxId;
// created at less than handled explicitly
$createdAtLessThanOrEqual = $filter->createdAtLessThanOrEqual;
$filter->createdAtLessThanOrEqual = null;
// build the criteria
$fileSyncFilter = new FileSyncFilter();
$filter->toObject($fileSyncFilter);
$baseCriteria = new Criteria();
$fileSyncFilter->attachToCriteria($baseCriteria);
$baseCriteria->add(FileSyncPeer::STATUS, FileSync::FILE_SYNC_STATUS_PENDING);
$baseCriteria->add(FileSyncPeer::FILE_TYPE, FileSync::FILE_SYNC_FILE_TYPE_FILE);
$baseCriteria->add(FileSyncPeer::DC, kDataCenterMgr::getCurrentDcId());
$baseCriteria->addAscendingOrderByColumn(FileSyncPeer::ID);
$baseCriteria->setLimit(self::MAX_FILESYNCS_PER_CHUNK);
$lockedFileSyncs = array();
$lockedFileSyncsSize = 0;
$limitReached = false;
$selectCount = 0;
$lastSelectId = 0;
$done = false;
while (!$done && $selectCount < self::MAX_FILESYNC_QUERIES_PER_CALL) {
// make sure last id is always increasing
if ($lastId <= $lastSelectId) {
KalturaLog::info("last id was decremented {$lastId} <= {$lastSelectId}, stopping");
break;
}
$lastSelectId = $lastId;
// clear the instance pool every once in a while (not clearing every time since
// some objects repeat between selects)
$selectCount++;
if ($selectCount % 5 == 0) {
FileSyncPeer::clearInstancePool();
}
// get a chunk of file syncs
// Note: starting slightly before the last id, because the ids may arrive out of order in the mysql replication
$c = clone $baseCriteria;
$idCriterion = $c->getNewCriterion(FileSyncPeer::ID, $lastId - 100, Criteria::GREATER_THAN);
$idCriterion->addAnd($c->getNewCriterion(FileSyncPeer::ID, $maxId, Criteria::LESS_THAN));
$c->addAnd($idCriterion);
// Note: disabling the criteria because it accumulates more and more criterions, and the status was already explicitly added
// once that bug is fixed, this can be removed
FileSyncPeer::setUseCriteriaFilter(false);
$fileSyncs = FileSyncPeer::doSelect($c);
FileSyncPeer::setUseCriteriaFilter(true);
if (!$fileSyncs) {
$lastId = $maxId;
break;
}
// if we got less than the limit no reason to perform any more queries
if (count($fileSyncs) < self::MAX_FILESYNCS_PER_CHUNK) {
$done = true;
}
$lastFileSync = end($fileSyncs);
$lastId = $lastFileSync->getId();
// filter by source dc
foreach ($fileSyncs as $index => $fileSync) {
if ($fileSync->getOriginalDc() != $sourceDc) {
unset($fileSyncs[$index]);
}
}
// filter by object type / sub type
$fileSyncs = array_filter($fileSyncs, array('FileSyncImportBatchService', 'shouldSyncFileObjectType'));
if (!$fileSyncs) {
continue;
}
// filter by created at
if ($createdAtLessThanOrEqual) {
$firstFileSync = reset($fileSyncs);
$prevLastId = $firstFileSync->getId();
foreach ($fileSyncs as $index => $fileSync) {
if ($fileSync->getCreatedAt(null) > $createdAtLessThanOrEqual) {
$done = true;
unset($fileSyncs[$index]);
if (!is_null($prevLastId)) {
$lastId = $prevLastId;
$prevLastId = null;
}
} else {
$prevLastId = $fileSync->getId();
}
}
if (!$fileSyncs) {
break;
}
}
// get locked file syncs with multi get
$lockKeys = array();
foreach ($fileSyncs as $fileSync) {
$lockKeys[] = self::LOCK_KEY_PREFIX . $fileSync->getId();
}
$lockKeys = $lockCache->get($lockKeys);
// try to lock file syncs
foreach ($fileSyncs as $fileSync) {
$curKey = self::LOCK_KEY_PREFIX . $fileSync->getId();
if (isset($lockKeys[$curKey])) {
KalturaLog::info('file sync ' . $fileSync->getId() . ' already locked');
continue;
}
if (!$lockCache->add($curKey, true, self::LOCK_EXPIRY)) {
KalturaLog::info('failed to lock file sync ' . $fileSync->getId());
continue;
}
KalturaLog::info('locked file sync ' . $fileSync->getId());
// get the original id if not set
if (!$fileSync->getOriginalId()) {
$originalFileSync = self::getOriginalFileSync($fileSync);
if (!$originalFileSync) {
KalturaLog::info('failed to get original file sync for ' . $fileSync->getId());
continue;
}
$fileSync->setOriginalId($originalFileSync->getId());
$fileSync->setCustomDataObj();
// update $fileSync->custom_data so that originalId will be set by fromObject
}
// add to the result set
$lockedFileSyncs[] = $fileSync;
$lockedFileSyncsSize += $fileSync->getFileSize();
if (count($lockedFileSyncs) >= $maxCount || $maxSize && $lockedFileSyncsSize >= $maxSize) {
$lastId = $fileSync->getId();
$limitReached = true;
break;
}
}
if ($limitReached) {
break;
}
}
// update the last id
// Note: it is possible that the last id will go back in case of race condition,
// but the only effect of this is that some file syncs will be scanned again
if (!$initialLastId || $lastId > $initialLastId) {
KalturaLog::info("setting lastId to [{$lastId}] for worker [{$workerId}]");
$keysCache->set(self::LAST_FILESYNC_ID_PREFIX . $workerId, $lastId);
}
// make sure all file syncs have a path
foreach ($lockedFileSyncs as $fileSync) {
if ($fileSync->getFileRoot() && $fileSync->getFilePath()) {
continue;
}
$fileSyncKey = kFileSyncUtils::getKeyForFileSync($fileSync);
list($fileRoot, $realPath) = kPathManager::getFilePathArr($fileSyncKey);
$fileSync->setFileRoot($fileRoot);
$fileSync->setFilePath($realPath);
}
// build the response object
$sourceDc = kDataCenterMgr::getDcById($sourceDc);
$result = new KalturaLockFileSyncsResponse();
$result->fileSyncs = KalturaFileSyncArray::fromDbArray($lockedFileSyncs, $this->getResponseProfile());
$result->limitReached = $limitReached;
$result->dcSecret = $sourceDc["secret"];
$result->baseUrl = isset($sourceDc["fileSyncImportUrl"]) ? $sourceDc["fileSyncImportUrl"] : $sourceDc["url"];
return $result;
}
/**
* @param FileSyncKey $key
* @param StorageProfile $externalStorage
* @return SyncFile
*/
public static function createPendingExternalSyncFileForKey(FileSyncKey $key, StorageProfile $externalStorage, $isDir = false)
{
$externalStorageId = $externalStorage->getId();
KalturaLog::debug("key [{$key}], externalStorage [{$externalStorageId}]");
list($fileRoot, $realPath) = kPathManager::getFilePathArr($key, $externalStorageId);
$c = FileSyncPeer::getCriteriaForFileSyncKey($key);
$c->add(FileSyncPeer::DC, $externalStorageId);
$fileSync = FileSyncPeer::doSelectOne($c);
if (!$fileSync) {
$fileSync = FileSync::createForFileSyncKey($key);
}
$fileSync->setDc($externalStorageId);
$fileSync->setFileRoot($fileRoot);
$fileSync->setFilePath($realPath);
$fileSync->setFileSize(-1);
$fileSync->setStatus(FileSync::FILE_SYNC_STATUS_PENDING);
$fileSync->setOriginal(false);
$fileSync->setIsDir($isDir);
if ($externalStorage->getProtocol() == StorageProfile::STORAGE_KALTURA_DC) {
$fileSync->setFileType(FileSync::FILE_SYNC_FILE_TYPE_FILE);
} else {
$fileSync->setFileType(FileSync::FILE_SYNC_FILE_TYPE_URL);
}
$fileSync->save();
kEventsManager::raiseEvent(new kObjectAddedEvent($fileSync));
return $fileSync;
}
