/**
* @action sync
* @param int $fileSyncId
* @param file $fileData
* @return KalturaFileSync
*/
function syncAction($fileSyncId, $fileData)
{
$dbFileSync = FileSyncPeer::retrieveByPK($fileSyncId);
if (!$dbFileSync) {
throw new APIException(APIErrors::INVALID_FILE_SYNC_ID, $fileSyncId);
}
$key = kFileSyncUtils::getKeyForFileSync($dbFileSync);
kFileSyncUtils::moveFromFile($fileData['tmp_name'], $key, false);
list($file_root, $real_path) = kPathManager::getFilePathArr($key);
$full_path = $file_root . $real_path;
chmod($full_path, 0644);
if (file_exists($full_path)) {
$dbFileSync->setFileRoot($file_root);
$dbFileSync->setFilePath($real_path);
$dbFileSync->setFileSizeFromPath($full_path);
$dbFileSync->setStatus(FileSync::FILE_SYNC_STATUS_READY);
} else {
$dbFileSync->setFileSize(-1);
$dbFileSync->setStatus(FileSync::FILE_SYNC_STATUS_ERROR);
}
$dbFileSync->save();
$fileSync = new KalturaFileSync();
$fileSync->fromObject($dbFileSync);
return $fileSync;
}
/**
* Update file sync by id
*
* @action update
* @param int $id
* @param KalturaFileSync $fileSync
* @return KalturaFileSync
*
* @throws FileSyncErrors::FILESYNC_ID_NOT_FOUND
*/
function updateAction($id, KalturaFileSync $fileSync)
{
$dbFileSync = FileSyncPeer::retrieveByPK($id);
if (!$dbFileSync) {
throw new KalturaAPIException(FileSyncErrors::FILESYNC_ID_NOT_FOUND, $id);
}
$fileSync->toUpdatableObject($dbFileSync);
$dbFileSync->save();
$fileSync = new KalturaFileSync();
$fileSync->fromObject($dbFileSync, $this->getResponseProfile());
return $fileSync;
}
/**
* batch getExclusiveFileSyncImportJob action allows to get a BatchJob of type FILESYNC_IMPORT
*
* @action getExclusiveFileSyncImportJobs
* @param KalturaExclusiveLockKey $lockKey The unique lock key from the batch-process. Is used for the locking mechanism
* @param int $maxExecutionTime The maximum time in seconds the job reguarly take. Is used for the locking mechanism when determining an unexpected termination of a batch-process.
* @param int $numberOfJobs The maximum number of jobs to return.
* @param KalturaBatchJobFilter $filter Set of rules to fetch only rartial list of jobs
* @param int $maxOffset The maximum offset we accept for the distance from the best result.
* @return KalturaBatchJobArray
*
* TODO remove the destFilePath from the job data and get it later using the api, then delete this method
*/
function getExclusiveFileSyncImportJobsAction(KalturaExclusiveLockKey $lockKey, $maxExecutionTime, $numberOfJobs, KalturaBatchJobFilter $filter = null, $maxOffset = null)
{
$coreJobs = $this->getExclusiveJobs($lockKey, $maxExecutionTime, $numberOfJobs, $filter, BatchJobType::FILESYNC_IMPORT, $maxOffset);
$jobs = KalturaBatchJobArray::fromBatchJobArray($coreJobs);
if ($jobs) {
foreach ($jobs as $index => $job) {
$data = $job->data;
// try to get destination path from file sync
$fileSyncId = $data->filesyncId;
$fileSync = FileSyncPeer::retrieveByPK($fileSyncId);
if (!$fileSync) {
KalturaLog::err("Failed to load file sync [{$fileSyncId}] aborting job [{$job->id}]");
$dbJob = BatchJobPeer::retrieveByPK($job->id);
$dbJob->setMessage("Failed to load file sync [{$fileSyncId}]");
kJobsManager::abortDbBatchJob($dbJob);
unset($jobs[$index]);
continue;
}
$fileSyncRoot = $fileSync->getFileRoot();
$fileSyncPath = $fileSync->getFilePath();
if ($fileSyncRoot && $fileSyncPath) {
// destination path set on filesync
$dest_path = $fileSyncRoot . $fileSyncPath;
} else {
// not set on filesync - get path from path manager
$fileSyncKey = kFileSyncUtils::getKeyForFileSync($fileSync);
list($file_root, $real_path) = kPathManager::getFilePathArr($fileSyncKey);
$dest_path = $file_root . $real_path;
// update filesync on database
$fileSync->setFileRoot($file_root);
$fileSync->setFilePath($real_path);
$fileSync->save();
}
// update job data with destination path if needed
if (!$data->destFilePath) {
$data->destFilePath = $dest_path;
$job->data = $data;
KalturaLog::log('Updating destination path for job id [$job->id]');
$this->updateJob($job);
}
if (!is_dir(dirname($dest_path)) && !@mkdir(dirname($dest_path), 0755, true)) {
KalturaLog::ERR("Cannot create directory [{$dest_path}] - " . error_get_last());
}
}
}
return $jobs;
}
protected function updatedFileSyncImportFailed(BatchJob $dbBatchJob, kFileSyncImportJobData $data, BatchJob $twinJob = null)
{
// Update relevant filesync as FAILED
$fileSyncId = $data->getFilesyncId();
if (!$fileSyncId) {
KalturaLog::err('File sync ID not found in job data.');
throw new KalturaAPIException(MultiCentersErrors::INVALID_FILESYNC_ID);
}
$fileSync = FileSyncPeer::retrieveByPK($fileSyncId);
if (!$fileSync) {
KalturaLog::err("Invalid filesync record with id [{$fileSyncId}]");
throw new KalturaAPIException(MultiCentersErrors::INVALID_FILESYNC_RECORD, $fileSyncId);
}
$fileSync->setStatus(FileSync::FILE_SYNC_STATUS_ERROR);
$fileSync->setReadyAt(time());
$fileSync->save();
return $dbBatchJob;
}
public function updatedJob(BatchJob $dbBatchJob)
{
switch ($dbBatchJob->getStatus()) {
case BatchJob::BATCHJOB_STATUS_FINISHED:
$data = $dbBatchJob->getData();
$kontikiFileSync = FileSyncPeer::retrieveByPK($data->getSrcFileSyncId());
/* @var $data kStorageExportJobData */
$asset = assetPeer::retrieveByFileSync($kontikiFileSync);
$asset->setTags(KontikiPlugin::KONTIKI_ASSET_TAG);
$asset->save();
//Get Kontiki file sync and set the external URL
$kontikiFileSync->setFileRoot("");
$kontikiFileSync->setFilePath($data->getContentMoid());
$kontikiFileSync->save();
break;
}
return true;
}
/**
* batch getExclusiveFileSyncImportJob action allows to get a BatchJob of type FILESYNC_IMPORT
*
* @action getExclusiveFileSyncImportJobs
* @param KalturaExclusiveLockKey $lockKey The unique lock key from the batch-process. Is used for the locking mechanism
* @param int $maxExecutionTime The maximum time in seconds the job reguarly take. Is used for the locking mechanism when determining an unexpected termination of a batch-process.
* @param int $numberOfJobs The maximum number of jobs to return.
* @param KalturaBatchJobFilter $filter Set of rules to fetch only rartial list of jobs
* @return KalturaBatchJobArray
*/
function getExclusiveFileSyncImportJobsAction(KalturaExclusiveLockKey $lockKey, $maxExecutionTime, $numberOfJobs, KalturaBatchJobFilter $filter = null)
{
$jobs = $this->getExclusiveJobsAction($lockKey, $maxExecutionTime, $numberOfJobs, $filter, BatchJobType::FILESYNC_IMPORT);
if ($jobs) {
foreach ($jobs as $job) {
$data = $job->data;
// try to get destination path from file sync
$fileSyncId = $data->filesyncId;
$fileSync = FileSyncPeer::retrieveByPK($fileSyncId);
if (!$fileSync) {
throw new KalturaAPIException(MultiCentersErrors::INVALID_FILESYNC_RECORD, $fileSyncId);
}
$fileSyncRoot = $fileSync->getFileRoot();
$fileSyncPath = $fileSync->getFilePath();
if ($fileSyncRoot && $fileSyncPath) {
// destination path set on filesync
$dest_path = $fileSyncRoot . $fileSyncPath;
} else {
// not set on filesync - get path from path manager
$fileSyncKey = kFileSyncUtils::getKeyForFileSync($fileSync);
list($file_root, $real_path) = kPathManager::getFilePathArr($fileSyncKey);
$dest_path = $file_root . $real_path;
// update filesync on database
$fileSync->setFileRoot($file_root);
$fileSync->setFilePath($real_path);
$fileSync->save();
}
// update job data with destination path if needed
if (!$data->destFilePath) {
$data->destFilePath = $dest_path;
$job->data = $data;
KalturaLog::log('Updating destination path for job id [$job->id]');
$this->updateJob($job);
}
if (!is_dir(dirname($dest_path)) && !@mkdir(dirname($dest_path), 0755, true)) {
KalturaLog::ERR("Cannot create directory [{$dest_path}] - " . error_get_last());
}
}
}
return $jobs;
}
/**
* @return BaseObject
*/
public function getObject()
{
switch ($this->getObjectType()) {
case BatchJobObjectType::ENTRY:
entryPeer::setUseCriteriaFilter(false);
$object = entryPeer::retrieveByPK($this->getObjectId());
entryPeer::setUseCriteriaFilter(true);
return $object;
case BatchJobObjectType::ASSET:
assetPeer::setUseCriteriaFilter(false);
$object = assetPeer::retrieveById($this->getObjectId());
assetPeer::setUseCriteriaFilter(true);
return $object;
case BatchJobObjectType::CATEGORY:
categoryPeer::setUseCriteriaFilter(false);
$object = categoryPeer::retrieveByPK($this->getObjectId());
categoryPeer::setUseCriteriaFilter(true);
return $object;
case BatchJobObjectType::FILE_SYNC:
FileSyncPeer::setUseCriteriaFilter(false);
$object = FileSyncPeer::retrieveByPK($this->getObjectId());
FileSyncPeer::setUseCriteriaFilter(true);
return $object;
default:
// TODO implement IBatchable in relevant plugins
return KalturaPluginManager::loadObject('IBatchable', $this->getObjectId());
}
return $this->aRootJob;
}
/**
* mark file as deleted, return deleted version
* @param FileSyncKey $key
* @param bool $strict
* @param bool $fromKalturaDcsOnly
* @return string
*/
public static function deleteSyncFileForKey(FileSyncKey $key, $strict = false, $fromKalturaDcsOnly = false)
{
if (!$key) {
if ($strict) {
throw new Exception("Empty key");
}
return null;
}
//Retrieve all file sync for key
$c = new Criteria();
$c = FileSyncPeer::getCriteriaForFileSyncKey($key);
if ($fromKalturaDcsOnly) {
$c->add(FileSyncPeer::FILE_TYPE, FileSync::FILE_SYNC_FILE_TYPE_URL, Criteria::NOT_EQUAL);
}
$file_sync_list = FileSyncPeer::doSelect($c);
foreach ($file_sync_list as $file_sync) {
/* @var $fileSync FileSync */
if ($file_sync->getLinkedId()) {
$newStatus = FileSync::FILE_SYNC_STATUS_PURGED;
self::decrementLinkCountForFileSync(FileSyncPeer::retrieveByPK($file_sync->getLinkedId()));
} else {
if ($file_sync->getLinkCount() == 0) {
$newStatus = FileSync::FILE_SYNC_STATUS_DELETED;
} elseif ($file_sync->getLinkCount() > 100) {
KalturaLog::notice("The file sync [" . $file_sync->getId() . "] is associated with [" . $file_sync->getLinkCount() . "] links and won't be deleted");
return null;
} else {
$newStatus = FileSync::FILE_SYNC_STATUS_PURGED;
self::convertLinksToFiles($file_sync);
}
}
$file_sync->setStatus($newStatus);
$file_sync->save();
}
}
/**
* @param BatchJob $dbBatchJob
* @param kStorageExportJobData $data
* @return BatchJob
*/
public static function handleStorageExportFailed(BatchJob $dbBatchJob, kStorageExportJobData $data)
{
KalturaLog::debug("Export to storage failed for sync file[" . $data->getSrcFileSyncId() . "]");
$fileSync = FileSyncPeer::retrieveByPK($data->getSrcFileSyncId());
$fileSync->setStatus(FileSync::FILE_SYNC_STATUS_ERROR);
$fileSync->save();
return $dbBatchJob;
}
protected function restoreFileSyncs(array $fileSyncs)
{
foreach ($fileSyncs as $fileSync) {
$shouldUnDelete = false;
if ($fileSync->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_FILE || $fileSync->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_URL) {
$shouldUnDelete = true;
} else {
if ($fileSync->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_LINK) {
$linkedId = $fileSync->getLinkedId();
FileSyncPeer::setUseCriteriaFilter(false);
$linkedFileSync = FileSyncPeer::retrieveByPK($linkedId);
FileSyncPeer::setUseCriteriaFilter(true);
if ($linkedFileSync->getStatus() == FileSync::FILE_SYNC_STATUS_READY) {
$shouldUnDelete = true;
kFileSyncUtils::incrementLinkCountForFileSync($linkedFileSync);
}
}
}
if ($shouldUnDelete) {
$fileSync->setStatus(FileSync::FILE_SYNC_STATUS_READY);
} else {
$fileSync->setStatus(FileSync::FILE_SYNC_STATUS_ERROR);
}
$fileSync->save();
}
}
/**
* resolve the source filesync when a FileSync input is a LINK
*
* @param FileSync $file
* @return FileSync
*/
public static function resolve(FileSync $file)
{
$parent = null;
if ($file->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_LINK) {
$source_file_sync = FileSyncPeer::retrieveByPK($file->getLinkedId());
if (!$source_file_sync) {
return $file;
}
$parent = self::resolve($source_file_sync);
}
if (!$parent) {
return $file;
} else {
return $parent;
}
}
public static function handleStorageDeleteFinished(BatchJob $dbBatchJob, kStorageDeleteJobData $data)
{
$fileSync = FileSyncPeer::retrieveByPK($data->getSrcFileSyncId());
if (!$fileSync) {
KalturaLog::err("FileSync [" . $data->getSrcFileSyncId() . "] not found");
return $dbBatchJob;
}
$fileSync->setStatus(FileSync::FILE_SYNC_STATUS_DELETED);
$fileSync->save();
return $dbBatchJob;
}
// find all links that already changed to files
$criteriaTemplate = new Criteria();
$criteriaTemplate->add(FileSyncPeer::LINKED_ID, 0, Criteria::GREATER_THAN);
$criteriaTemplate->add(FileSyncPeer::FILE_TYPE, FileSync::FILE_SYNC_FILE_TYPE_FILE);
$criteriaTemplate->add(FileSyncPeer::STATUS, array(FileSync::FILE_SYNC_STATUS_PURGED, FileSync::FILE_SYNC_STATUS_DELETED), Criteria::NOT_IN);
$criteriaTemplate->setLimit($countLimitEachLoop);
$fileSyncs = FileSyncPeer::doSelect($criteriaTemplate, $con);
while (count($fileSyncs)) {
$handledIds = array();
foreach ($fileSyncs as $fileSync) {
/* @var $fileSync FileSync */
if (in_array($fileSync->getId(), $handledIds)) {
continue;
}
FileSyncPeer::setUseCriteriaFilter(false);
$srcFileSync = FileSyncPeer::retrieveByPK($fileSync->getLinkedId());
FileSyncPeer::setUseCriteriaFilter(true);
// find all the links of the same source
$linksCriteria = new Criteria();
$linksCriteria->add(FileSyncPeer::DC, $fileSync->getDc());
$linksCriteria->add(FileSyncPeer::FILE_TYPE, FileSync::FILE_SYNC_FILE_TYPE_FILE);
$linksCriteria->add(FileSyncPeer::LINKED_ID, $fileSync->getLinkedId());
$linksCriteria->add(FileSyncPeer::STATUS, array(FileSync::FILE_SYNC_STATUS_PURGED, FileSync::FILE_SYNC_STATUS_DELETED), Criteria::NOT_IN);
$linksCriteria->addAscendingOrderByColumn(FileSyncPeer::PARTNER_ID);
$links = FileSyncPeer::doSelect($linksCriteria, $con);
KalturaStatement::setDryRun($dryRun);
// choose the first link and convert it to file
$firstLink = array_shift($links);
/* @var $firstLink FileSync */
if ($firstLink) {
$firstLink = cast($firstLink, 'MigrationFileSync');
/**
* Update relevant filesync as FAILED.
* No need to throw exception if the file sync not found, the job already marked as failed anyway.
*
* @param BatchJob $dbBatchJob
* @param kFileSyncImportJobData $data
* @param BatchJob $twinJob
* @return BatchJob
*/
protected function updatedFileSyncImportFailed(BatchJob $dbBatchJob, kFileSyncImportJobData $data, BatchJob $twinJob = null)
{
$fileSyncId = $data->getFilesyncId();
if (!$fileSyncId) {
KalturaLog::err('File sync ID not found in job data.');
return $dbBatchJob;
}
$fileSync = FileSyncPeer::retrieveByPK($fileSyncId);
if (!$fileSync) {
KalturaLog::err("Invalid filesync record with id [{$fileSyncId}]");
return $dbBatchJob;
}
$fileSync->setStatus(FileSync::FILE_SYNC_STATUS_ERROR);
$fileSync->save();
return $dbBatchJob;
}
function restoreFileSyncByKey(FileSyncKey $fileSyncKey)
{
KalturaLog::debug("file sync key: {$fileSyncKey}");
/* @var $entryFileSyncKey FileSyncKey */
FileSyncPeer::setUseCriteriaFilter(false);
$fileSyncs = FileSyncPeer::retrieveAllByFileSyncKey($fileSyncKey);
FileSyncPeer::setUseCriteriaFilter(true);
foreach ($fileSyncs as $fileSync) {
if ($fileSync->getStatus() == FileSync::FILE_SYNC_STATUS_DELETED) {
$shouldUnDelete = false;
if ($fileSync->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_FILE || $fileSync->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_URL) {
if (file_exists($fileSync->getFullPath())) {
$shouldUnDelete = true;
}
} else {
if ($fileSync->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_LINK) {
$linkedId = $fileSync->getLinkedId();
FileSyncPeer::setUseCriteriaFilter(false);
$linkedFileSync = FileSyncPeer::retrieveByPK($linkedId);
FileSyncPeer::setUseCriteriaFilter(true);
if ($linkedFileSync->getStatus() == FileSync::FILE_SYNC_STATUS_DELETED && file_exists($linkedFileSync->getFullPath())) {
$shouldUnDelete = true;
}
} else {
if ($fileSync->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_CACHE) {
$shouldUnDelete = false;
}
}
}
if ($shouldUnDelete) {
$fileSync->setStatus(FileSync::FILE_SYNC_STATUS_READY);
} else {
$fileSync->setStatus(FileSync::FILE_SYNC_STATUS_ERROR);
}
}
$fileSync->save();
}
}
public static function handleStorageDeleteFinished(BatchJob $dbBatchJob, kStorageDeleteJobData $data)
{
KalturaLog::debug("Remote storage file deletion finished for fileysnc ID:[ " . $data->getSrcFileSyncId() . "]");
$fileSync = FileSyncPeer::retrieveByPK($data->getSrcFileSyncId());
$fileSync->setStatus(FileSync::FILE_SYNC_STATUS_DELETED);
$fileSync->save();
return $dbBatchJob;
}
