/** * @action sync * @param int $fileSyncId * @param file $fileData * @return KalturaFileSync */ function syncAction($fileSyncId, $fileData) { $dbFileSync = FileSyncPeer::retrieveByPK($fileSyncId); if (!$dbFileSync) { throw new APIException(APIErrors::INVALID_FILE_SYNC_ID, $fileSyncId); } $key = kFileSyncUtils::getKeyForFileSync($dbFileSync); kFileSyncUtils::moveFromFile($fileData['tmp_name'], $key, false); list($file_root, $real_path) = kPathManager::getFilePathArr($key); $full_path = $file_root . $real_path; chmod($full_path, 0644); if (file_exists($full_path)) { $dbFileSync->setFileRoot($file_root); $dbFileSync->setFilePath($real_path); $dbFileSync->setFileSizeFromPath($full_path); $dbFileSync->setStatus(FileSync::FILE_SYNC_STATUS_READY); } else { $dbFileSync->setFileSize(-1); $dbFileSync->setStatus(FileSync::FILE_SYNC_STATUS_ERROR); } $dbFileSync->save(); $fileSync = new KalturaFileSync(); $fileSync->fromObject($dbFileSync); return $fileSync; }
/** * Update file sync by id * * @action update * @param int $id * @param KalturaFileSync $fileSync * @return KalturaFileSync * * @throws FileSyncErrors::FILESYNC_ID_NOT_FOUND */ function updateAction($id, KalturaFileSync $fileSync) { $dbFileSync = FileSyncPeer::retrieveByPK($id); if (!$dbFileSync) { throw new KalturaAPIException(FileSyncErrors::FILESYNC_ID_NOT_FOUND, $id); } $fileSync->toUpdatableObject($dbFileSync); $dbFileSync->save(); $fileSync = new KalturaFileSync(); $fileSync->fromObject($dbFileSync, $this->getResponseProfile()); return $fileSync; }
/** * batch getExclusiveFileSyncImportJob action allows to get a BatchJob of type FILESYNC_IMPORT * * @action getExclusiveFileSyncImportJobs * @param KalturaExclusiveLockKey $lockKey The unique lock key from the batch-process. Is used for the locking mechanism * @param int $maxExecutionTime The maximum time in seconds the job reguarly take. Is used for the locking mechanism when determining an unexpected termination of a batch-process. * @param int $numberOfJobs The maximum number of jobs to return. * @param KalturaBatchJobFilter $filter Set of rules to fetch only rartial list of jobs * @param int $maxOffset The maximum offset we accept for the distance from the best result. * @return KalturaBatchJobArray * * TODO remove the destFilePath from the job data and get it later using the api, then delete this method */ function getExclusiveFileSyncImportJobsAction(KalturaExclusiveLockKey $lockKey, $maxExecutionTime, $numberOfJobs, KalturaBatchJobFilter $filter = null, $maxOffset = null) { $coreJobs = $this->getExclusiveJobs($lockKey, $maxExecutionTime, $numberOfJobs, $filter, BatchJobType::FILESYNC_IMPORT, $maxOffset); $jobs = KalturaBatchJobArray::fromBatchJobArray($coreJobs); if ($jobs) { foreach ($jobs as $index => $job) { $data = $job->data; // try to get destination path from file sync $fileSyncId = $data->filesyncId; $fileSync = FileSyncPeer::retrieveByPK($fileSyncId); if (!$fileSync) { KalturaLog::err("Failed to load file sync [{$fileSyncId}] aborting job [{$job->id}]"); $dbJob = BatchJobPeer::retrieveByPK($job->id); $dbJob->setMessage("Failed to load file sync [{$fileSyncId}]"); kJobsManager::abortDbBatchJob($dbJob); unset($jobs[$index]); continue; } $fileSyncRoot = $fileSync->getFileRoot(); $fileSyncPath = $fileSync->getFilePath(); if ($fileSyncRoot && $fileSyncPath) { // destination path set on filesync $dest_path = $fileSyncRoot . $fileSyncPath; } else { // not set on filesync - get path from path manager $fileSyncKey = kFileSyncUtils::getKeyForFileSync($fileSync); list($file_root, $real_path) = kPathManager::getFilePathArr($fileSyncKey); $dest_path = $file_root . $real_path; // update filesync on database $fileSync->setFileRoot($file_root); $fileSync->setFilePath($real_path); $fileSync->save(); } // update job data with destination path if needed if (!$data->destFilePath) { $data->destFilePath = $dest_path; $job->data = $data; KalturaLog::log('Updating destination path for job id [$job->id]'); $this->updateJob($job); } if (!is_dir(dirname($dest_path)) && !@mkdir(dirname($dest_path), 0755, true)) { KalturaLog::ERR("Cannot create directory [{$dest_path}] - " . error_get_last()); } } } return $jobs; }
protected function updatedFileSyncImportFailed(BatchJob $dbBatchJob, kFileSyncImportJobData $data, BatchJob $twinJob = null) { // Update relevant filesync as FAILED $fileSyncId = $data->getFilesyncId(); if (!$fileSyncId) { KalturaLog::err('File sync ID not found in job data.'); throw new KalturaAPIException(MultiCentersErrors::INVALID_FILESYNC_ID); } $fileSync = FileSyncPeer::retrieveByPK($fileSyncId); if (!$fileSync) { KalturaLog::err("Invalid filesync record with id [{$fileSyncId}]"); throw new KalturaAPIException(MultiCentersErrors::INVALID_FILESYNC_RECORD, $fileSyncId); } $fileSync->setStatus(FileSync::FILE_SYNC_STATUS_ERROR); $fileSync->setReadyAt(time()); $fileSync->save(); return $dbBatchJob; }
public function updatedJob(BatchJob $dbBatchJob) { switch ($dbBatchJob->getStatus()) { case BatchJob::BATCHJOB_STATUS_FINISHED: $data = $dbBatchJob->getData(); $kontikiFileSync = FileSyncPeer::retrieveByPK($data->getSrcFileSyncId()); /* @var $data kStorageExportJobData */ $asset = assetPeer::retrieveByFileSync($kontikiFileSync); $asset->setTags(KontikiPlugin::KONTIKI_ASSET_TAG); $asset->save(); //Get Kontiki file sync and set the external URL $kontikiFileSync->setFileRoot(""); $kontikiFileSync->setFilePath($data->getContentMoid()); $kontikiFileSync->save(); break; } return true; }
/** * batch getExclusiveFileSyncImportJob action allows to get a BatchJob of type FILESYNC_IMPORT * * @action getExclusiveFileSyncImportJobs * @param KalturaExclusiveLockKey $lockKey The unique lock key from the batch-process. Is used for the locking mechanism * @param int $maxExecutionTime The maximum time in seconds the job reguarly take. Is used for the locking mechanism when determining an unexpected termination of a batch-process. * @param int $numberOfJobs The maximum number of jobs to return. * @param KalturaBatchJobFilter $filter Set of rules to fetch only rartial list of jobs * @return KalturaBatchJobArray */ function getExclusiveFileSyncImportJobsAction(KalturaExclusiveLockKey $lockKey, $maxExecutionTime, $numberOfJobs, KalturaBatchJobFilter $filter = null) { $jobs = $this->getExclusiveJobsAction($lockKey, $maxExecutionTime, $numberOfJobs, $filter, BatchJobType::FILESYNC_IMPORT); if ($jobs) { foreach ($jobs as $job) { $data = $job->data; // try to get destination path from file sync $fileSyncId = $data->filesyncId; $fileSync = FileSyncPeer::retrieveByPK($fileSyncId); if (!$fileSync) { throw new KalturaAPIException(MultiCentersErrors::INVALID_FILESYNC_RECORD, $fileSyncId); } $fileSyncRoot = $fileSync->getFileRoot(); $fileSyncPath = $fileSync->getFilePath(); if ($fileSyncRoot && $fileSyncPath) { // destination path set on filesync $dest_path = $fileSyncRoot . $fileSyncPath; } else { // not set on filesync - get path from path manager $fileSyncKey = kFileSyncUtils::getKeyForFileSync($fileSync); list($file_root, $real_path) = kPathManager::getFilePathArr($fileSyncKey); $dest_path = $file_root . $real_path; // update filesync on database $fileSync->setFileRoot($file_root); $fileSync->setFilePath($real_path); $fileSync->save(); } // update job data with destination path if needed if (!$data->destFilePath) { $data->destFilePath = $dest_path; $job->data = $data; KalturaLog::log('Updating destination path for job id [$job->id]'); $this->updateJob($job); } if (!is_dir(dirname($dest_path)) && !@mkdir(dirname($dest_path), 0755, true)) { KalturaLog::ERR("Cannot create directory [{$dest_path}] - " . error_get_last()); } } } return $jobs; }
/** * @return BaseObject */ public function getObject() { switch ($this->getObjectType()) { case BatchJobObjectType::ENTRY: entryPeer::setUseCriteriaFilter(false); $object = entryPeer::retrieveByPK($this->getObjectId()); entryPeer::setUseCriteriaFilter(true); return $object; case BatchJobObjectType::ASSET: assetPeer::setUseCriteriaFilter(false); $object = assetPeer::retrieveById($this->getObjectId()); assetPeer::setUseCriteriaFilter(true); return $object; case BatchJobObjectType::CATEGORY: categoryPeer::setUseCriteriaFilter(false); $object = categoryPeer::retrieveByPK($this->getObjectId()); categoryPeer::setUseCriteriaFilter(true); return $object; case BatchJobObjectType::FILE_SYNC: FileSyncPeer::setUseCriteriaFilter(false); $object = FileSyncPeer::retrieveByPK($this->getObjectId()); FileSyncPeer::setUseCriteriaFilter(true); return $object; default: // TODO implement IBatchable in relevant plugins return KalturaPluginManager::loadObject('IBatchable', $this->getObjectId()); } return $this->aRootJob; }
/** * mark file as deleted, return deleted version * @param FileSyncKey $key * @param bool $strict * @param bool $fromKalturaDcsOnly * @return string */ public static function deleteSyncFileForKey(FileSyncKey $key, $strict = false, $fromKalturaDcsOnly = false) { if (!$key) { if ($strict) { throw new Exception("Empty key"); } return null; } //Retrieve all file sync for key $c = new Criteria(); $c = FileSyncPeer::getCriteriaForFileSyncKey($key); if ($fromKalturaDcsOnly) { $c->add(FileSyncPeer::FILE_TYPE, FileSync::FILE_SYNC_FILE_TYPE_URL, Criteria::NOT_EQUAL); } $file_sync_list = FileSyncPeer::doSelect($c); foreach ($file_sync_list as $file_sync) { /* @var $fileSync FileSync */ if ($file_sync->getLinkedId()) { $newStatus = FileSync::FILE_SYNC_STATUS_PURGED; self::decrementLinkCountForFileSync(FileSyncPeer::retrieveByPK($file_sync->getLinkedId())); } else { if ($file_sync->getLinkCount() == 0) { $newStatus = FileSync::FILE_SYNC_STATUS_DELETED; } elseif ($file_sync->getLinkCount() > 100) { KalturaLog::notice("The file sync [" . $file_sync->getId() . "] is associated with [" . $file_sync->getLinkCount() . "] links and won't be deleted"); return null; } else { $newStatus = FileSync::FILE_SYNC_STATUS_PURGED; self::convertLinksToFiles($file_sync); } } $file_sync->setStatus($newStatus); $file_sync->save(); } }
/** * @param BatchJob $dbBatchJob * @param kStorageExportJobData $data * @return BatchJob */ public static function handleStorageExportFailed(BatchJob $dbBatchJob, kStorageExportJobData $data) { KalturaLog::debug("Export to storage failed for sync file[" . $data->getSrcFileSyncId() . "]"); $fileSync = FileSyncPeer::retrieveByPK($data->getSrcFileSyncId()); $fileSync->setStatus(FileSync::FILE_SYNC_STATUS_ERROR); $fileSync->save(); return $dbBatchJob; }
protected function restoreFileSyncs(array $fileSyncs) { foreach ($fileSyncs as $fileSync) { $shouldUnDelete = false; if ($fileSync->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_FILE || $fileSync->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_URL) { $shouldUnDelete = true; } else { if ($fileSync->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_LINK) { $linkedId = $fileSync->getLinkedId(); FileSyncPeer::setUseCriteriaFilter(false); $linkedFileSync = FileSyncPeer::retrieveByPK($linkedId); FileSyncPeer::setUseCriteriaFilter(true); if ($linkedFileSync->getStatus() == FileSync::FILE_SYNC_STATUS_READY) { $shouldUnDelete = true; kFileSyncUtils::incrementLinkCountForFileSync($linkedFileSync); } } } if ($shouldUnDelete) { $fileSync->setStatus(FileSync::FILE_SYNC_STATUS_READY); } else { $fileSync->setStatus(FileSync::FILE_SYNC_STATUS_ERROR); } $fileSync->save(); } }
/** * resolve the source filesync when a FileSync input is a LINK * * @param FileSync $file * @return FileSync */ public static function resolve(FileSync $file) { $parent = null; if ($file->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_LINK) { $source_file_sync = FileSyncPeer::retrieveByPK($file->getLinkedId()); if (!$source_file_sync) { return $file; } $parent = self::resolve($source_file_sync); } if (!$parent) { return $file; } else { return $parent; } }
public static function handleStorageDeleteFinished(BatchJob $dbBatchJob, kStorageDeleteJobData $data) { $fileSync = FileSyncPeer::retrieveByPK($data->getSrcFileSyncId()); if (!$fileSync) { KalturaLog::err("FileSync [" . $data->getSrcFileSyncId() . "] not found"); return $dbBatchJob; } $fileSync->setStatus(FileSync::FILE_SYNC_STATUS_DELETED); $fileSync->save(); return $dbBatchJob; }
// find all links that already changed to files $criteriaTemplate = new Criteria(); $criteriaTemplate->add(FileSyncPeer::LINKED_ID, 0, Criteria::GREATER_THAN); $criteriaTemplate->add(FileSyncPeer::FILE_TYPE, FileSync::FILE_SYNC_FILE_TYPE_FILE); $criteriaTemplate->add(FileSyncPeer::STATUS, array(FileSync::FILE_SYNC_STATUS_PURGED, FileSync::FILE_SYNC_STATUS_DELETED), Criteria::NOT_IN); $criteriaTemplate->setLimit($countLimitEachLoop); $fileSyncs = FileSyncPeer::doSelect($criteriaTemplate, $con); while (count($fileSyncs)) { $handledIds = array(); foreach ($fileSyncs as $fileSync) { /* @var $fileSync FileSync */ if (in_array($fileSync->getId(), $handledIds)) { continue; } FileSyncPeer::setUseCriteriaFilter(false); $srcFileSync = FileSyncPeer::retrieveByPK($fileSync->getLinkedId()); FileSyncPeer::setUseCriteriaFilter(true); // find all the links of the same source $linksCriteria = new Criteria(); $linksCriteria->add(FileSyncPeer::DC, $fileSync->getDc()); $linksCriteria->add(FileSyncPeer::FILE_TYPE, FileSync::FILE_SYNC_FILE_TYPE_FILE); $linksCriteria->add(FileSyncPeer::LINKED_ID, $fileSync->getLinkedId()); $linksCriteria->add(FileSyncPeer::STATUS, array(FileSync::FILE_SYNC_STATUS_PURGED, FileSync::FILE_SYNC_STATUS_DELETED), Criteria::NOT_IN); $linksCriteria->addAscendingOrderByColumn(FileSyncPeer::PARTNER_ID); $links = FileSyncPeer::doSelect($linksCriteria, $con); KalturaStatement::setDryRun($dryRun); // choose the first link and convert it to file $firstLink = array_shift($links); /* @var $firstLink FileSync */ if ($firstLink) { $firstLink = cast($firstLink, 'MigrationFileSync');
/** * Update relevant filesync as FAILED. * No need to throw exception if the file sync not found, the job already marked as failed anyway. * * @param BatchJob $dbBatchJob * @param kFileSyncImportJobData $data * @param BatchJob $twinJob * @return BatchJob */ protected function updatedFileSyncImportFailed(BatchJob $dbBatchJob, kFileSyncImportJobData $data, BatchJob $twinJob = null) { $fileSyncId = $data->getFilesyncId(); if (!$fileSyncId) { KalturaLog::err('File sync ID not found in job data.'); return $dbBatchJob; } $fileSync = FileSyncPeer::retrieveByPK($fileSyncId); if (!$fileSync) { KalturaLog::err("Invalid filesync record with id [{$fileSyncId}]"); return $dbBatchJob; } $fileSync->setStatus(FileSync::FILE_SYNC_STATUS_ERROR); $fileSync->save(); return $dbBatchJob; }
function restoreFileSyncByKey(FileSyncKey $fileSyncKey) { KalturaLog::debug("file sync key: {$fileSyncKey}"); /* @var $entryFileSyncKey FileSyncKey */ FileSyncPeer::setUseCriteriaFilter(false); $fileSyncs = FileSyncPeer::retrieveAllByFileSyncKey($fileSyncKey); FileSyncPeer::setUseCriteriaFilter(true); foreach ($fileSyncs as $fileSync) { if ($fileSync->getStatus() == FileSync::FILE_SYNC_STATUS_DELETED) { $shouldUnDelete = false; if ($fileSync->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_FILE || $fileSync->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_URL) { if (file_exists($fileSync->getFullPath())) { $shouldUnDelete = true; } } else { if ($fileSync->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_LINK) { $linkedId = $fileSync->getLinkedId(); FileSyncPeer::setUseCriteriaFilter(false); $linkedFileSync = FileSyncPeer::retrieveByPK($linkedId); FileSyncPeer::setUseCriteriaFilter(true); if ($linkedFileSync->getStatus() == FileSync::FILE_SYNC_STATUS_DELETED && file_exists($linkedFileSync->getFullPath())) { $shouldUnDelete = true; } } else { if ($fileSync->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_CACHE) { $shouldUnDelete = false; } } } if ($shouldUnDelete) { $fileSync->setStatus(FileSync::FILE_SYNC_STATUS_READY); } else { $fileSync->setStatus(FileSync::FILE_SYNC_STATUS_ERROR); } } $fileSync->save(); } }
public static function handleStorageDeleteFinished(BatchJob $dbBatchJob, kStorageDeleteJobData $data) { KalturaLog::debug("Remote storage file deletion finished for fileysnc ID:[ " . $data->getSrcFileSyncId() . "]"); $fileSync = FileSyncPeer::retrieveByPK($data->getSrcFileSyncId()); $fileSync->setStatus(FileSync::FILE_SYNC_STATUS_DELETED); $fileSync->save(); return $dbBatchJob; }