/** * @action sync * @param int $fileSyncId * @param file $fileData * @return KalturaFileSync */ function syncAction($fileSyncId, $fileData) { $dbFileSync = FileSyncPeer::retrieveByPK($fileSyncId); if (!$dbFileSync) { throw new APIException(APIErrors::INVALID_FILE_SYNC_ID, $fileSyncId); } $key = kFileSyncUtils::getKeyForFileSync($dbFileSync); kFileSyncUtils::moveFromFile($fileData['tmp_name'], $key, false); list($file_root, $real_path) = kPathManager::getFilePathArr($key); $full_path = $file_root . $real_path; chmod($full_path, 0644); if (file_exists($full_path)) { $dbFileSync->setFileRoot($file_root); $dbFileSync->setFilePath($real_path); $dbFileSync->setFileSizeFromPath($full_path); $dbFileSync->setStatus(FileSync::FILE_SYNC_STATUS_READY); } else { $dbFileSync->setFileSize(-1); $dbFileSync->setStatus(FileSync::FILE_SYNC_STATUS_ERROR); } $dbFileSync->save(); $fileSync = new KalturaFileSync(); $fileSync->fromObject($dbFileSync); return $fileSync; }
/** * @param FileSync $fileSync * @return string */ protected function doGetFileSyncUrl(FileSync $fileSync) { $url = parent::doGetFileSyncUrl($fileSync); $url = preg_replace('/^mp4:(\\/)*/', 'mp4:', $url); if ($this->protocol == StorageProfile::PLAY_FORMAT_HTTP) { $syncKey = kFileSyncUtils::getKeyForFileSync($fileSync); $url = $this->addEcSeek($url, $syncKey); } return $url; }
/** * @param string $id * @param int $type */ protected function syncableDeleted($id, $type) { $c = new Criteria(); $c->add(FileSyncPeer::OBJECT_ID, $id); $c->add(FileSyncPeer::OBJECT_TYPE, $type); $c->add(FileSyncPeer::STATUS, array(FileSync::FILE_SYNC_STATUS_PURGED, FileSync::FILE_SYNC_STATUS_DELETED), Criteria::NOT_IN); $fileSyncs = FileSyncPeer::doSelect($c); foreach ($fileSyncs as $fileSync) { $key = kFileSyncUtils::getKeyForFileSync($fileSync); kFileSyncUtils::deleteSyncFileForKey($key); } }
/** * batch getExclusiveFileSyncImportJob action allows to get a BatchJob of type FILESYNC_IMPORT * * @action getExclusiveFileSyncImportJobs * @param KalturaExclusiveLockKey $lockKey The unique lock key from the batch-process. Is used for the locking mechanism * @param int $maxExecutionTime The maximum time in seconds the job reguarly take. Is used for the locking mechanism when determining an unexpected termination of a batch-process. * @param int $numberOfJobs The maximum number of jobs to return. * @param KalturaBatchJobFilter $filter Set of rules to fetch only rartial list of jobs * @param int $maxOffset The maximum offset we accept for the distance from the best result. * @return KalturaBatchJobArray * * TODO remove the destFilePath from the job data and get it later using the api, then delete this method */ function getExclusiveFileSyncImportJobsAction(KalturaExclusiveLockKey $lockKey, $maxExecutionTime, $numberOfJobs, KalturaBatchJobFilter $filter = null, $maxOffset = null) { $coreJobs = $this->getExclusiveJobs($lockKey, $maxExecutionTime, $numberOfJobs, $filter, BatchJobType::FILESYNC_IMPORT, $maxOffset); $jobs = KalturaBatchJobArray::fromBatchJobArray($coreJobs); if ($jobs) { foreach ($jobs as $index => $job) { $data = $job->data; // try to get destination path from file sync $fileSyncId = $data->filesyncId; $fileSync = FileSyncPeer::retrieveByPK($fileSyncId); if (!$fileSync) { KalturaLog::err("Failed to load file sync [{$fileSyncId}] aborting job [{$job->id}]"); $dbJob = BatchJobPeer::retrieveByPK($job->id); $dbJob->setMessage("Failed to load file sync [{$fileSyncId}]"); kJobsManager::abortDbBatchJob($dbJob); unset($jobs[$index]); continue; } $fileSyncRoot = $fileSync->getFileRoot(); $fileSyncPath = $fileSync->getFilePath(); if ($fileSyncRoot && $fileSyncPath) { // destination path set on filesync $dest_path = $fileSyncRoot . $fileSyncPath; } else { // not set on filesync - get path from path manager $fileSyncKey = kFileSyncUtils::getKeyForFileSync($fileSync); list($file_root, $real_path) = kPathManager::getFilePathArr($fileSyncKey); $dest_path = $file_root . $real_path; // update filesync on database $fileSync->setFileRoot($file_root); $fileSync->setFilePath($real_path); $fileSync->save(); } // update job data with destination path if needed if (!$data->destFilePath) { $data->destFilePath = $dest_path; $job->data = $data; KalturaLog::log('Updating destination path for job id [$job->id]'); $this->updateJob($job); } if (!is_dir(dirname($dest_path)) && !@mkdir(dirname($dest_path), 0755, true)) { KalturaLog::ERR("Cannot create directory [{$dest_path}] - " . error_get_last()); } } } return $jobs; }
/** * Update relevant filesync as READY * * @param BatchJob $dbBatchJob * @param kFileSyncImportJobData $data * @param BatchJob $twinJob * @throws KalturaAPIException * @return BatchJob */ protected function updatedFileSyncImportFinished(BatchJob $dbBatchJob, kFileSyncImportJobData $data, BatchJob $twinJob = null) { $fileSyncId = $data->getFilesyncId(); if (!$fileSyncId) { KalturaLog::err('File sync ID not found in job data.'); throw new KalturaAPIException(MultiCentersErrors::INVALID_FILESYNC_ID); } $fileSync = FileSyncPeer::retrieveByPK($fileSyncId); if (!$fileSync) { KalturaLog::err("Invalid filesync record with id [{$fileSyncId}]"); throw new KalturaAPIException(MultiCentersErrors::INVALID_FILESYNC_RECORD, $fileSyncId); } $fileSync->setStatus(FileSync::FILE_SYNC_STATUS_READY); $fileSync->setFileSizeFromPath(kFileSyncUtils::getLocalFilePathForKey(kFileSyncUtils::getKeyForFileSync($fileSync))); $fileSync->save(); return $dbBatchJob; }
/** * batch getExclusiveFileSyncImportJob action allows to get a BatchJob of type FILESYNC_IMPORT * * @action getExclusiveFileSyncImportJobs * @param KalturaExclusiveLockKey $lockKey The unique lock key from the batch-process. Is used for the locking mechanism * @param int $maxExecutionTime The maximum time in seconds the job reguarly take. Is used for the locking mechanism when determining an unexpected termination of a batch-process. * @param int $numberOfJobs The maximum number of jobs to return. * @param KalturaBatchJobFilter $filter Set of rules to fetch only rartial list of jobs * @return KalturaBatchJobArray */ function getExclusiveFileSyncImportJobsAction(KalturaExclusiveLockKey $lockKey, $maxExecutionTime, $numberOfJobs, KalturaBatchJobFilter $filter = null) { $jobs = $this->getExclusiveJobsAction($lockKey, $maxExecutionTime, $numberOfJobs, $filter, BatchJobType::FILESYNC_IMPORT); if ($jobs) { foreach ($jobs as $job) { $data = $job->data; // try to get destination path from file sync $fileSyncId = $data->filesyncId; $fileSync = FileSyncPeer::retrieveByPK($fileSyncId); if (!$fileSync) { throw new KalturaAPIException(MultiCentersErrors::INVALID_FILESYNC_RECORD, $fileSyncId); } $fileSyncRoot = $fileSync->getFileRoot(); $fileSyncPath = $fileSync->getFilePath(); if ($fileSyncRoot && $fileSyncPath) { // destination path set on filesync $dest_path = $fileSyncRoot . $fileSyncPath; } else { // not set on filesync - get path from path manager $fileSyncKey = kFileSyncUtils::getKeyForFileSync($fileSync); list($file_root, $real_path) = kPathManager::getFilePathArr($fileSyncKey); $dest_path = $file_root . $real_path; // update filesync on database $fileSync->setFileRoot($file_root); $fileSync->setFilePath($real_path); $fileSync->save(); } // update job data with destination path if needed if (!$data->destFilePath) { $data->destFilePath = $dest_path; $job->data = $data; KalturaLog::log('Updating destination path for job id [$job->id]'); $this->updateJob($job); } if (!is_dir(dirname($dest_path)) && !@mkdir(dirname($dest_path), 0755, true)) { KalturaLog::ERR("Cannot create directory [{$dest_path}] - " . error_get_last()); } } } return $jobs; }
/** * batch lockPendingFileSyncs action locks file syncs for import by the file sync periodic worker * * @action lockPendingFileSyncs * @param KalturaFileSyncFilter $filter * @param int $workerId The id of the file sync import worker * @param int $sourceDc The id of the DC from which the file syncs should be pulled * @param int $maxCount The maximum number of file syncs that should be returned * @param int $maxSize The maximum total size of file syncs that should be returned, this limit may be exceeded by one file sync * @return KalturaLockFileSyncsResponse */ function lockPendingFileSyncsAction(KalturaFileSyncFilter $filter, $workerId, $sourceDc, $maxCount, $maxSize = null) { // need to explicitly disable the cache since this action may not perform any queries kApiCache::disableConditionalCache(); // for dual dc deployments, if source dc is not specified, set it to the remote dc if ($sourceDc < 0) { $sourceDc = 1 - kDataCenterMgr::getCurrentDcId(); } // get caches $keysCache = kCacheManager::getSingleLayerCache(kCacheManager::CACHE_TYPE_QUERY_CACHE_KEYS); if (!$keysCache) { throw new KalturaAPIException(MultiCentersErrors::GET_KEYS_CACHE_FAILED); } $lockCache = kCacheManager::getSingleLayerCache(kCacheManager::CACHE_TYPE_LOCK_KEYS); if (!$lockCache) { throw new KalturaAPIException(MultiCentersErrors::GET_LOCK_CACHE_FAILED); } // get the max id / last id $maxId = $keysCache->get(self::MAX_FILESYNC_ID_PREFIX . $sourceDc); if (!$maxId) { throw new KalturaAPIException(MultiCentersErrors::GET_MAX_FILESYNC_ID_FAILED, $sourceDc); } $initialLastId = $keysCache->get(self::LAST_FILESYNC_ID_PREFIX . $workerId); KalturaLog::info("got lastId [{$initialLastId}] for worker [{$workerId}]"); $lastId = $initialLastId ? $initialLastId : $maxId; // created at less than handled explicitly $createdAtLessThanOrEqual = $filter->createdAtLessThanOrEqual; $filter->createdAtLessThanOrEqual = null; // build the criteria $fileSyncFilter = new FileSyncFilter(); $filter->toObject($fileSyncFilter); $baseCriteria = new Criteria(); $fileSyncFilter->attachToCriteria($baseCriteria); $baseCriteria->add(FileSyncPeer::STATUS, FileSync::FILE_SYNC_STATUS_PENDING); $baseCriteria->add(FileSyncPeer::FILE_TYPE, FileSync::FILE_SYNC_FILE_TYPE_FILE); $baseCriteria->add(FileSyncPeer::DC, kDataCenterMgr::getCurrentDcId()); $baseCriteria->addAscendingOrderByColumn(FileSyncPeer::ID); $baseCriteria->setLimit(self::MAX_FILESYNCS_PER_CHUNK); $lockedFileSyncs = array(); $lockedFileSyncsSize = 0; $limitReached = false; $selectCount = 0; $lastSelectId = 0; $done = false; while (!$done && $selectCount < self::MAX_FILESYNC_QUERIES_PER_CALL) { // make sure last id is always increasing if ($lastId <= $lastSelectId) { KalturaLog::info("last id was decremented {$lastId} <= {$lastSelectId}, stopping"); break; } $lastSelectId = $lastId; // clear the instance pool every once in a while (not clearing every time since // some objects repeat between selects) $selectCount++; if ($selectCount % 5 == 0) { FileSyncPeer::clearInstancePool(); } // get a chunk of file syncs // Note: starting slightly before the last id, because the ids may arrive out of order in the mysql replication $c = clone $baseCriteria; $idCriterion = $c->getNewCriterion(FileSyncPeer::ID, $lastId - 100, Criteria::GREATER_THAN); $idCriterion->addAnd($c->getNewCriterion(FileSyncPeer::ID, $maxId, Criteria::LESS_THAN)); $c->addAnd($idCriterion); // Note: disabling the criteria because it accumulates more and more criterions, and the status was already explicitly added // once that bug is fixed, this can be removed FileSyncPeer::setUseCriteriaFilter(false); $fileSyncs = FileSyncPeer::doSelect($c); FileSyncPeer::setUseCriteriaFilter(true); if (!$fileSyncs) { $lastId = $maxId; break; } // if we got less than the limit no reason to perform any more queries if (count($fileSyncs) < self::MAX_FILESYNCS_PER_CHUNK) { $done = true; } $lastFileSync = end($fileSyncs); $lastId = $lastFileSync->getId(); // filter by source dc foreach ($fileSyncs as $index => $fileSync) { if ($fileSync->getOriginalDc() != $sourceDc) { unset($fileSyncs[$index]); } } // filter by object type / sub type $fileSyncs = array_filter($fileSyncs, array('FileSyncImportBatchService', 'shouldSyncFileObjectType')); if (!$fileSyncs) { continue; } // filter by created at if ($createdAtLessThanOrEqual) { $firstFileSync = reset($fileSyncs); $prevLastId = $firstFileSync->getId(); foreach ($fileSyncs as $index => $fileSync) { if ($fileSync->getCreatedAt(null) > $createdAtLessThanOrEqual) { $done = true; unset($fileSyncs[$index]); if (!is_null($prevLastId)) { $lastId = $prevLastId; $prevLastId = null; } } else { $prevLastId = $fileSync->getId(); } } if (!$fileSyncs) { break; } } // get locked file syncs with multi get $lockKeys = array(); foreach ($fileSyncs as $fileSync) { $lockKeys[] = self::LOCK_KEY_PREFIX . $fileSync->getId(); } $lockKeys = $lockCache->get($lockKeys); // try to lock file syncs foreach ($fileSyncs as $fileSync) { $curKey = self::LOCK_KEY_PREFIX . $fileSync->getId(); if (isset($lockKeys[$curKey])) { KalturaLog::info('file sync ' . $fileSync->getId() . ' already locked'); continue; } if (!$lockCache->add($curKey, true, self::LOCK_EXPIRY)) { KalturaLog::info('failed to lock file sync ' . $fileSync->getId()); continue; } KalturaLog::info('locked file sync ' . $fileSync->getId()); // get the original id if not set if (!$fileSync->getOriginalId()) { $originalFileSync = self::getOriginalFileSync($fileSync); if (!$originalFileSync) { KalturaLog::info('failed to get original file sync for ' . $fileSync->getId()); continue; } $fileSync->setOriginalId($originalFileSync->getId()); $fileSync->setCustomDataObj(); // update $fileSync->custom_data so that originalId will be set by fromObject } // add to the result set $lockedFileSyncs[] = $fileSync; $lockedFileSyncsSize += $fileSync->getFileSize(); if (count($lockedFileSyncs) >= $maxCount || $maxSize && $lockedFileSyncsSize >= $maxSize) { $lastId = $fileSync->getId(); $limitReached = true; break; } } if ($limitReached) { break; } } // update the last id // Note: it is possible that the last id will go back in case of race condition, // but the only effect of this is that some file syncs will be scanned again if (!$initialLastId || $lastId > $initialLastId) { KalturaLog::info("setting lastId to [{$lastId}] for worker [{$workerId}]"); $keysCache->set(self::LAST_FILESYNC_ID_PREFIX . $workerId, $lastId); } // make sure all file syncs have a path foreach ($lockedFileSyncs as $fileSync) { if ($fileSync->getFileRoot() && $fileSync->getFilePath()) { continue; } $fileSyncKey = kFileSyncUtils::getKeyForFileSync($fileSync); list($fileRoot, $realPath) = kPathManager::getFilePathArr($fileSyncKey); $fileSync->setFileRoot($fileRoot); $fileSync->setFilePath($realPath); } // build the response object $sourceDc = kDataCenterMgr::getDcById($sourceDc); $result = new KalturaLockFileSyncsResponse(); $result->fileSyncs = KalturaFileSyncArray::fromDbArray($lockedFileSyncs, $this->getResponseProfile()); $result->limitReached = $limitReached; $result->dcSecret = $sourceDc["secret"]; $result->baseUrl = isset($sourceDc["fileSyncImportUrl"]) ? $sourceDc["fileSyncImportUrl"] : $sourceDc["url"]; return $result; }
private function deleteOldFileSyncVersions(FileSync $newFileSync) { KalturaLog::debug('Deleting old file_sync versions for [' . $newFileSync->getId() . ']'); if (kConf::hasParam('num_of_old_file_sync_versions_to_keep')) { $keepCount = kConf::get('num_of_old_file_sync_versions_to_keep'); if (!is_numeric($newFileSync->getVersion())) { return; } $intVersion = intval($newFileSync->getVersion()); $c = new Criteria(); $c->add(FileSyncPeer::OBJECT_ID, $newFileSync->getObjectId()); $c->add(FileSyncPeer::OBJECT_TYPE, $newFileSync->getObjectType()); $c->add(FileSyncPeer::OBJECT_SUB_TYPE, $newFileSync->getObjectSubType()); $c->add(FileSyncPeer::STATUS, array(FileSync::FILE_SYNC_STATUS_PURGED, FileSync::FILE_SYNC_STATUS_DELETED), Criteria::NOT_IN); $c->setLimit(40); //we limit the number of files to delete in one run so there will be no out of memory issues $fileSyncs = FileSyncPeer::doSelect($c); foreach ($fileSyncs as $fileSync) { if (is_numeric($fileSync->getVersion())) { $currentIntVersion = intval($fileSync->getVersion()); if ($intVersion - $keepCount > $currentIntVersion) { $key = kFileSyncUtils::getKeyForFileSync($fileSync); self::deleteSyncFileForKey($key); } } } } }
/** * @param BatchJob $dbBatchJob * @param kStorageExportJobData $data * @return BatchJob */ public static function handleStorageExportFinished(BatchJob $dbBatchJob, kStorageExportJobData $data) { KalturaLog::debug("Export to storage finished for sync file[" . $data->getSrcFileSyncId() . "]"); $fileSync = FileSyncPeer::retrieveByPK($data->getSrcFileSyncId()); $fileSync->setStatus(FileSync::FILE_SYNC_STATUS_READY); $fileSync->save(); if ($dbBatchJob->getJobSubType() != StorageProfile::STORAGE_KALTURA_DC) { $partner = $dbBatchJob->getPartner(); if ($partner && $partner->getStorageDeleteFromKaltura()) { $syncKey = kFileSyncUtils::getKeyForFileSync($fileSync); kFileSyncUtils::deleteSyncFileForKey($syncKey, false, true); } } return $dbBatchJob; }
/** * @param FileSync $fileSync */ protected function fileSyncDelete(FileSync $fileSync, BatchJob $raisedJob = null) { $partnerId = $fileSync->getPartnerId(); $purgePermission = PermissionPeer::isValidForPartner('PURGE_FILES_ON_DELETE', $partnerId); if ($purgePermission) { $syncKey = kFileSyncUtils::getKeyForFileSync($fileSync); kJobsManager::addDeleteFileJob($raisedJob, null, $partnerId, $syncKey, $fileSync->getFullPath(), $fileSync->getDc()); } }
/** * @param FileSync $fileSync */ protected static function deleteFileSync(FileSync $fileSync) { KalturaLog::info("Deleting file sync [" . $fileSync->getId() . "]"); $key = kFileSyncUtils::getKeyForFileSync($fileSync); try { kFileSyncUtils::deleteSyncFileForKey($key); } catch (Exception $e) { KalturaLog::err($e); } self::incrementSummary('FileSync'); }
public function objectDeleted(BaseObject $object, BatchJob $raisedJob = null) { /* @var $object FileSync */ $syncKey = kFileSyncUtils::getKeyForFileSync($object); $entryId = null; switch ($object->getObjectType()) { case FileSyncObjectType::ENTRY: $entryId = $object->getObjectId(); break; case FileSyncObjectType::BATCHJOB: BatchJobPeer::setUseCriteriaFilter(false); $batchJob = BatchJobPeer::retrieveByPK($object->getObjectId()); if ($batchJob) { $entryId = $batchJob->getEntryId(); } BatchJobPeer::setUseCriteriaFilter(true); break; case FileSyncObjectType::ASSET: assetPeer::setUseCriteriaFilter(false); $asset = assetPeer::retrieveById($object->getId()); if ($asset) { $entryId = $asset->getEntryId(); //the next piece of code checks whether the entry to which //the deleted asset belongs to is a "replacement" entry $entry = entryPeer::retrieveByPKNoFilter($entryId); if (!$entry) { KalturaLog::alert("No entry found by the ID of [{$entryId}]"); } else { if ($entry->getReplacedEntryId()) { KalturaLog::info("Will not handle event - deleted asset belongs to replacement entry"); return; } } } assetPeer::setUseCriteriaFilter(true); break; } $storage = StorageProfilePeer::retrieveByPK($object->getDc()); kJobsManager::addStorageDeleteJob($raisedJob, $entryId, $storage, $object); }
// ---------------------------- $exportedFileSyncsCriteria = new Criteria(); $exportedFileSyncsCriteria->setLimit($loopLimit); $exportedFileSyncsCriteria->addAscendingOrderByColumn(FileSyncPeer::ID); $exportedFileSyncsCriteria->addAnd(FileSyncPeer::DC, $storageProfileId, Criteria::EQUAL); $exportedFileSyncsCriteria->addAnd(FileSyncPeer::STATUS, FileSync::FILE_SYNC_STATUS_READY, Criteria::EQUAL); $exportedFileSyncsCriteria->addAnd(FileSyncPeer::PARTNER_ID, $partnerId, Criteria::EQUAL); if ($lastFileSyncId) { $exportedFileSyncsCriteria->addAnd(FileSyncPeer::ID, $lastFileSyncId, Criteria::GREATER_THAN); } $exportedFileSyncs = FileSyncPeer::doSelect($exportedFileSyncsCriteria, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL3)); // ----------------------------------------------- // delete the local dcs copies of each file sync // ----------------------------------------------- foreach ($exportedFileSyncs as $exportedFileSync) { $lastFileSyncId = $exportedFileSync->getId(); $syncKey = kFileSyncUtils::getKeyForFileSync($exportedFileSync); echo 'Deleting file sync key - ' . serialize($syncKey) . PHP_EOL; kFileSyncUtils::deleteSyncFileForKey($syncKey, false, true); // 3rd param = true -> only delete from local dcs } // -------------------------- // check if loop should end // -------------------------- if (count($exportedFileSyncs) < $loopLimit) { $moreFileSyncs = false; } $exportedFileSyncs = null; kMemoryManager::clearMemory(); } echo '-- Done --' . PHP_EOL;
/** * @param BatchJob $dbBatchJob * @param kStorageExportJobData $data * @return BatchJob */ public static function handleStorageExportFinished(BatchJob $dbBatchJob, kStorageExportJobData $data) { KalturaLog::debug("Export to storage finished for sync file[" . $data->getSrcFileSyncId() . "]"); $fileSync = FileSyncPeer::retrieveByPK($data->getSrcFileSyncId()); $fileSync->setStatus(FileSync::FILE_SYNC_STATUS_READY); $fileSync->save(); // if an asset was exported - check if should set its status to READY $asset = assetPeer::retrieveByFileSync($fileSync); if ($asset && in_array($asset->getStatus(), array(asset::ASSET_STATUS_EXPORTING, asset::ASSET_STATUS_ERROR))) { $asset->setStatusLocalReady(); $asset->save(); if ($asset instanceof flavorAsset && $asset->getStatus() == asset::ASSET_STATUS_READY) { kBusinessPostConvertDL::handleConvertFinished($dbBatchJob, $asset); } } // check if all exports finished and delete local file sync according to configuration if ($asset && $asset->getStatus() == asset::ASSET_STATUS_READY && $dbBatchJob->getJobSubType() != StorageProfile::STORAGE_KALTURA_DC) { $partner = $dbBatchJob->getPartner(); if ($partner && $partner->getStorageDeleteFromKaltura()) { $syncKey = kFileSyncUtils::getKeyForFileSync($fileSync); kFileSyncUtils::deleteSyncFileForKey($syncKey, false, true); } } return $dbBatchJob; }