/**
* @param string $entryId
* @param FileSync $object
* @param int $fileSyncId
* @param string $sourceFileUrl
* @return BatchJob
*/
public static function addFileSyncImportJob($entryId, FileSync $fileSync, $sourceFileUrl, BatchJob $parentJob = null, $fileSize = null)
{
$partnerId = $fileSync->getPartnerId();
$fileSyncId = $fileSync->getId();
$dc = $fileSync->getDc();
KalturaLog::log(__METHOD__ . " entryId[{$entryId}], partnerId[{$partnerId}], fileSyncId[{$fileSyncId}], sourceFileUrl[{$sourceFileUrl}]");
$fileSyncImportData = new kFileSyncImportJobData();
$fileSyncImportData->setSourceUrl($sourceFileUrl);
$fileSyncImportData->setFilesyncId($fileSyncId);
$fileSyncImportData->setFileSize($fileSize);
// tmpFilePath and destFilePath will be set later during get exlusive call on the target data center
$batchJob = null;
if ($parentJob) {
$batchJob = $parentJob->createChild(BatchJobType::FILESYNC_IMPORT, null, true, $dc);
} else {
$batchJob = new BatchJob();
$batchJob->setDc($dc);
$batchJob->setEntryId($entryId);
$batchJob->setPartnerId($partnerId);
}
$batchJob->setObjectId($fileSyncId);
$batchJob->setObjectType(BatchJobObjectType::FILE_SYNC);
//In case file sync is of type data and holds flavor asset than we need to check if its the source asset that is being synced and raise it sync priority
if ($fileSync->getObjectType() == FileSyncObjectType::FLAVOR_ASSET && $fileSync->getObjectSubType() == entry::FILE_SYNC_ENTRY_SUB_TYPE_DATA && $fileSync->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_FILE) {
$assetdb = assetPeer::retrieveById($fileSync->getObjectId());
if ($assetdb) {
$isSourceAsset = $assetdb->getIsOriginal();
if ($isSourceAsset) {
$fileSyncImportData->setIsSourceAsset(true);
}
}
}
KalturaLog::log("Creating Filesync Import job, with file sync id: {$fileSyncId} size: {$fileSize}");
return kJobsManager::addJob($batchJob, $fileSyncImportData, BatchJobType::FILESYNC_IMPORT);
}
public function setStorageExportJobData(StorageProfile $externalStorage, FileSync $fileSync, $srcFileSyncLocalPath, $force = false)
{
/* @var $externalStorage KontikiStorageProfile */
$this->setServerUrl($externalStorage->getStorageUrl());
$this->setServiceToken($externalStorage->getServiceToken());
$this->setSrcFileSyncId($fileSync->getId());
if ($fileSync->getObjectType() != FileSyncObjectType::ASSET) {
throw new kCoreException("Incompatible filesync type", kCoreException::INTERNAL_SERVER_ERROR);
}
$this->setFlavorAssetId($fileSync->getObjectId());
}
/**
* @var StorageProfile $storage
* @var FileSync $fileSync
*/
public function setJobData(StorageProfile $storage, FileSync $filesync)
{
$this->setServerUrl($storage->getStorageUrl());
$this->setServerUsername($storage->getStorageUsername());
$this->setServerPassword($storage->getStoragePassword());
$this->setServerPrivateKey($storage->getPrivateKey());
$this->setServerPublicKey($storage->getPublicKey());
$this->setServerPassPhrase($storage->getPassPhrase());
$this->setFtpPassiveMode($storage->getStorageFtpPassiveMode());
$this->setSrcFileSyncId($filesync->getId());
$this->setDestFileSyncStoredPath($storage->getStorageBaseDir() . '/' . $filesync->getFilePath());
}
public function setStorageExportJobData(StorageProfile $externalStorage, FileSync $fileSync, $srcFileSyncLocalPath, $force = false)
{
$this->setServerUrl($externalStorage->getStorageUrl());
$this->setServerUsername($externalStorage->getStorageUsername());
$this->setServerPassword($externalStorage->getStoragePassword());
$this->setServerPrivateKey($externalStorage->getPrivateKey());
$this->setServerPublicKey($externalStorage->getPublicKey());
$this->setServerPassPhrase($externalStorage->getPassPhrase());
$this->setFtpPassiveMode($externalStorage->getStorageFtpPassiveMode());
$this->setSrcFileSyncLocalPath($srcFileSyncLocalPath);
$this->setSrcFileSyncId($fileSync->getId());
$this->setForce($force);
$this->setDestFileSyncStoredPath($externalStorage->getStorageBaseDir() . '/' . $fileSync->getFilePath());
$this->setCreateLink($externalStorage->getCreateFileLink());
}
/**
* @param string $entryId
* @param FileSync $object
* @param int $fileSyncId
* @param string $sourceFileUrl
* @return BatchJob
*/
public static function addFileSyncImportJob($entryId, FileSync $fileSync, $sourceFileUrl, BatchJob $parentJob = null, $fileSize = null)
{
$partnerId = $fileSync->getPartnerId();
$fileSyncId = $fileSync->getId();
$dc = $fileSync->getDc();
KalturaLog::log(__METHOD__ . " entryId[{$entryId}], partnerId[{$partnerId}], fileSyncId[{$fileSyncId}], sourceFileUrl[{$sourceFileUrl}]");
$fileSyncImportData = new kFileSyncImportJobData();
$fileSyncImportData->setSourceUrl($sourceFileUrl);
$fileSyncImportData->setFilesyncId($fileSyncId);
// tmpFilePath and destFilePath will be set later during get exlusive call on the target data center
$batchJob = null;
if ($parentJob) {
$batchJob = $parentJob->createChild(true, $dc);
} else {
$batchJob = new BatchJob();
$batchJob->setDc($dc);
$batchJob->setEntryId($entryId);
$batchJob->setPartnerId($partnerId);
}
$batchJob->setFileSize($fileSize);
KalturaLog::log("Creating Filesync Import job, with file sync id: {$fileSyncId} size: {$fileSize}");
return kJobsManager::addJob($batchJob, $fileSyncImportData, BatchJobType::FILESYNC_IMPORT);
}
/**
* @param BatchJob $parentJob
* @param string $entryId
* @param int $partnerId
* @param StorageProfile $externalStorage
* @param SyncFile $fileSync
* @param string $srcFileSyncLocalPath
* @param bool $force
*
* @return BatchJob
*/
public static function addStorageExportJob(BatchJob $parentJob = null, $entryId, $partnerId, StorageProfile $externalStorage, FileSync $fileSync, $srcFileSyncLocalPath, $force = false, $dc = null)
{
KalturaLog::debug("entryId[{$entryId}], partnerId[{$partnerId}], externalStorage id[" . $externalStorage->getId() . "], fileSync id[" . $fileSync->getId() . "], srcFileSyncLocalPath[{$srcFileSyncLocalPath}]");
$netStorageExportData = kStorageExportJobData::getInstance($externalStorage->getProtocol());
$netStorageExportData->setStorageExportJobData($externalStorage, $fileSync, $srcFileSyncLocalPath);
$batchJob = null;
if ($parentJob) {
$batchJob = $parentJob->createChild(BatchJobType::STORAGE_EXPORT, $externalStorage->getProtocol(), false);
} else {
$batchJob = new BatchJob();
$batchJob->setEntryId($entryId);
$batchJob->setPartnerId($partnerId);
}
$batchJob->setObjectId($fileSync->getId());
$batchJob->setObjectType(BatchJobObjectType::FILE_SYNC);
$batchJob->setJobSubType($externalStorage->getProtocol());
$batchJob->setDc($dc);
KalturaLog::log("Creating Storage export job, with source file: " . $netStorageExportData->getSrcFileSyncLocalPath());
return self::addJob($batchJob, $netStorageExportData, BatchJobType::STORAGE_EXPORT, $externalStorage->getProtocol());
}
/**
* Will fetch the content of the $file_sync.
* If the $local_file_path is specifid, will place the cotnet there
* @param FileSync $file_sync
* @return string
*/
public static function retrieveFileFromRemoteDataCenter(FileSync $file_sync)
{
KalturaLog::log("File sync [{$file_sync->getId()}]");
// LOG retrieval
$cmd_line = self::createCmdForRemoteDataCenter($file_sync);
$local_file_path = self::getLocalTempPathForFileSync($file_sync);
if (!file_exists($local_file_path)) {
KalturaLog::log("Executing " . $cmd_line);
exec($cmd_line);
clearstatcache();
if (!file_exists($local_file_path)) {
KalturaLog::err("Temp file not retrieved [{$local_file_path}]");
return false;
}
} else {
KalturaLog::log("Already exists in temp folder [{$local_file_path}]");
}
return file_get_contents($local_file_path);
}
/**
* @param BatchJob $parentJob
* @param string $entryId
* @param int $partnerId
* @param StorageProfile $externalStorage
* @param SyncFile $fileSync
* @param string $srcFileSyncLocalPath
* @param bool $force
*
* @return BatchJob
*/
public static function addStorageExportJob(BatchJob $parentJob = null, $entryId, $partnerId, StorageProfile $externalStorage, FileSync $fileSync, $srcFileSyncLocalPath, $force = false)
{
KalturaLog::log(__METHOD__ . " entryId[{$entryId}], partnerId[{$partnerId}], externalStorage id[" . $externalStorage->getId() . "], fileSync id[" . $fileSync->getId() . "], srcFileSyncLocalPath[{$srcFileSyncLocalPath}]");
$netStorageExportData = new kStorageExportJobData();
$netStorageExportData->setServerUrl($externalStorage->getStorageUrl());
$netStorageExportData->setServerUsername($externalStorage->getStorageUsername());
$netStorageExportData->setServerPassword($externalStorage->getStoragePassword());
$netStorageExportData->setFtpPassiveMode($externalStorage->getStorageFtpPassiveMode());
$netStorageExportData->setSrcFileSyncLocalPath($srcFileSyncLocalPath);
$netStorageExportData->setSrcFileSyncId($fileSync->getId());
$netStorageExportData->setForce($force);
$netStorageExportData->setDestFileSyncStoredPath($externalStorage->getStorageBaseDir() . '/' . $fileSync->getFilePath());
$batchJob = null;
if ($parentJob) {
$batchJob = $parentJob->createChild(false);
} else {
$batchJob = new BatchJob();
$batchJob->setEntryId($entryId);
$batchJob->setPartnerId($partnerId);
}
KalturaLog::log("Creating Storage export job, with source file: " . $netStorageExportData->getSrcFileSyncLocalPath());
return self::addJob($batchJob, $netStorageExportData, BatchJobType::STORAGE_EXPORT, $externalStorage->getProtocol());
}
/**
* @param BaseObject $object
*/
public function fileSyncCreated(FileSync $fileSync)
{
KalturaLog::debug("id [" . $fileSync->getId() . "]");
$object = kFileSyncUtils::retrieveObjectForFileSync($fileSync);
if (!$object || !$object instanceof ISyncableFile) {
KalturaLog::debug("Not instance of ISyncableFile");
return;
}
$auditTrail = self::createAuditTrail($object, AuditTrail::AUDIT_TRAIL_ACTION_FILE_SYNC_CREATED);
if (!$auditTrail) {
KalturaLog::debug("No audit created");
return;
}
$data = new kAuditTrailFileSyncCreateInfo();
$data->setVersion($fileSync->getVersion());
$data->setObjectSubType($fileSync->getObjectSubType());
$data->setDc($fileSync->getDc());
$data->setOriginal($fileSync->getOriginal());
$data->setFileType($fileSync->getFileType());
$auditTrail->setData($data);
$auditTrail->save();
}
/**
*
* @param FileSync $key
* @return ISyncableFile
*/
public static function retrieveObjectForFileSync(FileSync $file_sync)
{
KalturaLog::log(__METHOD__ . " - FileSync id [" . $file_sync->getId() . "]");
return kFileSyncObjectManager::retrieveObject($file_sync->getObjectType(), $file_sync->getObjectId());
}
/**
*
* Handle deleteion of original entries file sync when in the process of entry replacement.
* @param FileSync $fileSync
* @param array $fileSyncSubTypesToHandle
*/
private static function handleEntryReplacementFileSyncDeletion(FileSync $fileSync, $fileSyncSubTypesToHandle)
{
$c = new Criteria();
$c->add(FileSyncPeer::FILE_TYPE, array(FileSync::FILE_SYNC_FILE_TYPE_URL, FileSync::FILE_SYNC_FILE_TYPE_FILE), Criteria::IN);
$c->add(FileSyncPeer::OBJECT_TYPE, $fileSync->getObjectType());
$c->add(FileSyncPeer::OBJECT_SUB_TYPE, $fileSync->getObjectSubType());
$c->add(FileSyncPeer::PARTNER_ID, $fileSync->getPartnerId());
$c->add(FileSyncPeer::LINKED_ID, $fileSync->getId());
$originalEntryFileSync = FileSyncPeer::doSelectOne($c);
if (!$originalEntryFileSync) {
KalturaLog::info("Origianl entry file sync not found with the following details: [object_type, object_sub_type, Partner_id, Linked_id] [" . $fileSync->getObjectType() . ", " . $fileSync->getObjectSubType() . ", " . $fileSync->getPartnerId() . ", " . $fileSync->getId() . "]");
return;
}
$originalAssetToDeleteFileSyncFor = assetPeer::retrieveById($originalEntryFileSync->getObjectId());
if (!$originalAssetToDeleteFileSyncFor) {
KalturaLog::info("Could not find asset matching file sync object id " . $originalEntryFileSync->getObjectId());
return;
}
foreach ($fileSyncSubTypesToHandle as $fileSyncSubType) {
$syncKeyToDelete = $originalAssetToDeleteFileSyncFor->getSyncKey($fileSyncSubType);
kFileSyncUtils::deleteSyncFileForKey($syncKeyToDelete, false, true);
}
}
public static function addStorageDeleteJob(BatchJob $parentJob = null, $entryId = null, StorageProfile $storage, FileSync $fileSync)
{
$netStorageDeleteData = new kStorageDeleteJobData();
$netStorageDeleteData->setServerUrl($storage->getStorageUrl());
$netStorageDeleteData->setServerUsername($storage->getStorageUsername());
$netStorageDeleteData->setServerPassword($storage->getStoragePassword());
$netStorageDeleteData->setFtpPassiveMode($storage->getStorageFtpPassiveMode());
$netStorageDeleteData->setSrcFileSyncId($fileSync->getId());
$netStorageDeleteData->setDestFileSyncStoredPath($storage->getStorageBaseDir() . '/' . $fileSync->getFilePath());
if ($parentJob) {
$batchJob = $parentJob->createChild(false);
} else {
$batchJob = new BatchJob();
$batchJob->setEntryId($entryId);
$batchJob->setPartnerId($storage->getPartnerId());
}
KalturaLog::log("Creating Net-Storage Delete job, with source file: " . $netStorageDeleteData->getSrcFileSyncLocalPath());
return self::addJob($batchJob, $netStorageDeleteData, BatchJobType::STORAGE_DELETE, $storage->getProtocol());
}
/**
* Adds an object to the instance pool.
*
* Propel keeps cached copies of objects in an instance pool when they are retrieved
* from the database. In some cases -- especially when you override doSelect*()
* methods in your stub classes -- you may need to explicitly add objects
* to the cache in order to ensure that the same objects are always returned by doSelect*()
* and retrieveByPK*() calls.
*
* @param FileSync $value A FileSync object.
* @param string $key (optional) key to use for instance map (for performance boost if key was already calculated externally).
*/
public static function addInstanceToPool(FileSync $obj, $key = null)
{
if (Propel::isInstancePoolingEnabled()) {
if ($key === null) {
$key = (string) $obj->getId();
}
if (isset(self::$instances[$key]) || count(self::$instances) < kConf::get('max_num_instances_in_pool')) {
self::$instances[$key] = $obj;
kMemoryManager::registerPeer('FileSyncPeer');
}
}
}
/**
* Will fetch the content of the $file_sync.
* If the $local_file_path is specifid, will place the cotnet there
* @param FileSync $file_sync
* @return string
*/
public static function retrieveFileFromRemoteDataCenter(FileSync $file_sync)
{
KalturaLog::log(__METHOD__ . " - file_sync [{$file_sync->getId()}]");
// LOG retrieval
$cmd_line = self::createCmdForRemoteDataCenter($file_sync);
$local_file_path = self::getLocalTempPathForFileSync($file_sync);
if (!file_exists($local_file_path)) {
KalturaLog::log(__METHOD__ . " - executing " . $cmd_line);
exec($cmd_line);
} else {
KalturaLog::log(__METHOD__ . " - already exists in temp folder [{$local_file_path}]");
}
return file_get_contents($local_file_path);
}
private function deleteOldFileSyncVersions(FileSync $newFileSync)
{
KalturaLog::debug('Deleting old file_sync versions for [' . $newFileSync->getId() . ']');
if (kConf::hasParam('num_of_old_file_sync_versions_to_keep')) {
$keepCount = kConf::get('num_of_old_file_sync_versions_to_keep');
if (!is_numeric($newFileSync->getVersion())) {
return;
}
$intVersion = intval($newFileSync->getVersion());
$c = new Criteria();
$c->add(FileSyncPeer::OBJECT_ID, $newFileSync->getObjectId());
$c->add(FileSyncPeer::OBJECT_TYPE, $newFileSync->getObjectType());
$c->add(FileSyncPeer::OBJECT_SUB_TYPE, $newFileSync->getObjectSubType());
$c->add(FileSyncPeer::STATUS, array(FileSync::FILE_SYNC_STATUS_PURGED, FileSync::FILE_SYNC_STATUS_DELETED), Criteria::NOT_IN);
$c->setLimit(40);
//we limit the number of files to delete in one run so there will be no out of memory issues
$fileSyncs = FileSyncPeer::doSelect($c);
foreach ($fileSyncs as $fileSync) {
if (is_numeric($fileSync->getVersion())) {
$currentIntVersion = intval($fileSync->getVersion());
if ($intVersion - $keepCount > $currentIntVersion) {
$key = kFileSyncUtils::getKeyForFileSync($fileSync);
self::deleteSyncFileForKey($key);
}
}
}
}
}
/**
* Adds an object to the instance pool.
*
* Propel keeps cached copies of objects in an instance pool when they are retrieved
* from the database. In some cases -- especially when you override doSelect*()
* methods in your stub classes -- you may need to explicitly add objects
* to the cache in order to ensure that the same objects are always returned by doSelect*()
* and retrieveByPK*() calls.
*
* @param FileSync $value A FileSync object.
* @param string $key (optional) key to use for instance map (for performance boost if key was already calculated externally).
*/
public static function addInstanceToPool(FileSync $obj, $key = null)
{
if (Propel::isInstancePoolingEnabled()) {
if ($key === null) {
$key = (string) $obj->getId();
}
// if key === null
self::$instances[$key] = $obj;
}
}
/**
* @param FileSync $fileSync
*/
protected static function purgeFileSync(FileSync $fileSync)
{
KalturaLog::info("Purging file sync [" . $fileSync->getId() . "]");
$fullPath = $fileSync->getFullPath();
if ($fullPath && file_exists($fullPath)) {
KalturaLog::debug("Purging file sync [" . $fileSync->getId() . "] path [{$fullPath}]");
if (is_dir($fullPath)) {
$command = "rm -fr {$fullPath}";
KalturaLog::debug("Executing: {$command}");
if (self::$dryRun) {
self::incrementSummary('dirs');
} else {
$returnedValue = null;
passthru($command, $returnedValue);
if ($returnedValue === 0) {
self::incrementSummary('dirs');
} else {
KalturaLog::err("Failed purging file sync [" . $fileSync->getId() . "] directory path [{$fullPath}]");
return;
}
}
} else {
$fileSize = filesize($fullPath);
if (self::$dryRun || unlink($fullPath)) {
self::incrementSummary('bytes', $fileSize);
self::incrementSummary('files');
} else {
KalturaLog::err("Failed purging file sync [" . $fileSync->getId() . "] file path [{$fullPath}]");
return;
}
}
} else {
KalturaLog::debug("File sync [" . $fileSync->getId() . "] path [{$fullPath}] does not exist");
}
$fileSync->setStatus(FileSync::FILE_SYNC_STATUS_PURGED);
$fileSync->save();
}
