Пример #1
0
 /**
  * @param string $entryId
  * @param FileSync $object
  * @param int $fileSyncId
  * @param string $sourceFileUrl
  * @return BatchJob
  */
 public static function addFileSyncImportJob($entryId, FileSync $fileSync, $sourceFileUrl, BatchJob $parentJob = null, $fileSize = null)
 {
     $partnerId = $fileSync->getPartnerId();
     $fileSyncId = $fileSync->getId();
     $dc = $fileSync->getDc();
     KalturaLog::log(__METHOD__ . " entryId[{$entryId}], partnerId[{$partnerId}], fileSyncId[{$fileSyncId}], sourceFileUrl[{$sourceFileUrl}]");
     $fileSyncImportData = new kFileSyncImportJobData();
     $fileSyncImportData->setSourceUrl($sourceFileUrl);
     $fileSyncImportData->setFilesyncId($fileSyncId);
     $fileSyncImportData->setFileSize($fileSize);
     // tmpFilePath and destFilePath will be set later during get exlusive call on the target data center
     $batchJob = null;
     if ($parentJob) {
         $batchJob = $parentJob->createChild(BatchJobType::FILESYNC_IMPORT, null, true, $dc);
     } else {
         $batchJob = new BatchJob();
         $batchJob->setDc($dc);
         $batchJob->setEntryId($entryId);
         $batchJob->setPartnerId($partnerId);
     }
     $batchJob->setObjectId($fileSyncId);
     $batchJob->setObjectType(BatchJobObjectType::FILE_SYNC);
     //In case file sync is of type data and holds flavor asset than we need to check if its the source asset that is being synced and raise it sync priority
     if ($fileSync->getObjectType() == FileSyncObjectType::FLAVOR_ASSET && $fileSync->getObjectSubType() == entry::FILE_SYNC_ENTRY_SUB_TYPE_DATA && $fileSync->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_FILE) {
         $assetdb = assetPeer::retrieveById($fileSync->getObjectId());
         if ($assetdb) {
             $isSourceAsset = $assetdb->getIsOriginal();
             if ($isSourceAsset) {
                 $fileSyncImportData->setIsSourceAsset(true);
             }
         }
     }
     KalturaLog::log("Creating Filesync Import job, with file sync id: {$fileSyncId} size: {$fileSize}");
     return kJobsManager::addJob($batchJob, $fileSyncImportData, BatchJobType::FILESYNC_IMPORT);
 }
 public function setStorageExportJobData(StorageProfile $externalStorage, FileSync $fileSync, $srcFileSyncLocalPath, $force = false)
 {
     /* @var $externalStorage KontikiStorageProfile */
     $this->setServerUrl($externalStorage->getStorageUrl());
     $this->setServiceToken($externalStorage->getServiceToken());
     $this->setSrcFileSyncId($fileSync->getId());
     if ($fileSync->getObjectType() != FileSyncObjectType::ASSET) {
         throw new kCoreException("Incompatible filesync type", kCoreException::INTERNAL_SERVER_ERROR);
     }
     $this->setFlavorAssetId($fileSync->getObjectId());
 }
Пример #3
0
 /**
  * @var StorageProfile $storage
  * @var FileSync $fileSync
  */
 public function setJobData(StorageProfile $storage, FileSync $filesync)
 {
     $this->setServerUrl($storage->getStorageUrl());
     $this->setServerUsername($storage->getStorageUsername());
     $this->setServerPassword($storage->getStoragePassword());
     $this->setServerPrivateKey($storage->getPrivateKey());
     $this->setServerPublicKey($storage->getPublicKey());
     $this->setServerPassPhrase($storage->getPassPhrase());
     $this->setFtpPassiveMode($storage->getStorageFtpPassiveMode());
     $this->setSrcFileSyncId($filesync->getId());
     $this->setDestFileSyncStoredPath($storage->getStorageBaseDir() . '/' . $filesync->getFilePath());
 }
Пример #4
0
 public function setStorageExportJobData(StorageProfile $externalStorage, FileSync $fileSync, $srcFileSyncLocalPath, $force = false)
 {
     $this->setServerUrl($externalStorage->getStorageUrl());
     $this->setServerUsername($externalStorage->getStorageUsername());
     $this->setServerPassword($externalStorage->getStoragePassword());
     $this->setServerPrivateKey($externalStorage->getPrivateKey());
     $this->setServerPublicKey($externalStorage->getPublicKey());
     $this->setServerPassPhrase($externalStorage->getPassPhrase());
     $this->setFtpPassiveMode($externalStorage->getStorageFtpPassiveMode());
     $this->setSrcFileSyncLocalPath($srcFileSyncLocalPath);
     $this->setSrcFileSyncId($fileSync->getId());
     $this->setForce($force);
     $this->setDestFileSyncStoredPath($externalStorage->getStorageBaseDir() . '/' . $fileSync->getFilePath());
     $this->setCreateLink($externalStorage->getCreateFileLink());
 }
 /**
  * @param string $entryId
  * @param FileSync $object
  * @param int $fileSyncId
  * @param string $sourceFileUrl
  * @return BatchJob
  */
 public static function addFileSyncImportJob($entryId, FileSync $fileSync, $sourceFileUrl, BatchJob $parentJob = null, $fileSize = null)
 {
     $partnerId = $fileSync->getPartnerId();
     $fileSyncId = $fileSync->getId();
     $dc = $fileSync->getDc();
     KalturaLog::log(__METHOD__ . " entryId[{$entryId}], partnerId[{$partnerId}], fileSyncId[{$fileSyncId}], sourceFileUrl[{$sourceFileUrl}]");
     $fileSyncImportData = new kFileSyncImportJobData();
     $fileSyncImportData->setSourceUrl($sourceFileUrl);
     $fileSyncImportData->setFilesyncId($fileSyncId);
     // tmpFilePath and destFilePath will be set later during get exlusive call on the target data center
     $batchJob = null;
     if ($parentJob) {
         $batchJob = $parentJob->createChild(true, $dc);
     } else {
         $batchJob = new BatchJob();
         $batchJob->setDc($dc);
         $batchJob->setEntryId($entryId);
         $batchJob->setPartnerId($partnerId);
     }
     $batchJob->setFileSize($fileSize);
     KalturaLog::log("Creating Filesync Import job, with file sync id: {$fileSyncId} size: {$fileSize}");
     return kJobsManager::addJob($batchJob, $fileSyncImportData, BatchJobType::FILESYNC_IMPORT);
 }
Пример #6
0
 /**
  * @param BatchJob $parentJob
  * @param string $entryId
  * @param int $partnerId
  * @param StorageProfile $externalStorage
  * @param SyncFile $fileSync
  * @param string $srcFileSyncLocalPath
  * @param bool $force
  * 
  * @return BatchJob
  */
 public static function addStorageExportJob(BatchJob $parentJob = null, $entryId, $partnerId, StorageProfile $externalStorage, FileSync $fileSync, $srcFileSyncLocalPath, $force = false, $dc = null)
 {
     KalturaLog::debug("entryId[{$entryId}], partnerId[{$partnerId}], externalStorage id[" . $externalStorage->getId() . "], fileSync id[" . $fileSync->getId() . "], srcFileSyncLocalPath[{$srcFileSyncLocalPath}]");
     $netStorageExportData = kStorageExportJobData::getInstance($externalStorage->getProtocol());
     $netStorageExportData->setStorageExportJobData($externalStorage, $fileSync, $srcFileSyncLocalPath);
     $batchJob = null;
     if ($parentJob) {
         $batchJob = $parentJob->createChild(BatchJobType::STORAGE_EXPORT, $externalStorage->getProtocol(), false);
     } else {
         $batchJob = new BatchJob();
         $batchJob->setEntryId($entryId);
         $batchJob->setPartnerId($partnerId);
     }
     $batchJob->setObjectId($fileSync->getId());
     $batchJob->setObjectType(BatchJobObjectType::FILE_SYNC);
     $batchJob->setJobSubType($externalStorage->getProtocol());
     $batchJob->setDc($dc);
     KalturaLog::log("Creating Storage export job, with source file: " . $netStorageExportData->getSrcFileSyncLocalPath());
     return self::addJob($batchJob, $netStorageExportData, BatchJobType::STORAGE_EXPORT, $externalStorage->getProtocol());
 }
Пример #7
0
 /**
  * Will fetch the content of the $file_sync.
  * If the $local_file_path is specifid, will place the cotnet there
  * @param FileSync $file_sync
  * @return string
  */
 public static function retrieveFileFromRemoteDataCenter(FileSync $file_sync)
 {
     KalturaLog::log("File sync [{$file_sync->getId()}]");
     // LOG retrieval
     $cmd_line = self::createCmdForRemoteDataCenter($file_sync);
     $local_file_path = self::getLocalTempPathForFileSync($file_sync);
     if (!file_exists($local_file_path)) {
         KalturaLog::log("Executing " . $cmd_line);
         exec($cmd_line);
         clearstatcache();
         if (!file_exists($local_file_path)) {
             KalturaLog::err("Temp file not retrieved [{$local_file_path}]");
             return false;
         }
     } else {
         KalturaLog::log("Already exists in temp folder [{$local_file_path}]");
     }
     return file_get_contents($local_file_path);
 }
Пример #8
0
 /**
  * @param BatchJob $parentJob
  * @param string $entryId
  * @param int $partnerId
  * @param StorageProfile $externalStorage
  * @param SyncFile $fileSync
  * @param string $srcFileSyncLocalPath
  * @param bool $force
  * 
  * @return BatchJob
  */
 public static function addStorageExportJob(BatchJob $parentJob = null, $entryId, $partnerId, StorageProfile $externalStorage, FileSync $fileSync, $srcFileSyncLocalPath, $force = false)
 {
     KalturaLog::log(__METHOD__ . " entryId[{$entryId}], partnerId[{$partnerId}], externalStorage id[" . $externalStorage->getId() . "], fileSync id[" . $fileSync->getId() . "], srcFileSyncLocalPath[{$srcFileSyncLocalPath}]");
     $netStorageExportData = new kStorageExportJobData();
     $netStorageExportData->setServerUrl($externalStorage->getStorageUrl());
     $netStorageExportData->setServerUsername($externalStorage->getStorageUsername());
     $netStorageExportData->setServerPassword($externalStorage->getStoragePassword());
     $netStorageExportData->setFtpPassiveMode($externalStorage->getStorageFtpPassiveMode());
     $netStorageExportData->setSrcFileSyncLocalPath($srcFileSyncLocalPath);
     $netStorageExportData->setSrcFileSyncId($fileSync->getId());
     $netStorageExportData->setForce($force);
     $netStorageExportData->setDestFileSyncStoredPath($externalStorage->getStorageBaseDir() . '/' . $fileSync->getFilePath());
     $batchJob = null;
     if ($parentJob) {
         $batchJob = $parentJob->createChild(false);
     } else {
         $batchJob = new BatchJob();
         $batchJob->setEntryId($entryId);
         $batchJob->setPartnerId($partnerId);
     }
     KalturaLog::log("Creating Storage export job, with source file: " . $netStorageExportData->getSrcFileSyncLocalPath());
     return self::addJob($batchJob, $netStorageExportData, BatchJobType::STORAGE_EXPORT, $externalStorage->getProtocol());
 }
Пример #9
0
 /**
  * @param BaseObject $object
  */
 public function fileSyncCreated(FileSync $fileSync)
 {
     KalturaLog::debug("id [" . $fileSync->getId() . "]");
     $object = kFileSyncUtils::retrieveObjectForFileSync($fileSync);
     if (!$object || !$object instanceof ISyncableFile) {
         KalturaLog::debug("Not instance of ISyncableFile");
         return;
     }
     $auditTrail = self::createAuditTrail($object, AuditTrail::AUDIT_TRAIL_ACTION_FILE_SYNC_CREATED);
     if (!$auditTrail) {
         KalturaLog::debug("No audit created");
         return;
     }
     $data = new kAuditTrailFileSyncCreateInfo();
     $data->setVersion($fileSync->getVersion());
     $data->setObjectSubType($fileSync->getObjectSubType());
     $data->setDc($fileSync->getDc());
     $data->setOriginal($fileSync->getOriginal());
     $data->setFileType($fileSync->getFileType());
     $auditTrail->setData($data);
     $auditTrail->save();
 }
 /**
  *
  * @param FileSync $key
  * @return ISyncableFile
  */
 public static function retrieveObjectForFileSync(FileSync $file_sync)
 {
     KalturaLog::log(__METHOD__ . " - FileSync id [" . $file_sync->getId() . "]");
     return kFileSyncObjectManager::retrieveObject($file_sync->getObjectType(), $file_sync->getObjectId());
 }
Пример #11
0
 /**
  * 
  * Handle deleteion of original entries file sync when in the process of entry replacement.
  * @param FileSync $fileSync
  * @param array $fileSyncSubTypesToHandle
  */
 private static function handleEntryReplacementFileSyncDeletion(FileSync $fileSync, $fileSyncSubTypesToHandle)
 {
     $c = new Criteria();
     $c->add(FileSyncPeer::FILE_TYPE, array(FileSync::FILE_SYNC_FILE_TYPE_URL, FileSync::FILE_SYNC_FILE_TYPE_FILE), Criteria::IN);
     $c->add(FileSyncPeer::OBJECT_TYPE, $fileSync->getObjectType());
     $c->add(FileSyncPeer::OBJECT_SUB_TYPE, $fileSync->getObjectSubType());
     $c->add(FileSyncPeer::PARTNER_ID, $fileSync->getPartnerId());
     $c->add(FileSyncPeer::LINKED_ID, $fileSync->getId());
     $originalEntryFileSync = FileSyncPeer::doSelectOne($c);
     if (!$originalEntryFileSync) {
         KalturaLog::info("Origianl entry file sync not found with the following details: [object_type, object_sub_type, Partner_id, Linked_id] [" . $fileSync->getObjectType() . ", " . $fileSync->getObjectSubType() . ", " . $fileSync->getPartnerId() . ", " . $fileSync->getId() . "]");
         return;
     }
     $originalAssetToDeleteFileSyncFor = assetPeer::retrieveById($originalEntryFileSync->getObjectId());
     if (!$originalAssetToDeleteFileSyncFor) {
         KalturaLog::info("Could not find asset matching file sync object id " . $originalEntryFileSync->getObjectId());
         return;
     }
     foreach ($fileSyncSubTypesToHandle as $fileSyncSubType) {
         $syncKeyToDelete = $originalAssetToDeleteFileSyncFor->getSyncKey($fileSyncSubType);
         kFileSyncUtils::deleteSyncFileForKey($syncKeyToDelete, false, true);
     }
 }
 public static function addStorageDeleteJob(BatchJob $parentJob = null, $entryId = null, StorageProfile $storage, FileSync $fileSync)
 {
     $netStorageDeleteData = new kStorageDeleteJobData();
     $netStorageDeleteData->setServerUrl($storage->getStorageUrl());
     $netStorageDeleteData->setServerUsername($storage->getStorageUsername());
     $netStorageDeleteData->setServerPassword($storage->getStoragePassword());
     $netStorageDeleteData->setFtpPassiveMode($storage->getStorageFtpPassiveMode());
     $netStorageDeleteData->setSrcFileSyncId($fileSync->getId());
     $netStorageDeleteData->setDestFileSyncStoredPath($storage->getStorageBaseDir() . '/' . $fileSync->getFilePath());
     if ($parentJob) {
         $batchJob = $parentJob->createChild(false);
     } else {
         $batchJob = new BatchJob();
         $batchJob->setEntryId($entryId);
         $batchJob->setPartnerId($storage->getPartnerId());
     }
     KalturaLog::log("Creating Net-Storage Delete job, with source file: " . $netStorageDeleteData->getSrcFileSyncLocalPath());
     return self::addJob($batchJob, $netStorageDeleteData, BatchJobType::STORAGE_DELETE, $storage->getProtocol());
 }
Пример #13
0
 /**
  * Adds an object to the instance pool.
  *
  * Propel keeps cached copies of objects in an instance pool when they are retrieved
  * from the database.  In some cases -- especially when you override doSelect*()
  * methods in your stub classes -- you may need to explicitly add objects
  * to the cache in order to ensure that the same objects are always returned by doSelect*()
  * and retrieveByPK*() calls.
  *
  * @param      FileSync $value A FileSync object.
  * @param      string $key (optional) key to use for instance map (for performance boost if key was already calculated externally).
  */
 public static function addInstanceToPool(FileSync $obj, $key = null)
 {
     if (Propel::isInstancePoolingEnabled()) {
         if ($key === null) {
             $key = (string) $obj->getId();
         }
         if (isset(self::$instances[$key]) || count(self::$instances) < kConf::get('max_num_instances_in_pool')) {
             self::$instances[$key] = $obj;
             kMemoryManager::registerPeer('FileSyncPeer');
         }
     }
 }
Пример #14
0
 /**
  * Will fetch the content of the $file_sync.
  * If the $local_file_path is specifid, will place the cotnet there
  * @param FileSync $file_sync
  * @return string
  */
 public static function retrieveFileFromRemoteDataCenter(FileSync $file_sync)
 {
     KalturaLog::log(__METHOD__ . " - file_sync [{$file_sync->getId()}]");
     // LOG retrieval
     $cmd_line = self::createCmdForRemoteDataCenter($file_sync);
     $local_file_path = self::getLocalTempPathForFileSync($file_sync);
     if (!file_exists($local_file_path)) {
         KalturaLog::log(__METHOD__ . " - executing " . $cmd_line);
         exec($cmd_line);
     } else {
         KalturaLog::log(__METHOD__ . " - already exists in temp folder [{$local_file_path}]");
     }
     return file_get_contents($local_file_path);
 }
Пример #15
0
 private function deleteOldFileSyncVersions(FileSync $newFileSync)
 {
     KalturaLog::debug('Deleting old file_sync versions for [' . $newFileSync->getId() . ']');
     if (kConf::hasParam('num_of_old_file_sync_versions_to_keep')) {
         $keepCount = kConf::get('num_of_old_file_sync_versions_to_keep');
         if (!is_numeric($newFileSync->getVersion())) {
             return;
         }
         $intVersion = intval($newFileSync->getVersion());
         $c = new Criteria();
         $c->add(FileSyncPeer::OBJECT_ID, $newFileSync->getObjectId());
         $c->add(FileSyncPeer::OBJECT_TYPE, $newFileSync->getObjectType());
         $c->add(FileSyncPeer::OBJECT_SUB_TYPE, $newFileSync->getObjectSubType());
         $c->add(FileSyncPeer::STATUS, array(FileSync::FILE_SYNC_STATUS_PURGED, FileSync::FILE_SYNC_STATUS_DELETED), Criteria::NOT_IN);
         $c->setLimit(40);
         //we limit the number of files to delete in one run so there will be no out of memory issues
         $fileSyncs = FileSyncPeer::doSelect($c);
         foreach ($fileSyncs as $fileSync) {
             if (is_numeric($fileSync->getVersion())) {
                 $currentIntVersion = intval($fileSync->getVersion());
                 if ($intVersion - $keepCount > $currentIntVersion) {
                     $key = kFileSyncUtils::getKeyForFileSync($fileSync);
                     self::deleteSyncFileForKey($key);
                 }
             }
         }
     }
 }
 /**
  * Adds an object to the instance pool.
  *
  * Propel keeps cached copies of objects in an instance pool when they are retrieved
  * from the database.  In some cases -- especially when you override doSelect*()
  * methods in your stub classes -- you may need to explicitly add objects
  * to the cache in order to ensure that the same objects are always returned by doSelect*()
  * and retrieveByPK*() calls.
  *
  * @param      FileSync $value A FileSync object.
  * @param      string $key (optional) key to use for instance map (for performance boost if key was already calculated externally).
  */
 public static function addInstanceToPool(FileSync $obj, $key = null)
 {
     if (Propel::isInstancePoolingEnabled()) {
         if ($key === null) {
             $key = (string) $obj->getId();
         }
         // if key === null
         self::$instances[$key] = $obj;
     }
 }
Пример #17
0
 /**
  * @param FileSync $fileSync
  */
 protected static function purgeFileSync(FileSync $fileSync)
 {
     KalturaLog::info("Purging file sync [" . $fileSync->getId() . "]");
     $fullPath = $fileSync->getFullPath();
     if ($fullPath && file_exists($fullPath)) {
         KalturaLog::debug("Purging file sync [" . $fileSync->getId() . "] path [{$fullPath}]");
         if (is_dir($fullPath)) {
             $command = "rm -fr {$fullPath}";
             KalturaLog::debug("Executing: {$command}");
             if (self::$dryRun) {
                 self::incrementSummary('dirs');
             } else {
                 $returnedValue = null;
                 passthru($command, $returnedValue);
                 if ($returnedValue === 0) {
                     self::incrementSummary('dirs');
                 } else {
                     KalturaLog::err("Failed purging file sync [" . $fileSync->getId() . "] directory path [{$fullPath}]");
                     return;
                 }
             }
         } else {
             $fileSize = filesize($fullPath);
             if (self::$dryRun || unlink($fullPath)) {
                 self::incrementSummary('bytes', $fileSize);
                 self::incrementSummary('files');
             } else {
                 KalturaLog::err("Failed purging file sync [" . $fileSync->getId() . "] file path [{$fullPath}]");
                 return;
             }
         }
     } else {
         KalturaLog::debug("File sync [" . $fileSync->getId() . "] path [{$fullPath}] does not exist");
     }
     $fileSync->setStatus(FileSync::FILE_SYNC_STATUS_PURGED);
     $fileSync->save();
 }