示例#1
0
 /**
  * List file syce objects by filter and pager
  *
  * @action list
  * @param KalturaFileSyncFilter $filter
  * @param KalturaFilterPager $pager
  * @return KalturaFileSyncListResponse
  */
 function listAction(KalturaFileSyncFilter $filter = null, KalturaFilterPager $pager = null)
 {
     if (!$filter) {
         $filter = new KalturaFileSyncFilter();
     }
     if (!$pager) {
         $pager = new KalturaFilterPager();
     }
     $fileSyncFilter = new FileSyncFilter();
     $filter->toObject($fileSyncFilter);
     $c = new Criteria();
     $fileSyncFilter->attachToCriteria($c);
     $totalCount = FileSyncPeer::doCount($c);
     $pager->attachToCriteria($c);
     $dbList = FileSyncPeer::doSelect($c);
     $list = KalturaFileSyncArray::fromDbArray($dbList, $this->getResponseProfile());
     $response = new KalturaFileSyncListResponse();
     $response->objects = $list;
     $response->totalCount = $totalCount;
     return $response;
 }
 /**
  * batch lockPendingFileSyncs action locks file syncs for import by the file sync periodic worker
  *
  * @action lockPendingFileSyncs
  * @param KalturaFileSyncFilter $filter
  * @param int $workerId The id of the file sync import worker 
  * @param int $sourceDc The id of the DC from which the file syncs should be pulled
  * @param int $maxCount The maximum number of file syncs that should be returned
  * @param int $maxSize The maximum total size of file syncs that should be returned, this limit may be exceeded by one file sync
  * @return KalturaLockFileSyncsResponse
  */
 function lockPendingFileSyncsAction(KalturaFileSyncFilter $filter, $workerId, $sourceDc, $maxCount, $maxSize = null)
 {
     // need to explicitly disable the cache since this action may not perform any queries
     kApiCache::disableConditionalCache();
     // for dual dc deployments, if source dc is not specified, set it to the remote dc
     if ($sourceDc < 0) {
         $sourceDc = 1 - kDataCenterMgr::getCurrentDcId();
     }
     // get caches
     $keysCache = kCacheManager::getSingleLayerCache(kCacheManager::CACHE_TYPE_QUERY_CACHE_KEYS);
     if (!$keysCache) {
         throw new KalturaAPIException(MultiCentersErrors::GET_KEYS_CACHE_FAILED);
     }
     $lockCache = kCacheManager::getSingleLayerCache(kCacheManager::CACHE_TYPE_LOCK_KEYS);
     if (!$lockCache) {
         throw new KalturaAPIException(MultiCentersErrors::GET_LOCK_CACHE_FAILED);
     }
     // get the max id / last id
     $maxId = $keysCache->get(self::MAX_FILESYNC_ID_PREFIX . $sourceDc);
     if (!$maxId) {
         throw new KalturaAPIException(MultiCentersErrors::GET_MAX_FILESYNC_ID_FAILED, $sourceDc);
     }
     $initialLastId = $keysCache->get(self::LAST_FILESYNC_ID_PREFIX . $workerId);
     KalturaLog::info("got lastId [{$initialLastId}] for worker [{$workerId}]");
     $lastId = $initialLastId ? $initialLastId : $maxId;
     // created at less than handled explicitly
     $createdAtLessThanOrEqual = $filter->createdAtLessThanOrEqual;
     $filter->createdAtLessThanOrEqual = null;
     // build the criteria
     $fileSyncFilter = new FileSyncFilter();
     $filter->toObject($fileSyncFilter);
     $baseCriteria = new Criteria();
     $fileSyncFilter->attachToCriteria($baseCriteria);
     $baseCriteria->add(FileSyncPeer::STATUS, FileSync::FILE_SYNC_STATUS_PENDING);
     $baseCriteria->add(FileSyncPeer::FILE_TYPE, FileSync::FILE_SYNC_FILE_TYPE_FILE);
     $baseCriteria->add(FileSyncPeer::DC, kDataCenterMgr::getCurrentDcId());
     $baseCriteria->addAscendingOrderByColumn(FileSyncPeer::ID);
     $baseCriteria->setLimit(self::MAX_FILESYNCS_PER_CHUNK);
     $lockedFileSyncs = array();
     $lockedFileSyncsSize = 0;
     $limitReached = false;
     $selectCount = 0;
     $lastSelectId = 0;
     $done = false;
     while (!$done && $selectCount < self::MAX_FILESYNC_QUERIES_PER_CALL) {
         // make sure last id is always increasing
         if ($lastId <= $lastSelectId) {
             KalturaLog::info("last id was decremented {$lastId} <= {$lastSelectId}, stopping");
             break;
         }
         $lastSelectId = $lastId;
         // clear the instance pool every once in a while (not clearing every time since
         //	some objects repeat between selects)
         $selectCount++;
         if ($selectCount % 5 == 0) {
             FileSyncPeer::clearInstancePool();
         }
         // get a chunk of file syncs
         // Note: starting slightly before the last id, because the ids may arrive out of order in the mysql replication
         $c = clone $baseCriteria;
         $idCriterion = $c->getNewCriterion(FileSyncPeer::ID, $lastId - 100, Criteria::GREATER_THAN);
         $idCriterion->addAnd($c->getNewCriterion(FileSyncPeer::ID, $maxId, Criteria::LESS_THAN));
         $c->addAnd($idCriterion);
         // Note: disabling the criteria because it accumulates more and more criterions, and the status was already explicitly added
         //		once that bug is fixed, this can be removed
         FileSyncPeer::setUseCriteriaFilter(false);
         $fileSyncs = FileSyncPeer::doSelect($c);
         FileSyncPeer::setUseCriteriaFilter(true);
         if (!$fileSyncs) {
             $lastId = $maxId;
             break;
         }
         // if we got less than the limit no reason to perform any more queries
         if (count($fileSyncs) < self::MAX_FILESYNCS_PER_CHUNK) {
             $done = true;
         }
         $lastFileSync = end($fileSyncs);
         $lastId = $lastFileSync->getId();
         // filter by source dc
         foreach ($fileSyncs as $index => $fileSync) {
             if ($fileSync->getOriginalDc() != $sourceDc) {
                 unset($fileSyncs[$index]);
             }
         }
         // filter by object type / sub type
         $fileSyncs = array_filter($fileSyncs, array('FileSyncImportBatchService', 'shouldSyncFileObjectType'));
         if (!$fileSyncs) {
             continue;
         }
         // filter by created at
         if ($createdAtLessThanOrEqual) {
             $firstFileSync = reset($fileSyncs);
             $prevLastId = $firstFileSync->getId();
             foreach ($fileSyncs as $index => $fileSync) {
                 if ($fileSync->getCreatedAt(null) > $createdAtLessThanOrEqual) {
                     $done = true;
                     unset($fileSyncs[$index]);
                     if (!is_null($prevLastId)) {
                         $lastId = $prevLastId;
                         $prevLastId = null;
                     }
                 } else {
                     $prevLastId = $fileSync->getId();
                 }
             }
             if (!$fileSyncs) {
                 break;
             }
         }
         // get locked file syncs with multi get
         $lockKeys = array();
         foreach ($fileSyncs as $fileSync) {
             $lockKeys[] = self::LOCK_KEY_PREFIX . $fileSync->getId();
         }
         $lockKeys = $lockCache->get($lockKeys);
         // try to lock file syncs
         foreach ($fileSyncs as $fileSync) {
             $curKey = self::LOCK_KEY_PREFIX . $fileSync->getId();
             if (isset($lockKeys[$curKey])) {
                 KalturaLog::info('file sync ' . $fileSync->getId() . ' already locked');
                 continue;
             }
             if (!$lockCache->add($curKey, true, self::LOCK_EXPIRY)) {
                 KalturaLog::info('failed to lock file sync ' . $fileSync->getId());
                 continue;
             }
             KalturaLog::info('locked file sync ' . $fileSync->getId());
             // get the original id if not set
             if (!$fileSync->getOriginalId()) {
                 $originalFileSync = self::getOriginalFileSync($fileSync);
                 if (!$originalFileSync) {
                     KalturaLog::info('failed to get original file sync for ' . $fileSync->getId());
                     continue;
                 }
                 $fileSync->setOriginalId($originalFileSync->getId());
                 $fileSync->setCustomDataObj();
                 // update $fileSync->custom_data so that originalId will be set by fromObject
             }
             // add to the result set
             $lockedFileSyncs[] = $fileSync;
             $lockedFileSyncsSize += $fileSync->getFileSize();
             if (count($lockedFileSyncs) >= $maxCount || $maxSize && $lockedFileSyncsSize >= $maxSize) {
                 $lastId = $fileSync->getId();
                 $limitReached = true;
                 break;
             }
         }
         if ($limitReached) {
             break;
         }
     }
     // update the last id
     // Note: it is possible that the last id will go back in case of race condition,
     //		but the only effect of this is that some file syncs will be scanned again
     if (!$initialLastId || $lastId > $initialLastId) {
         KalturaLog::info("setting lastId to [{$lastId}] for worker [{$workerId}]");
         $keysCache->set(self::LAST_FILESYNC_ID_PREFIX . $workerId, $lastId);
     }
     // make sure all file syncs have a path
     foreach ($lockedFileSyncs as $fileSync) {
         if ($fileSync->getFileRoot() && $fileSync->getFilePath()) {
             continue;
         }
         $fileSyncKey = kFileSyncUtils::getKeyForFileSync($fileSync);
         list($fileRoot, $realPath) = kPathManager::getFilePathArr($fileSyncKey);
         $fileSync->setFileRoot($fileRoot);
         $fileSync->setFilePath($realPath);
     }
     // build the response object
     $sourceDc = kDataCenterMgr::getDcById($sourceDc);
     $result = new KalturaLockFileSyncsResponse();
     $result->fileSyncs = KalturaFileSyncArray::fromDbArray($lockedFileSyncs, $this->getResponseProfile());
     $result->limitReached = $limitReached;
     $result->dcSecret = $sourceDc["secret"];
     $result->baseUrl = isset($sourceDc["fileSyncImportUrl"]) ? $sourceDc["fileSyncImportUrl"] : $sourceDc["url"];
     return $result;
 }