/** * @return BaseObject */ public function getObject() { switch ($this->getObjectType()) { case BatchJobObjectType::ENTRY: entryPeer::setUseCriteriaFilter(false); $object = entryPeer::retrieveByPK($this->getObjectId()); entryPeer::setUseCriteriaFilter(true); return $object; case BatchJobObjectType::ASSET: assetPeer::setUseCriteriaFilter(false); $object = assetPeer::retrieveById($this->getObjectId()); assetPeer::setUseCriteriaFilter(true); return $object; case BatchJobObjectType::CATEGORY: categoryPeer::setUseCriteriaFilter(false); $object = categoryPeer::retrieveByPK($this->getObjectId()); categoryPeer::setUseCriteriaFilter(true); return $object; case BatchJobObjectType::FILE_SYNC: FileSyncPeer::setUseCriteriaFilter(false); $object = FileSyncPeer::retrieveByPK($this->getObjectId()); FileSyncPeer::setUseCriteriaFilter(true); return $object; default: // TODO implement IBatchable in relevant plugins return KalturaPluginManager::loadObject('IBatchable', $this->getObjectId()); } return $this->aRootJob; }
public static function wasFileSyncLimitationReached($object_id, $version, $object_type, $object_sub_type) { if ($version > self::FILE_SYNC_MIN_VERSION_VALIDATE) { $queryVersion = $version - self::FILE_SYNC_MIN_VERSION_VALIDATE; $c = new Criteria(); $c->add(FileSyncPeer::OBJECT_ID, $object_id); $c->add(FileSyncPeer::OBJECT_TYPE, $object_type); $c->add(FileSyncPeer::OBJECT_SUB_TYPE, $object_sub_type); //Version in the db is of type varchar so to avoid string compare we need to add this as custom $c->add(FileSyncPeer::VERSION, FileSyncPeer::VERSION . " <= {$queryVersion}", Criteria::CUSTOM); $c->addDescendingOrderByColumn(FileSyncPeer::CREATED_AT); FileSyncPeer::setUseCriteriaFilter(false); $res = FileSyncPeer::doSelectOne($c); FileSyncPeer::setUseCriteriaFilter(true); if ($res) { if ($res->getCreatedAt(null) > time() - 86400) { return true; } } } return false; }
/** * batch lockPendingFileSyncs action locks file syncs for import by the file sync periodic worker * * @action lockPendingFileSyncs * @param KalturaFileSyncFilter $filter * @param int $workerId The id of the file sync import worker * @param int $sourceDc The id of the DC from which the file syncs should be pulled * @param int $maxCount The maximum number of file syncs that should be returned * @param int $maxSize The maximum total size of file syncs that should be returned, this limit may be exceeded by one file sync * @return KalturaLockFileSyncsResponse */ function lockPendingFileSyncsAction(KalturaFileSyncFilter $filter, $workerId, $sourceDc, $maxCount, $maxSize = null) { // need to explicitly disable the cache since this action may not perform any queries kApiCache::disableConditionalCache(); // for dual dc deployments, if source dc is not specified, set it to the remote dc if ($sourceDc < 0) { $sourceDc = 1 - kDataCenterMgr::getCurrentDcId(); } // get caches $keysCache = kCacheManager::getSingleLayerCache(kCacheManager::CACHE_TYPE_QUERY_CACHE_KEYS); if (!$keysCache) { throw new KalturaAPIException(MultiCentersErrors::GET_KEYS_CACHE_FAILED); } $lockCache = kCacheManager::getSingleLayerCache(kCacheManager::CACHE_TYPE_LOCK_KEYS); if (!$lockCache) { throw new KalturaAPIException(MultiCentersErrors::GET_LOCK_CACHE_FAILED); } // get the max id / last id $maxId = $keysCache->get(self::MAX_FILESYNC_ID_PREFIX . $sourceDc); if (!$maxId) { throw new KalturaAPIException(MultiCentersErrors::GET_MAX_FILESYNC_ID_FAILED, $sourceDc); } $initialLastId = $keysCache->get(self::LAST_FILESYNC_ID_PREFIX . $workerId); KalturaLog::info("got lastId [{$initialLastId}] for worker [{$workerId}]"); $lastId = $initialLastId ? $initialLastId : $maxId; // created at less than handled explicitly $createdAtLessThanOrEqual = $filter->createdAtLessThanOrEqual; $filter->createdAtLessThanOrEqual = null; // build the criteria $fileSyncFilter = new FileSyncFilter(); $filter->toObject($fileSyncFilter); $baseCriteria = new Criteria(); $fileSyncFilter->attachToCriteria($baseCriteria); $baseCriteria->add(FileSyncPeer::STATUS, FileSync::FILE_SYNC_STATUS_PENDING); $baseCriteria->add(FileSyncPeer::FILE_TYPE, FileSync::FILE_SYNC_FILE_TYPE_FILE); $baseCriteria->add(FileSyncPeer::DC, kDataCenterMgr::getCurrentDcId()); $baseCriteria->addAscendingOrderByColumn(FileSyncPeer::ID); $baseCriteria->setLimit(self::MAX_FILESYNCS_PER_CHUNK); $lockedFileSyncs = array(); $lockedFileSyncsSize = 0; $limitReached = false; $selectCount = 0; $lastSelectId = 0; $done = false; while (!$done && $selectCount < self::MAX_FILESYNC_QUERIES_PER_CALL) { // make sure last id is always increasing if ($lastId <= $lastSelectId) { KalturaLog::info("last id was decremented {$lastId} <= {$lastSelectId}, stopping"); break; } $lastSelectId = $lastId; // clear the instance pool every once in a while (not clearing every time since // some objects repeat between selects) $selectCount++; if ($selectCount % 5 == 0) { FileSyncPeer::clearInstancePool(); } // get a chunk of file syncs // Note: starting slightly before the last id, because the ids may arrive out of order in the mysql replication $c = clone $baseCriteria; $idCriterion = $c->getNewCriterion(FileSyncPeer::ID, $lastId - 100, Criteria::GREATER_THAN); $idCriterion->addAnd($c->getNewCriterion(FileSyncPeer::ID, $maxId, Criteria::LESS_THAN)); $c->addAnd($idCriterion); // Note: disabling the criteria because it accumulates more and more criterions, and the status was already explicitly added // once that bug is fixed, this can be removed FileSyncPeer::setUseCriteriaFilter(false); $fileSyncs = FileSyncPeer::doSelect($c); FileSyncPeer::setUseCriteriaFilter(true); if (!$fileSyncs) { $lastId = $maxId; break; } // if we got less than the limit no reason to perform any more queries if (count($fileSyncs) < self::MAX_FILESYNCS_PER_CHUNK) { $done = true; } $lastFileSync = end($fileSyncs); $lastId = $lastFileSync->getId(); // filter by source dc foreach ($fileSyncs as $index => $fileSync) { if ($fileSync->getOriginalDc() != $sourceDc) { unset($fileSyncs[$index]); } } // filter by object type / sub type $fileSyncs = array_filter($fileSyncs, array('FileSyncImportBatchService', 'shouldSyncFileObjectType')); if (!$fileSyncs) { continue; } // filter by created at if ($createdAtLessThanOrEqual) { $firstFileSync = reset($fileSyncs); $prevLastId = $firstFileSync->getId(); foreach ($fileSyncs as $index => $fileSync) { if ($fileSync->getCreatedAt(null) > $createdAtLessThanOrEqual) { $done = true; unset($fileSyncs[$index]); if (!is_null($prevLastId)) { $lastId = $prevLastId; $prevLastId = null; } } else { $prevLastId = $fileSync->getId(); } } if (!$fileSyncs) { break; } } // get locked file syncs with multi get $lockKeys = array(); foreach ($fileSyncs as $fileSync) { $lockKeys[] = self::LOCK_KEY_PREFIX . $fileSync->getId(); } $lockKeys = $lockCache->get($lockKeys); // try to lock file syncs foreach ($fileSyncs as $fileSync) { $curKey = self::LOCK_KEY_PREFIX . $fileSync->getId(); if (isset($lockKeys[$curKey])) { KalturaLog::info('file sync ' . $fileSync->getId() . ' already locked'); continue; } if (!$lockCache->add($curKey, true, self::LOCK_EXPIRY)) { KalturaLog::info('failed to lock file sync ' . $fileSync->getId()); continue; } KalturaLog::info('locked file sync ' . $fileSync->getId()); // get the original id if not set if (!$fileSync->getOriginalId()) { $originalFileSync = self::getOriginalFileSync($fileSync); if (!$originalFileSync) { KalturaLog::info('failed to get original file sync for ' . $fileSync->getId()); continue; } $fileSync->setOriginalId($originalFileSync->getId()); $fileSync->setCustomDataObj(); // update $fileSync->custom_data so that originalId will be set by fromObject } // add to the result set $lockedFileSyncs[] = $fileSync; $lockedFileSyncsSize += $fileSync->getFileSize(); if (count($lockedFileSyncs) >= $maxCount || $maxSize && $lockedFileSyncsSize >= $maxSize) { $lastId = $fileSync->getId(); $limitReached = true; break; } } if ($limitReached) { break; } } // update the last id // Note: it is possible that the last id will go back in case of race condition, // but the only effect of this is that some file syncs will be scanned again if (!$initialLastId || $lastId > $initialLastId) { KalturaLog::info("setting lastId to [{$lastId}] for worker [{$workerId}]"); $keysCache->set(self::LAST_FILESYNC_ID_PREFIX . $workerId, $lastId); } // make sure all file syncs have a path foreach ($lockedFileSyncs as $fileSync) { if ($fileSync->getFileRoot() && $fileSync->getFilePath()) { continue; } $fileSyncKey = kFileSyncUtils::getKeyForFileSync($fileSync); list($fileRoot, $realPath) = kPathManager::getFilePathArr($fileSyncKey); $fileSync->setFileRoot($fileRoot); $fileSync->setFilePath($realPath); } // build the response object $sourceDc = kDataCenterMgr::getDcById($sourceDc); $result = new KalturaLockFileSyncsResponse(); $result->fileSyncs = KalturaFileSyncArray::fromDbArray($lockedFileSyncs, $this->getResponseProfile()); $result->limitReached = $limitReached; $result->dcSecret = $sourceDc["secret"]; $result->baseUrl = isset($sourceDc["fileSyncImportUrl"]) ? $sourceDc["fileSyncImportUrl"] : $sourceDc["url"]; return $result; }
/** * Code to be run after inserting to database * @param PropelPDO $con */ public function postInsert(PropelPDO $con = null) { FileSyncPeer::setUseCriteriaFilter(false); $this->reload(); FileSyncPeer::setUseCriteriaFilter(true); kEventsManager::raiseEvent(new kObjectCreatedEvent($this)); if ($this->copiedFrom) { kEventsManager::raiseEvent(new kObjectCopiedEvent($this->copiedFrom, $this)); } }
if ($fileSync) { $fileSyncs[] = $fileSync; } $fileSync = kFileSyncUtils::getLocalFileSyncForKey($uiConf->getSyncKey(uiConf::FILE_SYNC_UICONF_SUB_TYPE_CONFIG), false); if ($fileSync) { $fileSyncs[] = $fileSync; } $fileSync = kFileSyncUtils::getLocalFileSyncForKey($uiConf->getSyncKey(uiConf::FILE_SYNC_UICONF_SUB_TYPE_FEATURES), false); if ($fileSync) { $fileSyncs[] = $fileSync; } if (empty($fileSyncs)) { die("No file_sync found for UI-Conf ID [" . $uiConfId . "]"); } uiConfPeer::setUseCriteriaFilter(true); FileSyncPeer::setUseCriteriaFilter(true); foreach ($fileSyncs as $fileSync) { if (!file_exists($fileSync->getFullPath())) { die("UI-Conf file ID [" . $fileSync->getId() . "] file not found"); } } foreach ($fileSyncs as $fileSync) { if (method_exists($fileSync, 'setDeletedId')) { $fileSync->setDeletedId(0); } $fileSync->setStatus(FileSync::FILE_SYNC_STATUS_READY); $fileSync->save(); } $uiConf->setStatus(uiConf::UI_CONF_STATUS_READY); $uiConf->save(); KalturaLog::debug('Done');
protected function restoreFileSyncs(array $fileSyncs) { foreach ($fileSyncs as $fileSync) { $shouldUnDelete = false; if ($fileSync->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_FILE || $fileSync->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_URL) { $shouldUnDelete = true; } else { if ($fileSync->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_LINK) { $linkedId = $fileSync->getLinkedId(); FileSyncPeer::setUseCriteriaFilter(false); $linkedFileSync = FileSyncPeer::retrieveByPK($linkedId); FileSyncPeer::setUseCriteriaFilter(true); if ($linkedFileSync->getStatus() == FileSync::FILE_SYNC_STATUS_READY) { $shouldUnDelete = true; kFileSyncUtils::incrementLinkCountForFileSync($linkedFileSync); } } } if ($shouldUnDelete) { $fileSync->setStatus(FileSync::FILE_SYNC_STATUS_READY); } else { $fileSync->setStatus(FileSync::FILE_SYNC_STATUS_ERROR); } $fileSync->save(); } }
/** * Reloads this object from datastore based on primary key and (optionally) resets all associated objects. * * This will only work if the object has been saved and has a valid primary key set. * * @param boolean $deep (optional) Whether to also de-associated any related objects. * @param PropelPDO $con (optional) The PropelPDO connection to use. * @return void * @throws PropelException - if this object is deleted, unsaved or doesn't have pk match in db */ public function reload($deep = false, PropelPDO $con = null) { if ($this->isDeleted()) { throw new PropelException("Cannot reload a deleted object."); } if ($this->isNew()) { throw new PropelException("Cannot reload an unsaved object."); } if ($con === null) { $con = Propel::getConnection(FileSyncPeer::DATABASE_NAME, Propel::CONNECTION_READ); } // We don't need to alter the object instance pool; we're just modifying this instance // already in the pool. FileSyncPeer::setUseCriteriaFilter(false); $criteria = $this->buildPkeyCriteria(); FileSyncPeer::addSelectColumns($criteria); $stmt = BasePeer::doSelect($criteria, $con); FileSyncPeer::setUseCriteriaFilter(true); $row = $stmt->fetch(PDO::FETCH_NUM); $stmt->closeCursor(); if (!$row) { throw new PropelException('Cannot find matching row in the database to reload object values.'); } $this->hydrate($row, 0, true); // rehydrate if ($deep) { // also de-associate any related objects? } // if (deep) }
protected static function init() { kEventsManager::enableDeferredEvents(false); MetadataProfilePeer::setUseCriteriaFilter(false); MetadataPeer::setUseCriteriaFilter(false); entryPeer::setUseCriteriaFilter(false); uiConfPeer::setUseCriteriaFilter(false); assetPeer::setUseCriteriaFilter(false); PartnerPeer::setUseCriteriaFilter(false); FileSyncPeer::setUseCriteriaFilter(false); $options = getopt('hrl:p:o:b:e:', array('real-run', 'error-objects', 'old-versions', 'blocked-partners', 'files')); if (isset($options['h'])) { self::failWrongInputs(); } if (isset($options['blocked-partners'])) { self::$deleteDeletedPartnersFileSyncs = true; } if (isset($options['error-objects'])) { self::$deleteErrorObjects = true; } if (isset($options['old-versions'])) { self::$deleteOldVersions = true; } if (isset($options['files'])) { self::$purgeDeletedFileSyncs = true; } if (isset($options['r']) || isset($options['real-run'])) { self::$dryRun = false; } KalturaStatement::setDryRun(self::$dryRun); $cacheFilePath = kConf::get('cache_root_path') . '/scripts/deleteOldContent.cache'; if (file_exists($cacheFilePath)) { $cache = unserialize(file_get_contents($cacheFilePath)); if (isset($cache['oldVersionsStartUpdatedAt'])) { self::$oldVersionsStartUpdatedAt = $cache['oldVersionsStartUpdatedAt']; } if (isset($cache['purgeStartUpdatedAt'])) { self::$purgeStartUpdatedAt = $cache['purgeStartUpdatedAt']; } } if (!self::$purgeStartUpdatedAt) { $criteria = new Criteria(); $criteria->add(FileSyncPeer::UPDATED_AT, 0, Criteria::GREATER_THAN); $criteria->add(FileSyncPeer::DC, kDataCenterMgr::getCurrentDcId()); $criteria->add(FileSyncPeer::STATUS, FileSync::FILE_SYNC_STATUS_DELETED); $criteria->addSelectColumn('UNIX_TIMESTAMP(MIN(' . FileSyncPeer::UPDATED_AT . '))'); $stmt = FileSyncPeer::doSelectStmt($criteria); $mins = $stmt->fetchAll(PDO::FETCH_COLUMN); if (count($mins)) { self::$purgeStartUpdatedAt = reset($mins); } } if (is_null(self::$purgeStartUpdatedAt)) { self::$purgeStartUpdatedAt = 0; } self::$purgeNextStartUpdatedAt = self::$purgeStartUpdatedAt; $oldVersionsUpdatedAtPeriod = 30; // days if (isset($options['o'])) { if (!is_numeric($options['o']) || $options['o'] < 0) { self::failWrongInputs("Period of old versions to delete must be positive numeric of days"); } $oldVersionsUpdatedAtPeriod = $options['o']; } foreach (self::$oldVersionsStartUpdatedAt as $objectType => $oldVersionsStartUpdatedAt) { self::$oldVersionsEndUpdatedAt[$objectType] = $oldVersionsStartUpdatedAt + $oldVersionsUpdatedAtPeriod * 60 * 60 * 24; } // days $purgeUpdatedAtPeriod = 30; // days if (isset($options['p'])) { if (!is_numeric($options['p']) || $options['p'] < 0) { self::failWrongInputs("Period of purge must be positive numeric of days"); } $purgeUpdatedAtPeriod = $options['p']; } self::$purgeEndUpdatedAt = self::$purgeStartUpdatedAt + $purgeUpdatedAtPeriod * 60 * 60 * 24; // days $oldPartnersUpdatedAtPeriod = 24; // months if (isset($options['b'])) { if (!is_numeric($options['b']) || $options['b'] < 0) { self::failWrongInputs("Period of blocked partners to delete must be positive numeric of months"); } $oldPartnersUpdatedAtPeriod = $options['b']; } self::$oldPartnersUpdatedAt = time() - $oldPartnersUpdatedAtPeriod * 60 * 60 * 24 * 30; // months $errObjectsUpdatedAtPeriod = 24; // months if (isset($options['e'])) { if (!is_numeric($options['e']) || $options['e'] < 0) { self::failWrongInputs("Period of error objects to delete must be positive numeric of months"); } $errObjectsUpdatedAtPeriod = $options['e']; } self::$errObjectsUpdatedAt = time() - $errObjectsUpdatedAtPeriod * 60 * 60 * 24 * 30; // months if (isset($options['l'])) { if (!is_numeric($options['l']) || $options['l'] < 0) { self::failWrongInputs("Limit querymust be positive numeric value"); } self::$queryLimit = $options['l']; } }
/** * @param BatchJob $dbBatchJob * @param kConvertCollectionJobData $data * @return BatchJob */ public static function handleDeleteFileFinished(BatchJob $dbBatchJob, kDeleteFileJobData $data) { //Change status of the filesync to "purged" FileSyncPeer::setUseCriteriaFilter(false); $fileSyncFroDeletedFile = FileSyncPeer::retrieveByFileSyncKey($data->getSyncKey(), true); FileSyncPeer::setUseCriteriaFilter(true); $fileSyncFroDeletedFile->setStatus(FileSync::FILE_SYNC_STATUS_PURGED); $fileSyncFroDeletedFile->save(); return $dbBatchJob; }
/** * @param BatchJob $dbBatchJob * @param kConvertCollectionJobData $data * @return BatchJob */ public static function handleDeleteFileFinished(BatchJob $dbBatchJob, kDeleteFileJobData $data) { KalturaLog::debug("File delete finished for file path: " . $data->getLocalFileSyncPath() . ", data center: " . $dbBatchJob->getDc()); //Change status of the filesync to "purged" FileSyncPeer::setUseCriteriaFilter(false); $fileSyncFroDeletedFile = FileSyncPeer::retrieveByFileSyncKey($data->getSyncKey(), true); FileSyncPeer::setUseCriteriaFilter(true); $fileSyncFroDeletedFile->setStatus(FileSync::FILE_SYNC_STATUS_PURGED); $fileSyncFroDeletedFile->save(); return $dbBatchJob; }
{ fwrite(STDERR, $msg); } $iniDir = __DIR__ . '/../../../configurations/batch/'; if (isset($argc) && $argc > 2) { $iniDir = $argv[2]; } $keysCache = kCacheManager::getSingleLayerCache(kCacheManager::CACHE_TYPE_QUERY_CACHE_KEYS); if (!$keysCache) { die('failed to get keys cache'); } // get the max id / last id $maxId = $keysCache->get(MAX_FILESYNC_ID_PREFIX . kDataCenterMgr::getCurrentDcId()); writeOutput('Max id for dc [' . kDataCenterMgr::getCurrentDcId() . '] is [' . $maxId . "]\n"); $excludeFileSyncMap = getExcludeFileSyncMap(); FileSyncPeer::setUseCriteriaFilter(false); $fileSyncWorkers = getFileSyncWorkers($iniDir); foreach ($fileSyncWorkers as $fileSyncWorker) { $workerId = $fileSyncWorker['id']; $workerName = $fileSyncWorker['name']; $filter = $fileSyncWorker['filter']; $lastId = $keysCache->get(LAST_FILESYNC_ID_PREFIX . $workerId); // build the base criteria $baseCriteria = new Criteria(); $baseCriteria->add(FileSyncPeer::STATUS, FileSync::FILE_SYNC_STATUS_PENDING); $baseCriteria->add(FileSyncPeer::FILE_TYPE, FileSync::FILE_SYNC_FILE_TYPE_FILE); $baseCriteria->add(FileSyncPeer::DC, kDataCenterMgr::getCurrentDcId()); $idCriterion = $baseCriteria->getNewCriterion(FileSyncPeer::ID, $lastId - 100, Criteria::GREATER_THAN); $idCriterion->addAnd($baseCriteria->getNewCriterion(FileSyncPeer::ID, $maxId, Criteria::LESS_THAN)); $baseCriteria->addAnd($idCriterion); // init size limits from filter
function restoreFileSyncByKey(FileSyncKey $fileSyncKey) { KalturaLog::debug("file sync key: {$fileSyncKey}"); /* @var $entryFileSyncKey FileSyncKey */ FileSyncPeer::setUseCriteriaFilter(false); $fileSyncs = FileSyncPeer::retrieveAllByFileSyncKey($fileSyncKey); FileSyncPeer::setUseCriteriaFilter(true); foreach ($fileSyncs as $fileSync) { if ($fileSync->getStatus() == FileSync::FILE_SYNC_STATUS_DELETED) { $shouldUnDelete = false; if ($fileSync->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_FILE || $fileSync->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_URL) { if (file_exists($fileSync->getFullPath())) { $shouldUnDelete = true; } } else { if ($fileSync->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_LINK) { $linkedId = $fileSync->getLinkedId(); FileSyncPeer::setUseCriteriaFilter(false); $linkedFileSync = FileSyncPeer::retrieveByPK($linkedId); FileSyncPeer::setUseCriteriaFilter(true); if ($linkedFileSync->getStatus() == FileSync::FILE_SYNC_STATUS_DELETED && file_exists($linkedFileSync->getFullPath())) { $shouldUnDelete = true; } } else { if ($fileSync->getFileType() == FileSync::FILE_SYNC_FILE_TYPE_CACHE) { $shouldUnDelete = false; } } } if ($shouldUnDelete) { $fileSync->setStatus(FileSync::FILE_SYNC_STATUS_READY); } else { $fileSync->setStatus(FileSync::FILE_SYNC_STATUS_ERROR); } } $fileSync->save(); } }