public function clearMemory()
{
accessControlPeer::clearInstancePool();
BatchJobPeer::clearInstancePool();
BulkUploadResultPeer::clearInstancePool();
categoryPeer::clearInstancePool();
EmailIngestionProfilePeer::clearInstancePool();
entryPeer::clearInstancePool();
FileSyncPeer::clearInstancePool();
flavorAssetPeer::clearInstancePool();
flavorParamsConversionProfilePeer::clearInstancePool();
flavorParamsOutputPeer::clearInstancePool();
flavorParamsPeer::clearInstancePool();
kshowPeer::clearInstancePool();
mediaInfoPeer::clearInstancePool();
moderationFlagPeer::clearInstancePool();
moderationPeer::clearInstancePool();
notificationPeer::clearInstancePool();
roughcutEntryPeer::clearInstancePool();
SchedulerConfigPeer::clearInstancePool();
SchedulerPeer::clearInstancePool();
SchedulerStatusPeer::clearInstancePool();
SchedulerWorkerPeer::clearInstancePool();
StorageProfilePeer::clearInstancePool();
syndicationFeedPeer::clearInstancePool();
TrackEntryPeer::clearInstancePool();
uiConfPeer::clearInstancePool();
UploadTokenPeer::clearInstancePool();
// TODO clear default filters
// TODO call all memory cleaner plugins
if (function_exists('gc_collect_cycles')) {
// php 5.3 and above
gc_collect_cycles();
}
}
/**
* batch lockPendingFileSyncs action locks file syncs for import by the file sync periodic worker
*
* @action lockPendingFileSyncs
* @param KalturaFileSyncFilter $filter
* @param int $workerId The id of the file sync import worker
* @param int $sourceDc The id of the DC from which the file syncs should be pulled
* @param int $maxCount The maximum number of file syncs that should be returned
* @param int $maxSize The maximum total size of file syncs that should be returned, this limit may be exceeded by one file sync
* @return KalturaLockFileSyncsResponse
*/
function lockPendingFileSyncsAction(KalturaFileSyncFilter $filter, $workerId, $sourceDc, $maxCount, $maxSize = null)
{
// need to explicitly disable the cache since this action may not perform any queries
kApiCache::disableConditionalCache();
// for dual dc deployments, if source dc is not specified, set it to the remote dc
if ($sourceDc < 0) {
$sourceDc = 1 - kDataCenterMgr::getCurrentDcId();
}
// get caches
$keysCache = kCacheManager::getSingleLayerCache(kCacheManager::CACHE_TYPE_QUERY_CACHE_KEYS);
if (!$keysCache) {
throw new KalturaAPIException(MultiCentersErrors::GET_KEYS_CACHE_FAILED);
}
$lockCache = kCacheManager::getSingleLayerCache(kCacheManager::CACHE_TYPE_LOCK_KEYS);
if (!$lockCache) {
throw new KalturaAPIException(MultiCentersErrors::GET_LOCK_CACHE_FAILED);
}
// get the max id / last id
$maxId = $keysCache->get(self::MAX_FILESYNC_ID_PREFIX . $sourceDc);
if (!$maxId) {
throw new KalturaAPIException(MultiCentersErrors::GET_MAX_FILESYNC_ID_FAILED, $sourceDc);
}
$initialLastId = $keysCache->get(self::LAST_FILESYNC_ID_PREFIX . $workerId);
KalturaLog::info("got lastId [{$initialLastId}] for worker [{$workerId}]");
$lastId = $initialLastId ? $initialLastId : $maxId;
// created at less than handled explicitly
$createdAtLessThanOrEqual = $filter->createdAtLessThanOrEqual;
$filter->createdAtLessThanOrEqual = null;
// build the criteria
$fileSyncFilter = new FileSyncFilter();
$filter->toObject($fileSyncFilter);
$baseCriteria = new Criteria();
$fileSyncFilter->attachToCriteria($baseCriteria);
$baseCriteria->add(FileSyncPeer::STATUS, FileSync::FILE_SYNC_STATUS_PENDING);
$baseCriteria->add(FileSyncPeer::FILE_TYPE, FileSync::FILE_SYNC_FILE_TYPE_FILE);
$baseCriteria->add(FileSyncPeer::DC, kDataCenterMgr::getCurrentDcId());
$baseCriteria->addAscendingOrderByColumn(FileSyncPeer::ID);
$baseCriteria->setLimit(self::MAX_FILESYNCS_PER_CHUNK);
$lockedFileSyncs = array();
$lockedFileSyncsSize = 0;
$limitReached = false;
$selectCount = 0;
$lastSelectId = 0;
$done = false;
while (!$done && $selectCount < self::MAX_FILESYNC_QUERIES_PER_CALL) {
// make sure last id is always increasing
if ($lastId <= $lastSelectId) {
KalturaLog::info("last id was decremented {$lastId} <= {$lastSelectId}, stopping");
break;
}
$lastSelectId = $lastId;
// clear the instance pool every once in a while (not clearing every time since
// some objects repeat between selects)
$selectCount++;
if ($selectCount % 5 == 0) {
FileSyncPeer::clearInstancePool();
}
// get a chunk of file syncs
// Note: starting slightly before the last id, because the ids may arrive out of order in the mysql replication
$c = clone $baseCriteria;
$idCriterion = $c->getNewCriterion(FileSyncPeer::ID, $lastId - 100, Criteria::GREATER_THAN);
$idCriterion->addAnd($c->getNewCriterion(FileSyncPeer::ID, $maxId, Criteria::LESS_THAN));
$c->addAnd($idCriterion);
// Note: disabling the criteria because it accumulates more and more criterions, and the status was already explicitly added
// once that bug is fixed, this can be removed
FileSyncPeer::setUseCriteriaFilter(false);
$fileSyncs = FileSyncPeer::doSelect($c);
FileSyncPeer::setUseCriteriaFilter(true);
if (!$fileSyncs) {
$lastId = $maxId;
break;
}
// if we got less than the limit no reason to perform any more queries
if (count($fileSyncs) < self::MAX_FILESYNCS_PER_CHUNK) {
$done = true;
}
$lastFileSync = end($fileSyncs);
$lastId = $lastFileSync->getId();
// filter by source dc
foreach ($fileSyncs as $index => $fileSync) {
if ($fileSync->getOriginalDc() != $sourceDc) {
unset($fileSyncs[$index]);
}
}
// filter by object type / sub type
$fileSyncs = array_filter($fileSyncs, array('FileSyncImportBatchService', 'shouldSyncFileObjectType'));
if (!$fileSyncs) {
continue;
}
// filter by created at
if ($createdAtLessThanOrEqual) {
$firstFileSync = reset($fileSyncs);
$prevLastId = $firstFileSync->getId();
foreach ($fileSyncs as $index => $fileSync) {
if ($fileSync->getCreatedAt(null) > $createdAtLessThanOrEqual) {
$done = true;
unset($fileSyncs[$index]);
if (!is_null($prevLastId)) {
$lastId = $prevLastId;
$prevLastId = null;
}
} else {
$prevLastId = $fileSync->getId();
}
}
if (!$fileSyncs) {
break;
}
}
// get locked file syncs with multi get
$lockKeys = array();
foreach ($fileSyncs as $fileSync) {
$lockKeys[] = self::LOCK_KEY_PREFIX . $fileSync->getId();
}
$lockKeys = $lockCache->get($lockKeys);
// try to lock file syncs
foreach ($fileSyncs as $fileSync) {
$curKey = self::LOCK_KEY_PREFIX . $fileSync->getId();
if (isset($lockKeys[$curKey])) {
KalturaLog::info('file sync ' . $fileSync->getId() . ' already locked');
continue;
}
if (!$lockCache->add($curKey, true, self::LOCK_EXPIRY)) {
KalturaLog::info('failed to lock file sync ' . $fileSync->getId());
continue;
}
KalturaLog::info('locked file sync ' . $fileSync->getId());
// get the original id if not set
if (!$fileSync->getOriginalId()) {
$originalFileSync = self::getOriginalFileSync($fileSync);
if (!$originalFileSync) {
KalturaLog::info('failed to get original file sync for ' . $fileSync->getId());
continue;
}
$fileSync->setOriginalId($originalFileSync->getId());
$fileSync->setCustomDataObj();
// update $fileSync->custom_data so that originalId will be set by fromObject
}
// add to the result set
$lockedFileSyncs[] = $fileSync;
$lockedFileSyncsSize += $fileSync->getFileSize();
if (count($lockedFileSyncs) >= $maxCount || $maxSize && $lockedFileSyncsSize >= $maxSize) {
$lastId = $fileSync->getId();
$limitReached = true;
break;
}
}
if ($limitReached) {
break;
}
}
// update the last id
// Note: it is possible that the last id will go back in case of race condition,
// but the only effect of this is that some file syncs will be scanned again
if (!$initialLastId || $lastId > $initialLastId) {
KalturaLog::info("setting lastId to [{$lastId}] for worker [{$workerId}]");
$keysCache->set(self::LAST_FILESYNC_ID_PREFIX . $workerId, $lastId);
}
// make sure all file syncs have a path
foreach ($lockedFileSyncs as $fileSync) {
if ($fileSync->getFileRoot() && $fileSync->getFilePath()) {
continue;
}
$fileSyncKey = kFileSyncUtils::getKeyForFileSync($fileSync);
list($fileRoot, $realPath) = kPathManager::getFilePathArr($fileSyncKey);
$fileSync->setFileRoot($fileRoot);
$fileSync->setFilePath($realPath);
}
// build the response object
$sourceDc = kDataCenterMgr::getDcById($sourceDc);
$result = new KalturaLockFileSyncsResponse();
$result->fileSyncs = KalturaFileSyncArray::fromDbArray($lockedFileSyncs, $this->getResponseProfile());
$result->limitReached = $limitReached;
$result->dcSecret = $sourceDc["secret"];
$result->baseUrl = isset($sourceDc["fileSyncImportUrl"]) ? $sourceDc["fileSyncImportUrl"] : $sourceDc["url"];
return $result;
}
/**
* Method perform a DELETE on the database, given a FileSync or Criteria object OR a primary key value.
*
* @param mixed $values Criteria or FileSync object or primary key or array of primary keys
* which is used to create the DELETE statement
* @param PropelPDO $con the connection to use
* @return int The number of affected rows (if supported by underlying database driver). This includes CASCADE-related rows
* if supported by native driver or if emulated using Propel.
* @throws PropelException Any exceptions caught during processing will be
* rethrown wrapped into a PropelException.
*/
public static function doDelete($values, PropelPDO $con = null)
{
if ($con === null) {
$con = Propel::getConnection(FileSyncPeer::DATABASE_NAME, Propel::CONNECTION_WRITE);
}
if ($values instanceof Criteria) {
// invalidate the cache for all objects of this type, since we have no
// way of knowing (without running a query) what objects should be invalidated
// from the cache based on this Criteria.
FileSyncPeer::clearInstancePool();
// rename for clarity
$criteria = clone $values;
} elseif ($values instanceof FileSync) {
// it's a model object
// invalidate the cache for this single object
FileSyncPeer::removeInstanceFromPool($values);
// create criteria based on pk values
$criteria = $values->buildPkeyCriteria();
} else {
// it's a primary key, or an array of pks
$criteria = new Criteria(self::DATABASE_NAME);
$criteria->add(FileSyncPeer::ID, (array) $values, Criteria::IN);
// invalidate the cache for this object(s)
foreach ((array) $values as $singleval) {
FileSyncPeer::removeInstanceFromPool($singleval);
}
}
// Set the correct dbName
$criteria->setDbName(self::DATABASE_NAME);
$affectedRows = 0;
// initialize var to track total num of affected rows
try {
// use transaction because $criteria could contain info
// for more than one table or we could emulating ON DELETE CASCADE, etc.
$con->beginTransaction();
$affectedRows += BasePeer::doDelete($criteria, $con);
FileSyncPeer::clearRelatedInstancePool();
$con->commit();
return $affectedRows;
} catch (PropelException $e) {
$con->rollBack();
throw $e;
}
}
/**
* gets a source file of current DC, will make sure all links points to that source
* are converted to files on all DCs
*
* @param FileSyncKey $key
* @return void
*/
protected static function convertLinksToFiles(FileSync $fileSync)
{
$linkTotalCount = 0;
/* @var $fileSync FileSync */
// for each source, find its links and fix them
$c = new Criteria();
$c->add(FileSyncPeer::DC, $fileSync->getDc());
$c->add(FileSyncPeer::FILE_TYPE, array(FileSync::FILE_SYNC_FILE_TYPE_LINK, FileSync::FILE_SYNC_FILE_TYPE_URL), Criteria::IN);
$c->add(FileSyncPeer::LINKED_ID, $fileSync->getId());
$c->addAscendingOrderByColumn(FileSyncPeer::PARTNER_ID);
//relink the links into groups of 100 links
$c->setLimit(100);
$links = FileSyncPeer::doSelect($c);
//check if any links were returned in the do select if not no need to continue
if (!count($links)) {
return;
}
// choose the first link and convert it to file
$firstLink = array_shift($links);
/* @var $firstLink FileSync */
if ($firstLink) {
$firstLink->setStatus($fileSync->getStatus());
$firstLink->setFileSize($fileSync->getFileSize());
$firstLink->setFileRoot($fileSync->getFileRoot());
$firstLink->setFilePath($fileSync->getFilePath());
$firstLink->setFileType($fileSync->getFileType());
$firstLink->setLinkedId(0);
// keep it zero instead of null, that's the only way to know it used to be a link.
$firstLink->setIsDir($fileSync->getIsDir());
if (!is_null($fileSync->getOriginalDc())) {
$firstLink->setOriginalDc($fileSync->getOriginalDc());
$firstLink->unsetOriginalId();
// recalculate the original id when importing the file sync
}
$firstLink->save();
}
while (count($links)) {
// change all the rest of the links to point on the new file sync
foreach ($links as $link) {
$linkTotalCount += count($links);
/* @var $link FileSync */
$link->setStatus($fileSync->getStatus());
$link->setLinkedId($firstLink->getId());
$link->save();
}
FileSyncPeer::clearInstancePool();
$links = FileSyncPeer::doSelect($c);
}
if ($firstLink) {
$firstLink->setLinkCount($linkTotalCount);
$firstLink->save();
}
}
/**
* gets a source file of current DC, will make sure all links points to that source
* are converted to files on all DCs
*
* @param FileSyncKey $key
* @return void
*/
public static function convertLinksToFiles(FileSyncKey $key)
{
// fetch sources from all DCs
$c = new Criteria();
$c = FileSyncPeer::getCriteriaForFileSyncKey($key);
$fileSyncList = FileSyncPeer::doSelect($c);
foreach ($fileSyncList as $fileSync) {
/* @var $fileSync FileSync */
// for each source, find its links and fix them
$c = new Criteria();
$c->add(FileSyncPeer::DC, $fileSync->getDc());
$c->add(FileSyncPeer::FILE_TYPE, FileSync::FILE_SYNC_FILE_TYPE_LINK);
$c->add(FileSyncPeer::LINKED_ID, $fileSync->getId());
$c->addAscendingOrderByColumn(FileSyncPeer::PARTNER_ID);
//relink the links into groups of 100 links
$c->setLimit(100);
$links = FileSyncPeer::doSelect($c);
while (count($links)) {
// choose the first link and convert it to file
$firstLink = array_shift($links);
/* @var $firstLink FileSync */
if ($firstLink) {
$firstLink->setStatus($fileSync->getStatus());
$firstLink->setFileSize($fileSync->getFileSize());
$firstLink->setFileRoot($fileSync->getFileRoot());
$firstLink->setFilePath($fileSync->getFilePath());
$firstLink->setWamsAssetId($fileSync->getWamsAssetId());
$firstLink->setWamsUrl($fileSync->getWamsUrl());
$firstLink->setFileType(FileSync::FILE_SYNC_FILE_TYPE_FILE);
$firstLink->setLinkedId(0);
// keep it zero instead of null, that's the only way to know it used to be a link.
$firstLink->setLinkCount(count($links));
$firstLink->save();
}
// change all the rest of the links to point on the new file sync
foreach ($links as $link) {
/* @var $link FileSync */
$link->setStatus($fileSync->getStatus());
$link->setLinkedId($firstLink->getId());
$link->save();
}
FileSyncPeer::clearInstancePool();
$links = FileSyncPeer::doSelect($c);
}
}
}
private function clearMemory()
{
entryPeer::clearInstancePool();
flavorAssetPeer::clearInstancePool();
FileSyncPeer::clearInstancePool();
categoryPeer::clearInstancePool();
if (class_exists('MetadataPeer')) {
MetadataPeer::clearInstancePool();
MetadataProfilePeer::clearInstancePool();
}
}
public static function clearMemory()
{
accessControlPeer::clearInstancePool();
kuserPeer::clearInstancePool();
kshowPeer::clearInstancePool();
entryPeer::clearInstancePool();
// kvotePeer::clearInstancePool();
// commentPeer::clearInstancePool();
// flagPeer::clearInstancePool();
// favoritePeer::clearInstancePool();
// KshowKuserPeer::clearInstancePool();
// MailJobPeer::clearInstancePool();
SchedulerPeer::clearInstancePool();
SchedulerWorkerPeer::clearInstancePool();
SchedulerStatusPeer::clearInstancePool();
SchedulerConfigPeer::clearInstancePool();
ControlPanelCommandPeer::clearInstancePool();
BatchJobPeer::clearInstancePool();
// PriorityGroupPeer::clearInstancePool();
BulkUploadResultPeer::clearInstancePool();
// blockedEmailPeer::clearInstancePool();
// conversionPeer::clearInstancePool();
// flickrTokenPeer::clearInstancePool();
PuserKuserPeer::clearInstancePool();
// PuserRolePeer::clearInstancePool();
PartnerPeer::clearInstancePool();
// WidgetLogPeer::clearInstancePool();
// adminKuserPeer::clearInstancePool();
// notificationPeer::clearInstancePool();
moderationPeer::clearInstancePool();
moderationFlagPeer::clearInstancePool();
roughcutEntryPeer::clearInstancePool();
// widgetPeer::clearInstancePool();
uiConfPeer::clearInstancePool();
// PartnerStatsPeer::clearInstancePool();
// PartnerActivityPeer::clearInstancePool();
ConversionProfilePeer::clearInstancePool();
// ConversionParamsPeer::clearInstancePool();
// KceInstallationErrorPeer::clearInstancePool();
FileSyncPeer::clearInstancePool();
accessControlPeer::clearInstancePool();
mediaInfoPeer::clearInstancePool();
assetParamsPeer::clearInstancePool();
assetParamsOutputPeer::clearInstancePool();
assetPeer::clearInstancePool();
conversionProfile2Peer::clearInstancePool();
flavorParamsConversionProfilePeer::clearInstancePool();
categoryPeer::clearInstancePool();
syndicationFeedPeer::clearInstancePool();
TrackEntryPeer::clearInstancePool();
// SystemUserPeer::clearInstancePool();
StorageProfilePeer::clearInstancePool();
// EmailIngestionProfilePeer::clearInstancePool();
UploadTokenPeer::clearInstancePool();
// invalidSessionPeer::clearInstancePool();
DynamicEnumPeer::clearInstancePool();
UserLoginDataPeer::clearInstancePool();
PermissionPeer::clearInstancePool();
UserRolePeer::clearInstancePool();
PermissionItemPeer::clearInstancePool();
PermissionToPermissionItemPeer::clearInstancePool();
KuserToUserRolePeer::clearInstancePool();
$pluginInstances = KalturaPluginManager::getPluginInstances('IKalturaMemoryCleaner');
foreach ($pluginInstances as $pluginInstance) {
$pluginInstance->cleanMemory();
}
if (function_exists('gc_collect_cycles')) {
// php 5.3 and above
gc_collect_cycles();
}
}
