public function clearMemory() { accessControlPeer::clearInstancePool(); BatchJobPeer::clearInstancePool(); BulkUploadResultPeer::clearInstancePool(); categoryPeer::clearInstancePool(); EmailIngestionProfilePeer::clearInstancePool(); entryPeer::clearInstancePool(); FileSyncPeer::clearInstancePool(); flavorAssetPeer::clearInstancePool(); flavorParamsConversionProfilePeer::clearInstancePool(); flavorParamsOutputPeer::clearInstancePool(); flavorParamsPeer::clearInstancePool(); kshowPeer::clearInstancePool(); mediaInfoPeer::clearInstancePool(); moderationFlagPeer::clearInstancePool(); moderationPeer::clearInstancePool(); notificationPeer::clearInstancePool(); roughcutEntryPeer::clearInstancePool(); SchedulerConfigPeer::clearInstancePool(); SchedulerPeer::clearInstancePool(); SchedulerStatusPeer::clearInstancePool(); SchedulerWorkerPeer::clearInstancePool(); StorageProfilePeer::clearInstancePool(); syndicationFeedPeer::clearInstancePool(); TrackEntryPeer::clearInstancePool(); uiConfPeer::clearInstancePool(); UploadTokenPeer::clearInstancePool(); // TODO clear default filters // TODO call all memory cleaner plugins if (function_exists('gc_collect_cycles')) { // php 5.3 and above gc_collect_cycles(); } }
/** * batch lockPendingFileSyncs action locks file syncs for import by the file sync periodic worker * * @action lockPendingFileSyncs * @param KalturaFileSyncFilter $filter * @param int $workerId The id of the file sync import worker * @param int $sourceDc The id of the DC from which the file syncs should be pulled * @param int $maxCount The maximum number of file syncs that should be returned * @param int $maxSize The maximum total size of file syncs that should be returned, this limit may be exceeded by one file sync * @return KalturaLockFileSyncsResponse */ function lockPendingFileSyncsAction(KalturaFileSyncFilter $filter, $workerId, $sourceDc, $maxCount, $maxSize = null) { // need to explicitly disable the cache since this action may not perform any queries kApiCache::disableConditionalCache(); // for dual dc deployments, if source dc is not specified, set it to the remote dc if ($sourceDc < 0) { $sourceDc = 1 - kDataCenterMgr::getCurrentDcId(); } // get caches $keysCache = kCacheManager::getSingleLayerCache(kCacheManager::CACHE_TYPE_QUERY_CACHE_KEYS); if (!$keysCache) { throw new KalturaAPIException(MultiCentersErrors::GET_KEYS_CACHE_FAILED); } $lockCache = kCacheManager::getSingleLayerCache(kCacheManager::CACHE_TYPE_LOCK_KEYS); if (!$lockCache) { throw new KalturaAPIException(MultiCentersErrors::GET_LOCK_CACHE_FAILED); } // get the max id / last id $maxId = $keysCache->get(self::MAX_FILESYNC_ID_PREFIX . $sourceDc); if (!$maxId) { throw new KalturaAPIException(MultiCentersErrors::GET_MAX_FILESYNC_ID_FAILED, $sourceDc); } $initialLastId = $keysCache->get(self::LAST_FILESYNC_ID_PREFIX . $workerId); KalturaLog::info("got lastId [{$initialLastId}] for worker [{$workerId}]"); $lastId = $initialLastId ? $initialLastId : $maxId; // created at less than handled explicitly $createdAtLessThanOrEqual = $filter->createdAtLessThanOrEqual; $filter->createdAtLessThanOrEqual = null; // build the criteria $fileSyncFilter = new FileSyncFilter(); $filter->toObject($fileSyncFilter); $baseCriteria = new Criteria(); $fileSyncFilter->attachToCriteria($baseCriteria); $baseCriteria->add(FileSyncPeer::STATUS, FileSync::FILE_SYNC_STATUS_PENDING); $baseCriteria->add(FileSyncPeer::FILE_TYPE, FileSync::FILE_SYNC_FILE_TYPE_FILE); $baseCriteria->add(FileSyncPeer::DC, kDataCenterMgr::getCurrentDcId()); $baseCriteria->addAscendingOrderByColumn(FileSyncPeer::ID); $baseCriteria->setLimit(self::MAX_FILESYNCS_PER_CHUNK); $lockedFileSyncs = array(); $lockedFileSyncsSize = 0; $limitReached = false; $selectCount = 0; $lastSelectId = 0; $done = false; while (!$done && $selectCount < self::MAX_FILESYNC_QUERIES_PER_CALL) { // make sure last id is always increasing if ($lastId <= $lastSelectId) { KalturaLog::info("last id was decremented {$lastId} <= {$lastSelectId}, stopping"); break; } $lastSelectId = $lastId; // clear the instance pool every once in a while (not clearing every time since // some objects repeat between selects) $selectCount++; if ($selectCount % 5 == 0) { FileSyncPeer::clearInstancePool(); } // get a chunk of file syncs // Note: starting slightly before the last id, because the ids may arrive out of order in the mysql replication $c = clone $baseCriteria; $idCriterion = $c->getNewCriterion(FileSyncPeer::ID, $lastId - 100, Criteria::GREATER_THAN); $idCriterion->addAnd($c->getNewCriterion(FileSyncPeer::ID, $maxId, Criteria::LESS_THAN)); $c->addAnd($idCriterion); // Note: disabling the criteria because it accumulates more and more criterions, and the status was already explicitly added // once that bug is fixed, this can be removed FileSyncPeer::setUseCriteriaFilter(false); $fileSyncs = FileSyncPeer::doSelect($c); FileSyncPeer::setUseCriteriaFilter(true); if (!$fileSyncs) { $lastId = $maxId; break; } // if we got less than the limit no reason to perform any more queries if (count($fileSyncs) < self::MAX_FILESYNCS_PER_CHUNK) { $done = true; } $lastFileSync = end($fileSyncs); $lastId = $lastFileSync->getId(); // filter by source dc foreach ($fileSyncs as $index => $fileSync) { if ($fileSync->getOriginalDc() != $sourceDc) { unset($fileSyncs[$index]); } } // filter by object type / sub type $fileSyncs = array_filter($fileSyncs, array('FileSyncImportBatchService', 'shouldSyncFileObjectType')); if (!$fileSyncs) { continue; } // filter by created at if ($createdAtLessThanOrEqual) { $firstFileSync = reset($fileSyncs); $prevLastId = $firstFileSync->getId(); foreach ($fileSyncs as $index => $fileSync) { if ($fileSync->getCreatedAt(null) > $createdAtLessThanOrEqual) { $done = true; unset($fileSyncs[$index]); if (!is_null($prevLastId)) { $lastId = $prevLastId; $prevLastId = null; } } else { $prevLastId = $fileSync->getId(); } } if (!$fileSyncs) { break; } } // get locked file syncs with multi get $lockKeys = array(); foreach ($fileSyncs as $fileSync) { $lockKeys[] = self::LOCK_KEY_PREFIX . $fileSync->getId(); } $lockKeys = $lockCache->get($lockKeys); // try to lock file syncs foreach ($fileSyncs as $fileSync) { $curKey = self::LOCK_KEY_PREFIX . $fileSync->getId(); if (isset($lockKeys[$curKey])) { KalturaLog::info('file sync ' . $fileSync->getId() . ' already locked'); continue; } if (!$lockCache->add($curKey, true, self::LOCK_EXPIRY)) { KalturaLog::info('failed to lock file sync ' . $fileSync->getId()); continue; } KalturaLog::info('locked file sync ' . $fileSync->getId()); // get the original id if not set if (!$fileSync->getOriginalId()) { $originalFileSync = self::getOriginalFileSync($fileSync); if (!$originalFileSync) { KalturaLog::info('failed to get original file sync for ' . $fileSync->getId()); continue; } $fileSync->setOriginalId($originalFileSync->getId()); $fileSync->setCustomDataObj(); // update $fileSync->custom_data so that originalId will be set by fromObject } // add to the result set $lockedFileSyncs[] = $fileSync; $lockedFileSyncsSize += $fileSync->getFileSize(); if (count($lockedFileSyncs) >= $maxCount || $maxSize && $lockedFileSyncsSize >= $maxSize) { $lastId = $fileSync->getId(); $limitReached = true; break; } } if ($limitReached) { break; } } // update the last id // Note: it is possible that the last id will go back in case of race condition, // but the only effect of this is that some file syncs will be scanned again if (!$initialLastId || $lastId > $initialLastId) { KalturaLog::info("setting lastId to [{$lastId}] for worker [{$workerId}]"); $keysCache->set(self::LAST_FILESYNC_ID_PREFIX . $workerId, $lastId); } // make sure all file syncs have a path foreach ($lockedFileSyncs as $fileSync) { if ($fileSync->getFileRoot() && $fileSync->getFilePath()) { continue; } $fileSyncKey = kFileSyncUtils::getKeyForFileSync($fileSync); list($fileRoot, $realPath) = kPathManager::getFilePathArr($fileSyncKey); $fileSync->setFileRoot($fileRoot); $fileSync->setFilePath($realPath); } // build the response object $sourceDc = kDataCenterMgr::getDcById($sourceDc); $result = new KalturaLockFileSyncsResponse(); $result->fileSyncs = KalturaFileSyncArray::fromDbArray($lockedFileSyncs, $this->getResponseProfile()); $result->limitReached = $limitReached; $result->dcSecret = $sourceDc["secret"]; $result->baseUrl = isset($sourceDc["fileSyncImportUrl"]) ? $sourceDc["fileSyncImportUrl"] : $sourceDc["url"]; return $result; }
/** * Method perform a DELETE on the database, given a FileSync or Criteria object OR a primary key value. * * @param mixed $values Criteria or FileSync object or primary key or array of primary keys * which is used to create the DELETE statement * @param PropelPDO $con the connection to use * @return int The number of affected rows (if supported by underlying database driver). This includes CASCADE-related rows * if supported by native driver or if emulated using Propel. * @throws PropelException Any exceptions caught during processing will be * rethrown wrapped into a PropelException. */ public static function doDelete($values, PropelPDO $con = null) { if ($con === null) { $con = Propel::getConnection(FileSyncPeer::DATABASE_NAME, Propel::CONNECTION_WRITE); } if ($values instanceof Criteria) { // invalidate the cache for all objects of this type, since we have no // way of knowing (without running a query) what objects should be invalidated // from the cache based on this Criteria. FileSyncPeer::clearInstancePool(); // rename for clarity $criteria = clone $values; } elseif ($values instanceof FileSync) { // it's a model object // invalidate the cache for this single object FileSyncPeer::removeInstanceFromPool($values); // create criteria based on pk values $criteria = $values->buildPkeyCriteria(); } else { // it's a primary key, or an array of pks $criteria = new Criteria(self::DATABASE_NAME); $criteria->add(FileSyncPeer::ID, (array) $values, Criteria::IN); // invalidate the cache for this object(s) foreach ((array) $values as $singleval) { FileSyncPeer::removeInstanceFromPool($singleval); } } // Set the correct dbName $criteria->setDbName(self::DATABASE_NAME); $affectedRows = 0; // initialize var to track total num of affected rows try { // use transaction because $criteria could contain info // for more than one table or we could emulating ON DELETE CASCADE, etc. $con->beginTransaction(); $affectedRows += BasePeer::doDelete($criteria, $con); FileSyncPeer::clearRelatedInstancePool(); $con->commit(); return $affectedRows; } catch (PropelException $e) { $con->rollBack(); throw $e; } }
/** * gets a source file of current DC, will make sure all links points to that source * are converted to files on all DCs * * @param FileSyncKey $key * @return void */ protected static function convertLinksToFiles(FileSync $fileSync) { $linkTotalCount = 0; /* @var $fileSync FileSync */ // for each source, find its links and fix them $c = new Criteria(); $c->add(FileSyncPeer::DC, $fileSync->getDc()); $c->add(FileSyncPeer::FILE_TYPE, array(FileSync::FILE_SYNC_FILE_TYPE_LINK, FileSync::FILE_SYNC_FILE_TYPE_URL), Criteria::IN); $c->add(FileSyncPeer::LINKED_ID, $fileSync->getId()); $c->addAscendingOrderByColumn(FileSyncPeer::PARTNER_ID); //relink the links into groups of 100 links $c->setLimit(100); $links = FileSyncPeer::doSelect($c); //check if any links were returned in the do select if not no need to continue if (!count($links)) { return; } // choose the first link and convert it to file $firstLink = array_shift($links); /* @var $firstLink FileSync */ if ($firstLink) { $firstLink->setStatus($fileSync->getStatus()); $firstLink->setFileSize($fileSync->getFileSize()); $firstLink->setFileRoot($fileSync->getFileRoot()); $firstLink->setFilePath($fileSync->getFilePath()); $firstLink->setFileType($fileSync->getFileType()); $firstLink->setLinkedId(0); // keep it zero instead of null, that's the only way to know it used to be a link. $firstLink->setIsDir($fileSync->getIsDir()); if (!is_null($fileSync->getOriginalDc())) { $firstLink->setOriginalDc($fileSync->getOriginalDc()); $firstLink->unsetOriginalId(); // recalculate the original id when importing the file sync } $firstLink->save(); } while (count($links)) { // change all the rest of the links to point on the new file sync foreach ($links as $link) { $linkTotalCount += count($links); /* @var $link FileSync */ $link->setStatus($fileSync->getStatus()); $link->setLinkedId($firstLink->getId()); $link->save(); } FileSyncPeer::clearInstancePool(); $links = FileSyncPeer::doSelect($c); } if ($firstLink) { $firstLink->setLinkCount($linkTotalCount); $firstLink->save(); } }
/** * gets a source file of current DC, will make sure all links points to that source * are converted to files on all DCs * * @param FileSyncKey $key * @return void */ public static function convertLinksToFiles(FileSyncKey $key) { // fetch sources from all DCs $c = new Criteria(); $c = FileSyncPeer::getCriteriaForFileSyncKey($key); $fileSyncList = FileSyncPeer::doSelect($c); foreach ($fileSyncList as $fileSync) { /* @var $fileSync FileSync */ // for each source, find its links and fix them $c = new Criteria(); $c->add(FileSyncPeer::DC, $fileSync->getDc()); $c->add(FileSyncPeer::FILE_TYPE, FileSync::FILE_SYNC_FILE_TYPE_LINK); $c->add(FileSyncPeer::LINKED_ID, $fileSync->getId()); $c->addAscendingOrderByColumn(FileSyncPeer::PARTNER_ID); //relink the links into groups of 100 links $c->setLimit(100); $links = FileSyncPeer::doSelect($c); while (count($links)) { // choose the first link and convert it to file $firstLink = array_shift($links); /* @var $firstLink FileSync */ if ($firstLink) { $firstLink->setStatus($fileSync->getStatus()); $firstLink->setFileSize($fileSync->getFileSize()); $firstLink->setFileRoot($fileSync->getFileRoot()); $firstLink->setFilePath($fileSync->getFilePath()); $firstLink->setWamsAssetId($fileSync->getWamsAssetId()); $firstLink->setWamsUrl($fileSync->getWamsUrl()); $firstLink->setFileType(FileSync::FILE_SYNC_FILE_TYPE_FILE); $firstLink->setLinkedId(0); // keep it zero instead of null, that's the only way to know it used to be a link. $firstLink->setLinkCount(count($links)); $firstLink->save(); } // change all the rest of the links to point on the new file sync foreach ($links as $link) { /* @var $link FileSync */ $link->setStatus($fileSync->getStatus()); $link->setLinkedId($firstLink->getId()); $link->save(); } FileSyncPeer::clearInstancePool(); $links = FileSyncPeer::doSelect($c); } } }
private function clearMemory() { entryPeer::clearInstancePool(); flavorAssetPeer::clearInstancePool(); FileSyncPeer::clearInstancePool(); categoryPeer::clearInstancePool(); if (class_exists('MetadataPeer')) { MetadataPeer::clearInstancePool(); MetadataProfilePeer::clearInstancePool(); } }
public static function clearMemory() { accessControlPeer::clearInstancePool(); kuserPeer::clearInstancePool(); kshowPeer::clearInstancePool(); entryPeer::clearInstancePool(); // kvotePeer::clearInstancePool(); // commentPeer::clearInstancePool(); // flagPeer::clearInstancePool(); // favoritePeer::clearInstancePool(); // KshowKuserPeer::clearInstancePool(); // MailJobPeer::clearInstancePool(); SchedulerPeer::clearInstancePool(); SchedulerWorkerPeer::clearInstancePool(); SchedulerStatusPeer::clearInstancePool(); SchedulerConfigPeer::clearInstancePool(); ControlPanelCommandPeer::clearInstancePool(); BatchJobPeer::clearInstancePool(); // PriorityGroupPeer::clearInstancePool(); BulkUploadResultPeer::clearInstancePool(); // blockedEmailPeer::clearInstancePool(); // conversionPeer::clearInstancePool(); // flickrTokenPeer::clearInstancePool(); PuserKuserPeer::clearInstancePool(); // PuserRolePeer::clearInstancePool(); PartnerPeer::clearInstancePool(); // WidgetLogPeer::clearInstancePool(); // adminKuserPeer::clearInstancePool(); // notificationPeer::clearInstancePool(); moderationPeer::clearInstancePool(); moderationFlagPeer::clearInstancePool(); roughcutEntryPeer::clearInstancePool(); // widgetPeer::clearInstancePool(); uiConfPeer::clearInstancePool(); // PartnerStatsPeer::clearInstancePool(); // PartnerActivityPeer::clearInstancePool(); ConversionProfilePeer::clearInstancePool(); // ConversionParamsPeer::clearInstancePool(); // KceInstallationErrorPeer::clearInstancePool(); FileSyncPeer::clearInstancePool(); accessControlPeer::clearInstancePool(); mediaInfoPeer::clearInstancePool(); assetParamsPeer::clearInstancePool(); assetParamsOutputPeer::clearInstancePool(); assetPeer::clearInstancePool(); conversionProfile2Peer::clearInstancePool(); flavorParamsConversionProfilePeer::clearInstancePool(); categoryPeer::clearInstancePool(); syndicationFeedPeer::clearInstancePool(); TrackEntryPeer::clearInstancePool(); // SystemUserPeer::clearInstancePool(); StorageProfilePeer::clearInstancePool(); // EmailIngestionProfilePeer::clearInstancePool(); UploadTokenPeer::clearInstancePool(); // invalidSessionPeer::clearInstancePool(); DynamicEnumPeer::clearInstancePool(); UserLoginDataPeer::clearInstancePool(); PermissionPeer::clearInstancePool(); UserRolePeer::clearInstancePool(); PermissionItemPeer::clearInstancePool(); PermissionToPermissionItemPeer::clearInstancePool(); KuserToUserRolePeer::clearInstancePool(); $pluginInstances = KalturaPluginManager::getPluginInstances('IKalturaMemoryCleaner'); foreach ($pluginInstances as $pluginInstance) { $pluginInstance->cleanMemory(); } if (function_exists('gc_collect_cycles')) { // php 5.3 and above gc_collect_cycles(); } }