public function writeSolrLog($entry) { KalturaLog::debug("writeSolrLog " . $entry->getId()); $solrLog = new SphinxLog(); $solrLog->setEntryId($entry->getId()); $solrLog->setPartnerId($entry->getPartnerId()); $solrLog->save(myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_SOLR_LOG)); }
public static function alternativeCon($con, $queryDB = kQueryCache::QUERY_DB_UNDEFINED) { if ($con === null) { $con = myDbHelper::alternativeCon($con); } if ($con === null) { $con = myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL3); } return $con; }
public static function alternativeCon($con) { if ($con === null) { $con = myDbHelper::alternativeCon($con); } if ($con === null) { $con = myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL3); } return $con; }
public function getConfigs() { $c = new Criteria(); $c->clearSelectColumns(); $c->addSelectColumn('MAX(' . SchedulerConfigPeer::ID . ')'); $c->addGroupByColumn(SchedulerConfigPeer::VARIABLE); $c->addGroupByColumn(SchedulerConfigPeer::VARIABLE_PART); $c->addAscendingOrderByColumn(SchedulerConfigPeer::VARIABLE); $c->add(SchedulerConfigPeer::SCHEDULER_ID, $this->id); $c->add(SchedulerConfigPeer::WORKER_ID, null); $rs = SchedulerConfigPeer::doSelectStmt($c, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2)); $configIds = $rs->fetchAll(PDO::FETCH_COLUMN, 0); return SchedulerConfigPeer::retrieveByPKs($configIds, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2)); }
public static function doQueueStatus(PDO $con = null) { if (is_null($con)) { $con = myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2); } // $con = Propel::getConnection(BatchJobPeer::DATABASE_NAME); $stmt = null; try { $now = time(); $sql = "SELECT\t\tAVG(TIMEDIFF(NOW(), CREATED_AT)) AS CREATED_AT_AVG,\n\t\t\t\t\t\t\t\tCOUNT(JOB_TYPE) AS JOB_TYPE_COUNT, \n\t\t\t\t\t\t\t\tJOB_TYPE \n\t\t\t\t\tFROM\t\tbatch_job \n\t\t\t\t\tFORCE INDEX (created_at_job_type_status_index)\n\t\t\t\t\tWHERE\t\tSTATUS IN (" . BatchJob::BATCHJOB_STATUS_PENDING . ',' . BatchJob::BATCHJOB_STATUS_RETRY . ")\n\t\t\t\t\tAND\t\t\t(\n\t\t\t\t\t\t\t\t\tCHECK_AGAIN_TIMEOUT < {$now}\n\t\t\t\t\t\t\t\t\tOR\n\t\t\t\t\t\t\t\t\tCHECK_AGAIN_TIMEOUT IS NULL\n\t\t\t\t\t\t\t\t)\n\t\t\t\t\tAND\t\t\tJOB_TYPE IS NOT NULL\n\t\t\t\t\tAND\t\t\tCREATED_AT > DATE_ADD(NOW(), INTERVAL -1 HOUR)\n\t\t\t\t\tGROUP BY\tJOB_TYPE"; $stmt = $con->query($sql); } catch (Exception $e) { Propel::log($e->getMessage(), Propel::LOG_ERR); throw new PropelException($e); } return $stmt->fetchAll(); }
<?php set_time_limit(0); ini_set("memory_limit", "700M"); error_reporting(E_ALL); chdir(dirname(__FILE__)); define('ROOT_DIR', realpath(dirname(__FILE__) . '/../../')); require_once ROOT_DIR . '/infra/bootstrap_base.php'; require_once ROOT_DIR . '/infra/KAutoloader.php'; KAutoloader::addClassPath(KAutoloader::buildPath(KALTURA_ROOT_PATH, "vendor", "propel", "*")); KAutoloader::setClassMapFilePath(kConf::get("cache_root_path") . '/scripts/' . basename(__FILE__) . '.cache'); KAutoloader::register(); KalturaLog::setLogger(new KalturaStdoutLogger()); $dbConf = kConf::getDB(); DbManager::setConfig($dbConf); DbManager::initialize(); $createdAt = time() - 60 * 60 * 24; $jobType = 0; $c = new Criteria(); // $c->add(BatchJobPeer::CREATED_AT, $createdAt, Criteria::GREATER_THAN); $c->add(BatchJobPeer::JOB_TYPE, $jobType); // $c->add(BatchJobPeer::STATUS, BatchJob::BATCHJOB_STATUS_PENDING); $c->clearSelectColumns(); $c->addSelectColumn('MAX(' . BatchJobPeer::PRIORITY . ')'); $stmt = BatchJobPeer::doSelectStmt($c, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2)); $maxPriority = $stmt->fetchColumn(); var_dump($maxPriority);
require_once SF_ROOT_DIR . DIRECTORY_SEPARATOR . 'apps' . DIRECTORY_SEPARATOR . SF_APP . DIRECTORY_SEPARATOR . 'config' . DIRECTORY_SEPARATOR . 'config.php'; KAutoloader::addClassPath(KAutoloader::buildPath(KALTURA_ROOT_PATH, "vendor", "propel", "*")); KAutoloader::setClassMapFilePath(kConf::get("cache_root_path") . '/scripts/' . basename(__FILE__) . '.cache'); KAutoloader::register(); date_default_timezone_set(kConf::get("date_default_timezone")); $loggerConfigPath = ROOT_DIR . '/scripts/logger.ini'; $config = new Zend_Config_Ini($loggerConfigPath); KalturaLog::initLog($config); KalturaLog::setContext(basename(__FILE__)); KalturaLog::info("Starting script"); KalturaLog::info("Initializing database..."); DbManager::setConfig(kConf::getDB()); DbManager::initialize(); KalturaLog::info("Database initialized successfully"); $syncType = 'kuser'; $dbh = myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_DWH); $sql = "CALL get_data_for_operational('{$syncType}')"; $count = 0; $rows = $dbh->query($sql)->fetchAll(); foreach ($rows as $row) { $kuser = kuserPeer::retrieveByPK($row['kuser_id']); if (is_null($kuser)) { KalturaLog::err('Couldn\'t find kuser [' . $row['kuser_id'] . ']'); continue; } $kuser->setStorageSize($row['storage_size']); $kuser->save(); $count++; KalturaLog::debug('Successfully saved kuser [' . $row['kuser_id'] . ']'); if ($count % 500) { kuserPeer::clearInstancePool();
$lastFileSyncId = null; $loopLimit = 500; while ($moreFileSyncs) { // ---------------------------- // get next remote file syncs // ---------------------------- $exportedFileSyncsCriteria = new Criteria(); $exportedFileSyncsCriteria->setLimit($loopLimit); $exportedFileSyncsCriteria->addAscendingOrderByColumn(FileSyncPeer::ID); $exportedFileSyncsCriteria->addAnd(FileSyncPeer::DC, $storageProfileId, Criteria::EQUAL); $exportedFileSyncsCriteria->addAnd(FileSyncPeer::STATUS, FileSync::FILE_SYNC_STATUS_READY, Criteria::EQUAL); $exportedFileSyncsCriteria->addAnd(FileSyncPeer::PARTNER_ID, $partnerId, Criteria::EQUAL); if ($lastFileSyncId) { $exportedFileSyncsCriteria->addAnd(FileSyncPeer::ID, $lastFileSyncId, Criteria::GREATER_THAN); } $exportedFileSyncs = FileSyncPeer::doSelect($exportedFileSyncsCriteria, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL3)); // ----------------------------------------------- // delete the local dcs copies of each file sync // ----------------------------------------------- foreach ($exportedFileSyncs as $exportedFileSync) { $lastFileSyncId = $exportedFileSync->getId(); $syncKey = kFileSyncUtils::getKeyForFileSync($exportedFileSync); echo 'Deleting file sync key - ' . serialize($syncKey) . PHP_EOL; kFileSyncUtils::deleteSyncFileForKey($syncKey, false, true); // 3rd param = true -> only delete from local dcs } // -------------------------- // check if loop should end // -------------------------- if (count($exportedFileSyncs) < $loopLimit) { $moreFileSyncs = false;
if (is_null($entry)) { KalturaLog::err('Couldn\'t find entry [' . $entryId . ']'); continue; } if ($entry->getViews() != $views || $entry->getPlays() != $plays) { $entry->setViews($views); $entry->setPlays($plays); KalturaLog::debug('Successfully saved entry [' . $entryId . ']'); try { // update entry without setting the updated at $updateSql = "UPDATE entry set views='{$views}',plays='{$plays}' WHERE id='{$entryId}'"; $stmt = $connection->prepare($updateSql); $stmt->execute(); $affectedRows = $stmt->rowCount(); KalturaLog::log("AffectedRows: " . $affectedRows); // update sphinx log directly $sql = $sphinxMgr->getSphinxSaveSql($entry, false); $sphinxLog = new SphinxLog(); $sphinxLog->setEntryId($entryId); $sphinxLog->setPartnerId($entry->getPartnerId()); $sphinxLog->setSql($sql); $sphinxLog->save(myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_SPHINX_LOG)); } catch (Exception $e) { KalturaLog::log($e->getMessage(), Propel::LOG_ERR); } } $count++; if ($count % 500 === 0) { entryPeer::clearInstancePool(); } }
public static function alternativeCon($con, $queryDB = kQueryCache::QUERY_DB_UNDEFINED) { return myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_SPHINX_LOG); }
public static function getExpiredJobs() { $jobTypes = kPluginableEnumsManager::coreValues('BatchJobType'); $executionAttempts2jobTypes = array(); // Map between max execution attempts and job types foreach ($jobTypes as $jobType) { $executionAttempts = BatchJobLockPeer::getMaxExecutionAttempts($jobType); if (array_key_exists($executionAttempts, $executionAttempts2jobTypes)) { $executionAttempts2jobTypes[$executionAttempts][] = $jobType; } else { $executionAttempts2jobTypes[$executionAttempts] = array($jobType); } } // create query $c = new Criteria(); $c->add(BatchJobLockPeer::STATUS, BatchJob::BATCHJOB_STATUS_FATAL, Criteria::NOT_EQUAL); $c->add(BatchJobLockPeer::DC, kDataCenterMgr::getCurrentDcId()); // each DC should clean its own jobs // Query for each job type $batchJobLocks = array(); foreach ($executionAttempts2jobTypes as $execAttempts => $jobTypes) { $typedCrit = clone $c; $typedCrit->add(BatchJobLockPeer::EXECUTION_ATTEMPTS, $execAttempts, Criteria::GREATER_THAN); $typedCrit->add(BatchJobLockPeer::JOB_TYPE, $jobTypes, Criteria::IN); $typedJobs = BatchJobLockPeer::doSelect($typedCrit, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2)); foreach ($typedJobs as $typedJob) { $batchJobLocks[$typedJob->getId()] = $typedJob; } } // get matching batch jobs return BatchJobPeer::retrieveByPKs(array_keys($batchJobLocks)); }
} else { $c->add(BatchJobLockPeer::PARTNER_ID, $partnerId); } } $c->setLimit(CHUNK_SIZE); $movedJobsCount = 0; while ($movedJobsCount < $maxMovedJobs) { $jobLocks = BatchJobLockPeer::doSelect($c, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2)); if (!$jobLocks) { break; } $initialMovedJobsCount = $movedJobsCount; foreach ($jobLocks as $jobLock) { /* @var $jobLock BatchJobLock */ /* @var $job BatchJob */ $job = $jobLock->getBatchJob(myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2)); // check whether the job can be moved $jobData = $job->getData(); /* @var $jobData kConvartableJobData */ $srcFileSyncs = $jobData->getSrcFileSyncs(); if (count($srcFileSyncs) != 1) { continue; } // unexpected - multiple sources for doc convert $srcFileSync = reset($srcFileSyncs); /* @var $srcFileSync kSourceFileSyncDescriptor */ $sourceAsset = assetPeer::retrieveById($srcFileSync->getAssetId()); if (!$sourceAsset) { continue; } // unexpected - source flavor asset not found
public static function alternativeCon($con) { return myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_SPHINX_LOG); }
public static function getExpiredJobs() { $jobTypes = kPluginableEnumsManager::coreValues('BatchJobType'); $c = new Criteria(); $c->add(BatchJobPeer::STATUS, BatchJob::BATCHJOB_STATUS_FATAL, Criteria::NOT_EQUAL); $c->add(BatchJobPeer::DC, kDataCenterMgr::getCurrentDcId()); // each DC should clean its own jobs // $c->add(BatchJobPeer::PROCESSOR_EXPIRATION, time(), Criteria::LESS_THAN); // $c->add(BatchJobPeer::SCHEDULER_ID, 0, Criteria::GREATER_THAN); // $c->add(BatchJobPeer::WORKER_ID, 0, Criteria::GREATER_THAN); $jobs = array(); foreach ($jobTypes as $jobType) { $typedCrit = clone $c; $typedCrit->add(BatchJobPeer::EXECUTION_ATTEMPTS, BatchJobPeer::getMaxExecutionAttempts($jobType), Criteria::GREATER_THAN); $typedCrit->add(BatchJobPeer::JOB_TYPE, $jobType); $typedJobs = BatchJobPeer::doSelect($typedCrit, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2)); foreach ($typedJobs as $typedJob) { $jobs[] = $typedJob; } } return $jobs; }
/** * Update Conversion Profile by ID * * @action update * @param int $id * @param KalturaConversionProfile $conversionProfile * @return KalturaConversionProfile */ public function updateAction($id, KalturaConversionProfile $conversionProfile) { if ($conversionProfile->name !== null) { $conversionProfile->validatePropertyMinLength("name", 1); } if ($conversionProfile->flavorParamsIds !== null) { $conversionProfile->validateFlavorParamsIds(); } $conversionProfileDb = conversionProfile2Peer::retrieveByPK($id); if (!$conversionProfileDb) { throw new KalturaAPIException(KalturaErrors::CONVERSION_PROFILE_ID_NOT_FOUND, $id); } $conversionProfile->toUpdatableObject($conversionProfileDb); $conversionProfileDb->setCreationMode(conversionProfile2::CONVERSION_PROFILE_2_CREATION_MODE_KMC); $conversionProfileDb->save(); if ($conversionProfile->flavorParamsIds !== null) { $this->deleteFlavorParamsRelation($conversionProfileDb); $this->addFlavorParamsRelation($conversionProfileDb, $conversionProfile->getFlavorParamsAsArray()); } $conversionProfile->fromObject($conversionProfileDb); // load flavor params id with the same connection (master connection) that was used for insert $con = myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_MASTER); $conversionProfile->loadFlavorParamsIds($conversionProfileDb, $con); return $conversionProfile; }
/** * @param SchedulerWorker $dbData * @return KalturaScheduler */ public function statusFromObject($dbData) { $this->fromObject($dbData); $this->lockedJobs = KalturaBatchJobArray::fromBatchJobArray($dbData->getLockedJobs()); $this->avgWait = BatchJobPeer::doAvgTimeDiff($this->type, BatchJobPeer::CREATED_AT, BatchJobPeer::QUEUE_TIME, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2)); $this->avgWork = BatchJobPeer::doAvgTimeDiff($this->type, BatchJobPeer::QUEUE_TIME, BatchJobPeer::FINISH_TIME, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2)); return $this; }
/** * batch freeExclusiveJobAction action allows to get a generic BatchJob * * @action freeExclusiveJob * @param int $id The id of the job * @param KalturaExclusiveLockKey $lockKey The unique lock key from the batch-process. Is used for the locking mechanism * @param KalturaBatchJobType $jobType The type of the job * @param bool $resetExecutionAttempts Resets the job execution attampts to zero * @return KalturaFreeJobResponse */ function freeExclusiveJobAction($id, KalturaExclusiveLockKey $lockKey, $jobType, $resetExecutionAttempts = false) { $jobType = kPluginableEnumsManager::apiToCore('BatchJobType', $jobType); $job = BatchJobPeer::retrieveByPK($id); // verifies that the job is of the right type if ($job->getJobType() != $jobType) { throw new KalturaAPIException(APIErrors::UPDATE_EXCLUSIVE_JOB_WRONG_TYPE, $id, $lockKey, null); } $job = kBatchManager::freeExclusiveBatchJob($id, $lockKey->toObject(), $resetExecutionAttempts); $batchJob = new KalturaBatchJob(); // start from blank $batchJob->fromObject($job); // gets queues length $c = new Criteria(); $c->add(BatchJobPeer::STATUS, array(KalturaBatchJobStatus::PENDING, KalturaBatchJobStatus::RETRY, KalturaBatchJobStatus::ALMOST_DONE), Criteria::IN); $c->add(BatchJobPeer::JOB_TYPE, $jobType); $queueSize = BatchJobPeer::doCount($c, false, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2)); if (!$queueSize) { // gets queues length $c = new Criteria(); $c->add(BatchJobPeer::BATCH_INDEX, null, Criteria::ISNOTNULL); $c->add(BatchJobPeer::PROCESSOR_EXPIRATION, time(), Criteria::GREATER_THAN); $c->add(BatchJobPeer::EXECUTION_ATTEMPTS, BatchJobPeer::getMaxExecutionAttempts($jobType), Criteria::LESS_THAN); $c->add(BatchJobPeer::JOB_TYPE, $jobType); $queueSize = BatchJobPeer::doCount($c, false, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2)); } $response = new KalturaFreeJobResponse(); $response->job = $batchJob; $response->jobType = $jobType; $response->queueSize = $queueSize; return $response; }
DbManager::setConfig($dbConf); DbManager::initialize(); if (count($argv) !== 2) { die('pleas provide partner id as input' . PHP_EOL . 'to run script: ' . basename(__FILE__) . ' X' . PHP_EOL . 'whereas X is partner id' . PHP_EOL); } $partner_id = @$argv[1]; $partner = PartnerPeer::retrieveByPK($partner_id); if (!$partner) { die('no such partner.' . PHP_EOL); } $partner->setAppearInSearch(mySearchUtils::DISPLAY_IN_SEARCH_KALTURA_NETWORK); $partner->save(); $c = new Criteria(); $c->add(entryPeer::PARTNER_ID, $partner_id); $c->add(entryPeer::DISPLAY_IN_SEARCH, mySearchUtils::DISPLAY_IN_SEARCH_PARTNER_ONLY); $c->addOr(entryPeer::DISPLAY_IN_SEARCH, mySearchUtils::DISPLAY_IN_SEARCH_NONE); $c->setLimit(200); $con = myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2); $entries = entryPeer::doSelect($c, $con); $changedEntriesCounter = 0; while (count($entries)) { $changedEntriesCounter += count($entries); foreach ($entries as $entry) { echo "changed DISPLAY_IN_SEARCH for entry: " . $entry->getId() . "\n"; $entry->setDisplayInSearch(mySearchUtils::DISPLAY_IN_SEARCH_KALTURA_NETWORK); $entry->save(); } entryPeer::clearInstancePool(); $entries = entryPeer::doSelect($c, $con); } echo "Done. {$changedEntriesCounter} entries where changed";
public function isValid($partner_id, $puser_id, $type = false) { if (!$this->valid_string) { return self::INVALID_STR; } if (!$this->matchPartner($partner_id)) { return self::INVALID_PARTNER; } if (!$this->matchUser($puser_id)) { return self::INVALID_USER; } if ($type !== false) { // do not check ks type if (!$this->type == $type) { return self::INVALID_TYPE; } } if ($this->expired()) { return self::EXPIRED; } if ($this->original_str) { $invalid = invalidSessionPeer::isInvalid($this->original_str, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2)); if ($invalid) { return self::LOGOUT; } } return self::OK; }
/** * @param string $sql * @param IIndexable $object * @return bool */ public function execSphinx($sql, IIndexable $object) { KalturaLog::debug($sql); $sphinxLog = new SphinxLog(); $sphinxLog->setEntryId($object->getEntryId()); $sphinxLog->setPartnerId($object->getPartnerId()); $sphinxLog->setSql($sql); $sphinxLog->save(myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_SPHINX_LOG)); if (!kConf::hasParam('exec_sphinx') || !kConf::get('exec_sphinx')) { return true; } $con = DbManager::getSphinxConnection(); $ret = $con->exec($sql); if ($ret) { return true; } $arr = $con->errorInfo(); KalturaLog::err($arr[2]); return false; }
/** * batch getFullStatus action returns the status of all schedulers and queues * * @action getFullStatus * @return KalturaFullStatusResponse */ function getFullStatusAction() { $response = new KalturaFullStatusResponse(); // gets queues length // $c = new Criteria(); // $c->add(BatchJobPeer::STATUS, array(KalturaBatchJobStatus::PENDING, KalturaBatchJobStatus::RETRY), Criteria::IN); // $c->addGroupByColumn(BatchJobPeer::JOB_TYPE); // $c->addSelectColumn('AVG(DATEDIFF(NOW(),' . BatchJobPeer::CREATED_AT . '))'); $queueList = BatchJobPeer::doQueueStatus(myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2)); $response->queuesStatus = KalturaBatchQueuesStatusArray::fromBatchQueuesStatusArray($queueList); $response->schedulers = KalturaSchedulerArray::statusFromSchedulerArray(SchedulerPeer::doSelect(new Criteria(), myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2))); return $response; }
/** * @param string $sql * @param IIndexable $object * @return bool */ public function execSphinx($sql, IIndexable $object) { // limit the number of large sphinx SQLs to 1/min per object, since they load the sphinx database // and sphinx servers. the upper limit of 'slightly less than 1MB' is because of the max_allowed_packet // limit in mysql (by default 1MB). the upper limit is here to prevent the addition of too many category // users (for example), that will render the category un-indexable if (strlen($sql) > 128 * 1024 && strlen($sql) < 1000000) { $lockKey = 'large_sql_lock_' . get_class($object) . '_' . $object->getId(); $cache = kCacheManager::getSingleLayerCache(kCacheManager::CACHE_TYPE_SPHINX_STICKY_SESSIONS); if ($cache && !$cache->add($lockKey, true, 60)) { KalturaLog::log('skipping sql for key ' . $lockKey); return; } } KalturaLog::debug($sql); $sphinxLog = new SphinxLog(); $sphinxLog->setExecutedServerId($this->retrieveSphinxConnectionId()); $sphinxLog->setObjectId($object->getId()); $sphinxLog->setObjectType(get_class($object)); $sphinxLog->setEntryId($object->getEntryId()); $sphinxLog->setPartnerId($object->getPartnerId()); $sphinxLog->setSql($sql); $sphinxLog->save(myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_SPHINX_LOG)); if (!kConf::get('exec_sphinx', 'local', 0)) { return true; } $sphinxConnection = DbManager::getSphinxConnection(false); $ret = $sphinxConnection->exec($sql); if ($ret) { return true; } $arr = $sphinxConnection->errorInfo(); KalturaLog::err($arr[2]); return false; }
$c->addAnd(BatchJobPeer::STATUS, $nonFinalBatchStatuses, Criteria::IN); $batchCount = BatchJobPeer::doCount($c, false, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL3)); if ($batchCount >= $maxConcurrentJobs) { sleep(30); continue; } $curLimit = $maxConcurrentJobs - $batchCount; $currentExported = 0; $c = new Criteria(); $c->add(entryPeer::PARTNER_ID, $partnerId); if ($lastCreatedAt) { $c->addAnd(entryPeer::CREATED_AT, $lastCreatedAt, Criteria::LESS_EQUAL); } $c->addDescendingOrderByColumn(entryPeer::CREATED_AT); $c->setLimit($curLimit); $entries = entryPeer::doSelect($c, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL3)); foreach ($entries as $entry) { if (in_array($entry->getId(), $processedIds)) { continue; } $processedIds[] = $entry->getId(); $lastCreatedAt = $entry->getCreatedAt(null); $keys = array(); $keys[] = $entry->getSyncKey(entry::FILE_SYNC_ENTRY_SUB_TYPE_ISM); $keys[] = $entry->getSyncKey(entry::FILE_SYNC_ENTRY_SUB_TYPE_ISMC); $flavors = assetPeer::retrieveReadyFlavorsByEntryId($entry->getId()); foreach ($flavors as $flavor) { if (!$flavorParamsArr || in_array($flavor->getFlavorParamsId(), $flavorParamsArr)) { $keys[] = $flavor->getSyncKey(flavorAsset::FILE_SYNC_FLAVOR_ASSET_SUB_TYPE_ASSET); } }
/** * @param string $sql * @param IIndexable $object * @return bool */ public function execSphinx($sql, IIndexable $object) { KalturaLog::debug($sql); $sphinxConnection = null; $sphinxConnectionId = null; if (kConf::hasParam('exec_sphinx') && kConf::get('exec_sphinx')) { $sphinxConnection = DbManager::getSphinxConnection(false); $sphinxServer = SphinxLogServerPeer::retrieveByLocalServer($sphinxConnection->getHostName()); if ($sphinxServer) { $sphinxConnectionId = $sphinxServer->getId(); } } $sphinxLog = new SphinxLog(); $sphinxLog->setExecutedServerId($sphinxConnectionId); $sphinxLog->setObjectId($object->getId()); $sphinxLog->setObjectType(get_class($object)); $sphinxLog->setEntryId($object->getEntryId()); $sphinxLog->setPartnerId($object->getPartnerId()); $sphinxLog->setSql($sql); $sphinxLog->save(myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_SPHINX_LOG)); if (!$sphinxConnection) { return true; } $sphinxConnection = DbManager::getSphinxConnection(false); $ret = $sphinxConnection->exec($sql); if ($ret) { return true; } $arr = $sphinxConnection->errorInfo(); KalturaLog::err($arr[2]); return false; }
public function getDirectChildJobs() { $c = new Criteria(); $c->add(BatchJobPeer::PARENT_JOB_ID, $this->id); return BatchJobPeer::doSelect($c, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2)); }
/** * Update Conversion Profile by ID * * @action update * @param int $id * @param KalturaConversionProfile $conversionProfile * @return KalturaConversionProfile */ public function updateAction($id, KalturaConversionProfile $conversionProfile) { $conversionProfileDb = conversionProfile2Peer::retrieveByPK($id); if (!$conversionProfileDb) { throw new KalturaAPIException(KalturaErrors::CONVERSION_PROFILE_ID_NOT_FOUND, $id); } $conversionProfile->toUpdatableObject($conversionProfileDb); $conversionProfileDb->setCreationMode(conversionProfile2::CONVERSION_PROFILE_2_CREATION_MODE_KMC); if ($conversionProfile->xslTransformation) { $conversionProfileDb->incrementXslVersion(); } $conversionProfileDb->save(); if ($conversionProfile->flavorParamsIds !== null) { $this->deleteFlavorParamsRelation($conversionProfileDb, $conversionProfile->flavorParamsIds); $this->addFlavorParamsRelation($conversionProfileDb, $conversionProfile->getFlavorParamsAsArray()); } if ($conversionProfile->xslTransformation) { $xsl = html_entity_decode($conversionProfile->xslTransformation); $key = $conversionProfileDb->getSyncKey(conversionProfile2::FILE_SYNC_MRSS_XSL); kFileSyncUtils::file_put_contents($key, $xsl); } $conversionProfile->fromObject($conversionProfileDb); // load flavor params id with the same connection (master connection) that was used for insert $con = myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_MASTER); $conversionProfile->loadFlavorParamsIds($conversionProfileDb, $con); return $conversionProfile; }
if (isset($lastLogs[$dc])) { $serverLastLog = $lastLogs[$dc]; if ($serverLastLog->getLastLogId() >= $sphinxLog->getId()) { KalturaLog::debug('Last log id [' . $serverLastLog->getLastLogId() . "] dc [{$dc}] is larger than id [" . $sphinxLog->getId() . "]"); continue; } } else { $serverLastLog = new SphinxLogServer(); $serverLastLog->setServer($sphinxServer); $serverLastLog->setDc($dc); $lastLogs[$dc] = $serverLastLog; } try { $sql = $sphinxLog->getSql(); $affected = $sphinxCon->exec($sql); if (!$affected) { $errorInfo = $sphinxCon->errorInfo(); // if(!preg_match('/^duplicate id/', $errorInfo[2])) // die("No affected records [" . $sphinxCon->errorCode() . "]\n" . print_r($sphinxCon->errorInfo(), true)); } $serverLastLog->setLastLogId($sphinxLog->getId()); $serverLastLog->save(myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_SPHINX_LOG)); } catch (Exception $e) { KalturaLog::err($e->getMessage()); } } unset($sphinxCon); SphinxLogPeer::clearInstancePool(); $sphinxLogs = SphinxLogPeer::retrieveByLastId($lastLogs); } KalturaLog::log('Done');
private function compareOldToNewQueries($criteria, $sphinxOldQuery, $mysqlOldQuery) { $pdo = DbManager::getSphinxConnection(); $successfulTest = true; $OldSphinxQueryIds = array(); $OldQueryIds = array('just somthing to put so this is not empty'); $criteria->applyFilters(); $sphinxNewQueryids = $criteria->getFetchedIds(); try { var_dump($sphinxOldQuery); $stmtSphinxOldQuery = $pdo->query($sphinxOldQuery); if (!$stmtSphinxOldQuery) { echo "Invalid sphinx query [" . $sphinxOldQuery; } $OldSphinxQueryIds = $stmtSphinxOldQuery->fetchAll(PDO::FETCH_COLUMN, 2); } catch (Exception $x) { echo 'error in old sphinx select: ' . $x->getMessage() . PHP_EOL; echo 'old sphinx query: ' . $sphinxOldQuery . PHP_EOL; $successfulTest = false; } if ($mysqlOldQuery != null) { try { $con = myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2); /* var PDOStatement $stmt*/ $stmt = $con->query($mysqlOldQuery); $OldQueryIds = $stmt->fetchAll(PDO::FETCH_COLUMN); $stmt->execute(); } catch (Exception $x) { echo 'error in mysql old select: ' . $x->getMessage() . PHP_EOL; echo 'old mysqk query: ' . ${$mysqlOldQuery} . PHP_EOL; $successfulTest = false; } } $OldQueryIds = array_slice($OldQueryIds, 0, 500); if (!$successfulTest || is_null($mysqlOldQuery) || count(array_diff($OldQueryIds, $sphinxNewQueryids)) || count(array_diff($sphinxNewQueryids, $OldQueryIds))) { if ($successfulTest && !(count(array_diff($OldSphinxQueryIds, $sphinxNewQueryids)) || count(array_diff($sphinxNewQueryids, $OldSphinxQueryIds)))) { echo 'Test OK : ' . PHP_EOL; echo 'Test Faild - but was the same in sphinx: ' . print_r($OldSphinxQueryIds, true) . print_r($sphinxNewQueryids, true) . PHP_EOL; $successfulTest = true; } else { echo 'Test Faild: ' . PHP_EOL; if ($successfulTest && ($mysqlOldQuery != null && strstr($mysqlOldQuery, 'COUNT(*)'))) { echo 'Test ignored: ' . PHP_EOL; $successfulTest = true; } else { $successfulTest = false; } } } if ($successfulTest) { echo 'Successful Test' . PHP_EOL; } else { echo 'Fail Test' . PHP_EOL; } return $successfulTest; }
public static function alternativeCon($con, $queryDB = kQueryCache::QUERY_DB_UNDEFINED) { if ($con === null) { switch ($queryDB) { case kQueryCache::QUERY_DB_MASTER: $con = myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_MASTER); break; case kQueryCache::QUERY_DB_SLAVE: $con = myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2); break; } } if ($con === null) { $con = myDbHelper::alternativeCon($con); } if ($con === null) { $con = Propel::getConnection(MetadataProfilePeer::DATABASE_NAME, Propel::CONNECTION_READ); } return $con; }
public static function getNextJobPriority($jobType) { //$priorities = array(1 => 33, 2 => 27, 3 => 20, 4 => 13, 5 => 7); $priorities = kConf::get('priority_percent'); $createdAt = time() - kConf::get('priority_time_range'); // $createdAt = kConf::get('priority_time_range'); $c = new Criteria(); $c->add(BatchJobPeer::CREATED_AT, $createdAt, Criteria::GREATER_THAN); $c->add(BatchJobPeer::JOB_TYPE, $jobType); $c->add(BatchJobPeer::STATUS, BatchJob::BATCHJOB_STATUS_PENDING); $c->clearSelectColumns(); $c->addSelectColumn('MAX(' . BatchJobPeer::PRIORITY . ')'); $stmt = BatchJobPeer::doSelectStmt($c, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2)); $maxPriority = $stmt->fetchColumn(); // gets the current queues $c = new Criteria(); $c->add(BatchJobPeer::CREATED_AT, $createdAt, Criteria::GREATER_THAN); $c->add(BatchJobPeer::JOB_TYPE, $jobType); $c->add(BatchJobPeer::STATUS, BatchJob::BATCHJOB_STATUS_PENDING, Criteria::GREATER_THAN); $c->addGroupByColumn(BatchJobPeer::PRIORITY); // To prevent stress on the master DB - use the slave for checking the queue sizes $queues = BatchJobPeer::doCountGroupBy($c, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2)); // copy the queues and calcs the total $total = 0; $queues_size = array(); foreach ($queues as $queue) { $queues_size[$queue['PRIORITY']] = $queue[BatchJobPeer::COUNT]; $total += $queue[BatchJobPeer::COUNT]; } // go over the priorities and see if its percent not used foreach ($priorities as $priority => $top_percent) { if ($priority > $maxPriority) { continue; } if (!isset($queues_size[$priority])) { return $priority; } $percent = $queues_size[$priority] / ($total / 100); if ($percent < $top_percent) { return $priority; } } return 1; }