public function writeSolrLog($entry)
{
KalturaLog::debug("writeSolrLog " . $entry->getId());
$solrLog = new SphinxLog();
$solrLog->setEntryId($entry->getId());
$solrLog->setPartnerId($entry->getPartnerId());
$solrLog->save(myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_SOLR_LOG));
}
public static function alternativeCon($con, $queryDB = kQueryCache::QUERY_DB_UNDEFINED)
{
if ($con === null) {
$con = myDbHelper::alternativeCon($con);
}
if ($con === null) {
$con = myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL3);
}
return $con;
}
public static function alternativeCon($con)
{
if ($con === null) {
$con = myDbHelper::alternativeCon($con);
}
if ($con === null) {
$con = myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL3);
}
return $con;
}
public function getConfigs()
{
$c = new Criteria();
$c->clearSelectColumns();
$c->addSelectColumn('MAX(' . SchedulerConfigPeer::ID . ')');
$c->addGroupByColumn(SchedulerConfigPeer::VARIABLE);
$c->addGroupByColumn(SchedulerConfigPeer::VARIABLE_PART);
$c->addAscendingOrderByColumn(SchedulerConfigPeer::VARIABLE);
$c->add(SchedulerConfigPeer::SCHEDULER_ID, $this->id);
$c->add(SchedulerConfigPeer::WORKER_ID, null);
$rs = SchedulerConfigPeer::doSelectStmt($c, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2));
$configIds = $rs->fetchAll(PDO::FETCH_COLUMN, 0);
return SchedulerConfigPeer::retrieveByPKs($configIds, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2));
}
public static function doQueueStatus(PDO $con = null)
{
if (is_null($con)) {
$con = myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2);
}
// $con = Propel::getConnection(BatchJobPeer::DATABASE_NAME);
$stmt = null;
try {
$now = time();
$sql = "SELECT\t\tAVG(TIMEDIFF(NOW(), CREATED_AT)) AS CREATED_AT_AVG,\n\t\t\t\t\t\t\t\tCOUNT(JOB_TYPE) AS JOB_TYPE_COUNT, \n\t\t\t\t\t\t\t\tJOB_TYPE \n\t\t\t\t\tFROM\t\tbatch_job \n\t\t\t\t\tFORCE INDEX (created_at_job_type_status_index)\n\t\t\t\t\tWHERE\t\tSTATUS IN (" . BatchJob::BATCHJOB_STATUS_PENDING . ',' . BatchJob::BATCHJOB_STATUS_RETRY . ")\n\t\t\t\t\tAND\t\t\t(\n\t\t\t\t\t\t\t\t\tCHECK_AGAIN_TIMEOUT < {$now}\n\t\t\t\t\t\t\t\t\tOR\n\t\t\t\t\t\t\t\t\tCHECK_AGAIN_TIMEOUT IS NULL\n\t\t\t\t\t\t\t\t)\n\t\t\t\t\tAND\t\t\tJOB_TYPE IS NOT NULL\n\t\t\t\t\tAND\t\t\tCREATED_AT > DATE_ADD(NOW(), INTERVAL -1 HOUR)\n\t\t\t\t\tGROUP BY\tJOB_TYPE";
$stmt = $con->query($sql);
} catch (Exception $e) {
Propel::log($e->getMessage(), Propel::LOG_ERR);
throw new PropelException($e);
}
return $stmt->fetchAll();
}
<?php
set_time_limit(0);
ini_set("memory_limit", "700M");
error_reporting(E_ALL);
chdir(dirname(__FILE__));
define('ROOT_DIR', realpath(dirname(__FILE__) . '/../../'));
require_once ROOT_DIR . '/infra/bootstrap_base.php';
require_once ROOT_DIR . '/infra/KAutoloader.php';
KAutoloader::addClassPath(KAutoloader::buildPath(KALTURA_ROOT_PATH, "vendor", "propel", "*"));
KAutoloader::setClassMapFilePath(kConf::get("cache_root_path") . '/scripts/' . basename(__FILE__) . '.cache');
KAutoloader::register();
KalturaLog::setLogger(new KalturaStdoutLogger());
$dbConf = kConf::getDB();
DbManager::setConfig($dbConf);
DbManager::initialize();
$createdAt = time() - 60 * 60 * 24;
$jobType = 0;
$c = new Criteria();
// $c->add(BatchJobPeer::CREATED_AT, $createdAt, Criteria::GREATER_THAN);
$c->add(BatchJobPeer::JOB_TYPE, $jobType);
// $c->add(BatchJobPeer::STATUS, BatchJob::BATCHJOB_STATUS_PENDING);
$c->clearSelectColumns();
$c->addSelectColumn('MAX(' . BatchJobPeer::PRIORITY . ')');
$stmt = BatchJobPeer::doSelectStmt($c, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2));
$maxPriority = $stmt->fetchColumn();
var_dump($maxPriority);
require_once SF_ROOT_DIR . DIRECTORY_SEPARATOR . 'apps' . DIRECTORY_SEPARATOR . SF_APP . DIRECTORY_SEPARATOR . 'config' . DIRECTORY_SEPARATOR . 'config.php';
KAutoloader::addClassPath(KAutoloader::buildPath(KALTURA_ROOT_PATH, "vendor", "propel", "*"));
KAutoloader::setClassMapFilePath(kConf::get("cache_root_path") . '/scripts/' . basename(__FILE__) . '.cache');
KAutoloader::register();
date_default_timezone_set(kConf::get("date_default_timezone"));
$loggerConfigPath = ROOT_DIR . '/scripts/logger.ini';
$config = new Zend_Config_Ini($loggerConfigPath);
KalturaLog::initLog($config);
KalturaLog::setContext(basename(__FILE__));
KalturaLog::info("Starting script");
KalturaLog::info("Initializing database...");
DbManager::setConfig(kConf::getDB());
DbManager::initialize();
KalturaLog::info("Database initialized successfully");
$syncType = 'kuser';
$dbh = myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_DWH);
$sql = "CALL get_data_for_operational('{$syncType}')";
$count = 0;
$rows = $dbh->query($sql)->fetchAll();
foreach ($rows as $row) {
$kuser = kuserPeer::retrieveByPK($row['kuser_id']);
if (is_null($kuser)) {
KalturaLog::err('Couldn\'t find kuser [' . $row['kuser_id'] . ']');
continue;
}
$kuser->setStorageSize($row['storage_size']);
$kuser->save();
$count++;
KalturaLog::debug('Successfully saved kuser [' . $row['kuser_id'] . ']');
if ($count % 500) {
kuserPeer::clearInstancePool();
$lastFileSyncId = null;
$loopLimit = 500;
while ($moreFileSyncs) {
// ----------------------------
// get next remote file syncs
// ----------------------------
$exportedFileSyncsCriteria = new Criteria();
$exportedFileSyncsCriteria->setLimit($loopLimit);
$exportedFileSyncsCriteria->addAscendingOrderByColumn(FileSyncPeer::ID);
$exportedFileSyncsCriteria->addAnd(FileSyncPeer::DC, $storageProfileId, Criteria::EQUAL);
$exportedFileSyncsCriteria->addAnd(FileSyncPeer::STATUS, FileSync::FILE_SYNC_STATUS_READY, Criteria::EQUAL);
$exportedFileSyncsCriteria->addAnd(FileSyncPeer::PARTNER_ID, $partnerId, Criteria::EQUAL);
if ($lastFileSyncId) {
$exportedFileSyncsCriteria->addAnd(FileSyncPeer::ID, $lastFileSyncId, Criteria::GREATER_THAN);
}
$exportedFileSyncs = FileSyncPeer::doSelect($exportedFileSyncsCriteria, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL3));
// -----------------------------------------------
// delete the local dcs copies of each file sync
// -----------------------------------------------
foreach ($exportedFileSyncs as $exportedFileSync) {
$lastFileSyncId = $exportedFileSync->getId();
$syncKey = kFileSyncUtils::getKeyForFileSync($exportedFileSync);
echo 'Deleting file sync key - ' . serialize($syncKey) . PHP_EOL;
kFileSyncUtils::deleteSyncFileForKey($syncKey, false, true);
// 3rd param = true -> only delete from local dcs
}
// --------------------------
// check if loop should end
// --------------------------
if (count($exportedFileSyncs) < $loopLimit) {
$moreFileSyncs = false;
if (is_null($entry)) {
KalturaLog::err('Couldn\'t find entry [' . $entryId . ']');
continue;
}
if ($entry->getViews() != $views || $entry->getPlays() != $plays) {
$entry->setViews($views);
$entry->setPlays($plays);
KalturaLog::debug('Successfully saved entry [' . $entryId . ']');
try {
// update entry without setting the updated at
$updateSql = "UPDATE entry set views='{$views}',plays='{$plays}' WHERE id='{$entryId}'";
$stmt = $connection->prepare($updateSql);
$stmt->execute();
$affectedRows = $stmt->rowCount();
KalturaLog::log("AffectedRows: " . $affectedRows);
// update sphinx log directly
$sql = $sphinxMgr->getSphinxSaveSql($entry, false);
$sphinxLog = new SphinxLog();
$sphinxLog->setEntryId($entryId);
$sphinxLog->setPartnerId($entry->getPartnerId());
$sphinxLog->setSql($sql);
$sphinxLog->save(myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_SPHINX_LOG));
} catch (Exception $e) {
KalturaLog::log($e->getMessage(), Propel::LOG_ERR);
}
}
$count++;
if ($count % 500 === 0) {
entryPeer::clearInstancePool();
}
}
public static function alternativeCon($con, $queryDB = kQueryCache::QUERY_DB_UNDEFINED)
{
return myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_SPHINX_LOG);
}
public static function getExpiredJobs()
{
$jobTypes = kPluginableEnumsManager::coreValues('BatchJobType');
$executionAttempts2jobTypes = array();
// Map between max execution attempts and job types
foreach ($jobTypes as $jobType) {
$executionAttempts = BatchJobLockPeer::getMaxExecutionAttempts($jobType);
if (array_key_exists($executionAttempts, $executionAttempts2jobTypes)) {
$executionAttempts2jobTypes[$executionAttempts][] = $jobType;
} else {
$executionAttempts2jobTypes[$executionAttempts] = array($jobType);
}
}
// create query
$c = new Criteria();
$c->add(BatchJobLockPeer::STATUS, BatchJob::BATCHJOB_STATUS_FATAL, Criteria::NOT_EQUAL);
$c->add(BatchJobLockPeer::DC, kDataCenterMgr::getCurrentDcId());
// each DC should clean its own jobs
// Query for each job type
$batchJobLocks = array();
foreach ($executionAttempts2jobTypes as $execAttempts => $jobTypes) {
$typedCrit = clone $c;
$typedCrit->add(BatchJobLockPeer::EXECUTION_ATTEMPTS, $execAttempts, Criteria::GREATER_THAN);
$typedCrit->add(BatchJobLockPeer::JOB_TYPE, $jobTypes, Criteria::IN);
$typedJobs = BatchJobLockPeer::doSelect($typedCrit, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2));
foreach ($typedJobs as $typedJob) {
$batchJobLocks[$typedJob->getId()] = $typedJob;
}
}
// get matching batch jobs
return BatchJobPeer::retrieveByPKs(array_keys($batchJobLocks));
}
} else {
$c->add(BatchJobLockPeer::PARTNER_ID, $partnerId);
}
}
$c->setLimit(CHUNK_SIZE);
$movedJobsCount = 0;
while ($movedJobsCount < $maxMovedJobs) {
$jobLocks = BatchJobLockPeer::doSelect($c, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2));
if (!$jobLocks) {
break;
}
$initialMovedJobsCount = $movedJobsCount;
foreach ($jobLocks as $jobLock) {
/* @var $jobLock BatchJobLock */
/* @var $job BatchJob */
$job = $jobLock->getBatchJob(myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2));
// check whether the job can be moved
$jobData = $job->getData();
/* @var $jobData kConvartableJobData */
$srcFileSyncs = $jobData->getSrcFileSyncs();
if (count($srcFileSyncs) != 1) {
continue;
}
// unexpected - multiple sources for doc convert
$srcFileSync = reset($srcFileSyncs);
/* @var $srcFileSync kSourceFileSyncDescriptor */
$sourceAsset = assetPeer::retrieveById($srcFileSync->getAssetId());
if (!$sourceAsset) {
continue;
}
// unexpected - source flavor asset not found
public static function alternativeCon($con)
{
return myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_SPHINX_LOG);
}
public static function getExpiredJobs()
{
$jobTypes = kPluginableEnumsManager::coreValues('BatchJobType');
$c = new Criteria();
$c->add(BatchJobPeer::STATUS, BatchJob::BATCHJOB_STATUS_FATAL, Criteria::NOT_EQUAL);
$c->add(BatchJobPeer::DC, kDataCenterMgr::getCurrentDcId());
// each DC should clean its own jobs
// $c->add(BatchJobPeer::PROCESSOR_EXPIRATION, time(), Criteria::LESS_THAN);
// $c->add(BatchJobPeer::SCHEDULER_ID, 0, Criteria::GREATER_THAN);
// $c->add(BatchJobPeer::WORKER_ID, 0, Criteria::GREATER_THAN);
$jobs = array();
foreach ($jobTypes as $jobType) {
$typedCrit = clone $c;
$typedCrit->add(BatchJobPeer::EXECUTION_ATTEMPTS, BatchJobPeer::getMaxExecutionAttempts($jobType), Criteria::GREATER_THAN);
$typedCrit->add(BatchJobPeer::JOB_TYPE, $jobType);
$typedJobs = BatchJobPeer::doSelect($typedCrit, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2));
foreach ($typedJobs as $typedJob) {
$jobs[] = $typedJob;
}
}
return $jobs;
}
/**
* Update Conversion Profile by ID
*
* @action update
* @param int $id
* @param KalturaConversionProfile $conversionProfile
* @return KalturaConversionProfile
*/
public function updateAction($id, KalturaConversionProfile $conversionProfile)
{
if ($conversionProfile->name !== null) {
$conversionProfile->validatePropertyMinLength("name", 1);
}
if ($conversionProfile->flavorParamsIds !== null) {
$conversionProfile->validateFlavorParamsIds();
}
$conversionProfileDb = conversionProfile2Peer::retrieveByPK($id);
if (!$conversionProfileDb) {
throw new KalturaAPIException(KalturaErrors::CONVERSION_PROFILE_ID_NOT_FOUND, $id);
}
$conversionProfile->toUpdatableObject($conversionProfileDb);
$conversionProfileDb->setCreationMode(conversionProfile2::CONVERSION_PROFILE_2_CREATION_MODE_KMC);
$conversionProfileDb->save();
if ($conversionProfile->flavorParamsIds !== null) {
$this->deleteFlavorParamsRelation($conversionProfileDb);
$this->addFlavorParamsRelation($conversionProfileDb, $conversionProfile->getFlavorParamsAsArray());
}
$conversionProfile->fromObject($conversionProfileDb);
// load flavor params id with the same connection (master connection) that was used for insert
$con = myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_MASTER);
$conversionProfile->loadFlavorParamsIds($conversionProfileDb, $con);
return $conversionProfile;
}
/**
* @param SchedulerWorker $dbData
* @return KalturaScheduler
*/
public function statusFromObject($dbData)
{
$this->fromObject($dbData);
$this->lockedJobs = KalturaBatchJobArray::fromBatchJobArray($dbData->getLockedJobs());
$this->avgWait = BatchJobPeer::doAvgTimeDiff($this->type, BatchJobPeer::CREATED_AT, BatchJobPeer::QUEUE_TIME, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2));
$this->avgWork = BatchJobPeer::doAvgTimeDiff($this->type, BatchJobPeer::QUEUE_TIME, BatchJobPeer::FINISH_TIME, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2));
return $this;
}
/**
* batch freeExclusiveJobAction action allows to get a generic BatchJob
*
* @action freeExclusiveJob
* @param int $id The id of the job
* @param KalturaExclusiveLockKey $lockKey The unique lock key from the batch-process. Is used for the locking mechanism
* @param KalturaBatchJobType $jobType The type of the job
* @param bool $resetExecutionAttempts Resets the job execution attampts to zero
* @return KalturaFreeJobResponse
*/
function freeExclusiveJobAction($id, KalturaExclusiveLockKey $lockKey, $jobType, $resetExecutionAttempts = false)
{
$jobType = kPluginableEnumsManager::apiToCore('BatchJobType', $jobType);
$job = BatchJobPeer::retrieveByPK($id);
// verifies that the job is of the right type
if ($job->getJobType() != $jobType) {
throw new KalturaAPIException(APIErrors::UPDATE_EXCLUSIVE_JOB_WRONG_TYPE, $id, $lockKey, null);
}
$job = kBatchManager::freeExclusiveBatchJob($id, $lockKey->toObject(), $resetExecutionAttempts);
$batchJob = new KalturaBatchJob();
// start from blank
$batchJob->fromObject($job);
// gets queues length
$c = new Criteria();
$c->add(BatchJobPeer::STATUS, array(KalturaBatchJobStatus::PENDING, KalturaBatchJobStatus::RETRY, KalturaBatchJobStatus::ALMOST_DONE), Criteria::IN);
$c->add(BatchJobPeer::JOB_TYPE, $jobType);
$queueSize = BatchJobPeer::doCount($c, false, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2));
if (!$queueSize) {
// gets queues length
$c = new Criteria();
$c->add(BatchJobPeer::BATCH_INDEX, null, Criteria::ISNOTNULL);
$c->add(BatchJobPeer::PROCESSOR_EXPIRATION, time(), Criteria::GREATER_THAN);
$c->add(BatchJobPeer::EXECUTION_ATTEMPTS, BatchJobPeer::getMaxExecutionAttempts($jobType), Criteria::LESS_THAN);
$c->add(BatchJobPeer::JOB_TYPE, $jobType);
$queueSize = BatchJobPeer::doCount($c, false, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2));
}
$response = new KalturaFreeJobResponse();
$response->job = $batchJob;
$response->jobType = $jobType;
$response->queueSize = $queueSize;
return $response;
}
DbManager::setConfig($dbConf);
DbManager::initialize();
if (count($argv) !== 2) {
die('pleas provide partner id as input' . PHP_EOL . 'to run script: ' . basename(__FILE__) . ' X' . PHP_EOL . 'whereas X is partner id' . PHP_EOL);
}
$partner_id = @$argv[1];
$partner = PartnerPeer::retrieveByPK($partner_id);
if (!$partner) {
die('no such partner.' . PHP_EOL);
}
$partner->setAppearInSearch(mySearchUtils::DISPLAY_IN_SEARCH_KALTURA_NETWORK);
$partner->save();
$c = new Criteria();
$c->add(entryPeer::PARTNER_ID, $partner_id);
$c->add(entryPeer::DISPLAY_IN_SEARCH, mySearchUtils::DISPLAY_IN_SEARCH_PARTNER_ONLY);
$c->addOr(entryPeer::DISPLAY_IN_SEARCH, mySearchUtils::DISPLAY_IN_SEARCH_NONE);
$c->setLimit(200);
$con = myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2);
$entries = entryPeer::doSelect($c, $con);
$changedEntriesCounter = 0;
while (count($entries)) {
$changedEntriesCounter += count($entries);
foreach ($entries as $entry) {
echo "changed DISPLAY_IN_SEARCH for entry: " . $entry->getId() . "\n";
$entry->setDisplayInSearch(mySearchUtils::DISPLAY_IN_SEARCH_KALTURA_NETWORK);
$entry->save();
}
entryPeer::clearInstancePool();
$entries = entryPeer::doSelect($c, $con);
}
echo "Done. {$changedEntriesCounter} entries where changed";
public function isValid($partner_id, $puser_id, $type = false)
{
if (!$this->valid_string) {
return self::INVALID_STR;
}
if (!$this->matchPartner($partner_id)) {
return self::INVALID_PARTNER;
}
if (!$this->matchUser($puser_id)) {
return self::INVALID_USER;
}
if ($type !== false) {
// do not check ks type
if (!$this->type == $type) {
return self::INVALID_TYPE;
}
}
if ($this->expired()) {
return self::EXPIRED;
}
if ($this->original_str) {
$invalid = invalidSessionPeer::isInvalid($this->original_str, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2));
if ($invalid) {
return self::LOGOUT;
}
}
return self::OK;
}
/**
* @param string $sql
* @param IIndexable $object
* @return bool
*/
public function execSphinx($sql, IIndexable $object)
{
KalturaLog::debug($sql);
$sphinxLog = new SphinxLog();
$sphinxLog->setEntryId($object->getEntryId());
$sphinxLog->setPartnerId($object->getPartnerId());
$sphinxLog->setSql($sql);
$sphinxLog->save(myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_SPHINX_LOG));
if (!kConf::hasParam('exec_sphinx') || !kConf::get('exec_sphinx')) {
return true;
}
$con = DbManager::getSphinxConnection();
$ret = $con->exec($sql);
if ($ret) {
return true;
}
$arr = $con->errorInfo();
KalturaLog::err($arr[2]);
return false;
}
/**
* batch getFullStatus action returns the status of all schedulers and queues
*
* @action getFullStatus
* @return KalturaFullStatusResponse
*/
function getFullStatusAction()
{
$response = new KalturaFullStatusResponse();
// gets queues length
// $c = new Criteria();
// $c->add(BatchJobPeer::STATUS, array(KalturaBatchJobStatus::PENDING, KalturaBatchJobStatus::RETRY), Criteria::IN);
// $c->addGroupByColumn(BatchJobPeer::JOB_TYPE);
// $c->addSelectColumn('AVG(DATEDIFF(NOW(),' . BatchJobPeer::CREATED_AT . '))');
$queueList = BatchJobPeer::doQueueStatus(myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2));
$response->queuesStatus = KalturaBatchQueuesStatusArray::fromBatchQueuesStatusArray($queueList);
$response->schedulers = KalturaSchedulerArray::statusFromSchedulerArray(SchedulerPeer::doSelect(new Criteria(), myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2)));
return $response;
}
/**
* @param string $sql
* @param IIndexable $object
* @return bool
*/
public function execSphinx($sql, IIndexable $object)
{
// limit the number of large sphinx SQLs to 1/min per object, since they load the sphinx database
// and sphinx servers. the upper limit of 'slightly less than 1MB' is because of the max_allowed_packet
// limit in mysql (by default 1MB). the upper limit is here to prevent the addition of too many category
// users (for example), that will render the category un-indexable
if (strlen($sql) > 128 * 1024 && strlen($sql) < 1000000) {
$lockKey = 'large_sql_lock_' . get_class($object) . '_' . $object->getId();
$cache = kCacheManager::getSingleLayerCache(kCacheManager::CACHE_TYPE_SPHINX_STICKY_SESSIONS);
if ($cache && !$cache->add($lockKey, true, 60)) {
KalturaLog::log('skipping sql for key ' . $lockKey);
return;
}
}
KalturaLog::debug($sql);
$sphinxLog = new SphinxLog();
$sphinxLog->setExecutedServerId($this->retrieveSphinxConnectionId());
$sphinxLog->setObjectId($object->getId());
$sphinxLog->setObjectType(get_class($object));
$sphinxLog->setEntryId($object->getEntryId());
$sphinxLog->setPartnerId($object->getPartnerId());
$sphinxLog->setSql($sql);
$sphinxLog->save(myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_SPHINX_LOG));
if (!kConf::get('exec_sphinx', 'local', 0)) {
return true;
}
$sphinxConnection = DbManager::getSphinxConnection(false);
$ret = $sphinxConnection->exec($sql);
if ($ret) {
return true;
}
$arr = $sphinxConnection->errorInfo();
KalturaLog::err($arr[2]);
return false;
}
$c->addAnd(BatchJobPeer::STATUS, $nonFinalBatchStatuses, Criteria::IN);
$batchCount = BatchJobPeer::doCount($c, false, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL3));
if ($batchCount >= $maxConcurrentJobs) {
sleep(30);
continue;
}
$curLimit = $maxConcurrentJobs - $batchCount;
$currentExported = 0;
$c = new Criteria();
$c->add(entryPeer::PARTNER_ID, $partnerId);
if ($lastCreatedAt) {
$c->addAnd(entryPeer::CREATED_AT, $lastCreatedAt, Criteria::LESS_EQUAL);
}
$c->addDescendingOrderByColumn(entryPeer::CREATED_AT);
$c->setLimit($curLimit);
$entries = entryPeer::doSelect($c, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL3));
foreach ($entries as $entry) {
if (in_array($entry->getId(), $processedIds)) {
continue;
}
$processedIds[] = $entry->getId();
$lastCreatedAt = $entry->getCreatedAt(null);
$keys = array();
$keys[] = $entry->getSyncKey(entry::FILE_SYNC_ENTRY_SUB_TYPE_ISM);
$keys[] = $entry->getSyncKey(entry::FILE_SYNC_ENTRY_SUB_TYPE_ISMC);
$flavors = assetPeer::retrieveReadyFlavorsByEntryId($entry->getId());
foreach ($flavors as $flavor) {
if (!$flavorParamsArr || in_array($flavor->getFlavorParamsId(), $flavorParamsArr)) {
$keys[] = $flavor->getSyncKey(flavorAsset::FILE_SYNC_FLAVOR_ASSET_SUB_TYPE_ASSET);
}
}
/**
* @param string $sql
* @param IIndexable $object
* @return bool
*/
public function execSphinx($sql, IIndexable $object)
{
KalturaLog::debug($sql);
$sphinxConnection = null;
$sphinxConnectionId = null;
if (kConf::hasParam('exec_sphinx') && kConf::get('exec_sphinx')) {
$sphinxConnection = DbManager::getSphinxConnection(false);
$sphinxServer = SphinxLogServerPeer::retrieveByLocalServer($sphinxConnection->getHostName());
if ($sphinxServer) {
$sphinxConnectionId = $sphinxServer->getId();
}
}
$sphinxLog = new SphinxLog();
$sphinxLog->setExecutedServerId($sphinxConnectionId);
$sphinxLog->setObjectId($object->getId());
$sphinxLog->setObjectType(get_class($object));
$sphinxLog->setEntryId($object->getEntryId());
$sphinxLog->setPartnerId($object->getPartnerId());
$sphinxLog->setSql($sql);
$sphinxLog->save(myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_SPHINX_LOG));
if (!$sphinxConnection) {
return true;
}
$sphinxConnection = DbManager::getSphinxConnection(false);
$ret = $sphinxConnection->exec($sql);
if ($ret) {
return true;
}
$arr = $sphinxConnection->errorInfo();
KalturaLog::err($arr[2]);
return false;
}
public function getDirectChildJobs()
{
$c = new Criteria();
$c->add(BatchJobPeer::PARENT_JOB_ID, $this->id);
return BatchJobPeer::doSelect($c, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2));
}
/**
* Update Conversion Profile by ID
*
* @action update
* @param int $id
* @param KalturaConversionProfile $conversionProfile
* @return KalturaConversionProfile
*/
public function updateAction($id, KalturaConversionProfile $conversionProfile)
{
$conversionProfileDb = conversionProfile2Peer::retrieveByPK($id);
if (!$conversionProfileDb) {
throw new KalturaAPIException(KalturaErrors::CONVERSION_PROFILE_ID_NOT_FOUND, $id);
}
$conversionProfile->toUpdatableObject($conversionProfileDb);
$conversionProfileDb->setCreationMode(conversionProfile2::CONVERSION_PROFILE_2_CREATION_MODE_KMC);
if ($conversionProfile->xslTransformation) {
$conversionProfileDb->incrementXslVersion();
}
$conversionProfileDb->save();
if ($conversionProfile->flavorParamsIds !== null) {
$this->deleteFlavorParamsRelation($conversionProfileDb, $conversionProfile->flavorParamsIds);
$this->addFlavorParamsRelation($conversionProfileDb, $conversionProfile->getFlavorParamsAsArray());
}
if ($conversionProfile->xslTransformation) {
$xsl = html_entity_decode($conversionProfile->xslTransformation);
$key = $conversionProfileDb->getSyncKey(conversionProfile2::FILE_SYNC_MRSS_XSL);
kFileSyncUtils::file_put_contents($key, $xsl);
}
$conversionProfile->fromObject($conversionProfileDb);
// load flavor params id with the same connection (master connection) that was used for insert
$con = myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_MASTER);
$conversionProfile->loadFlavorParamsIds($conversionProfileDb, $con);
return $conversionProfile;
}
if (isset($lastLogs[$dc])) {
$serverLastLog = $lastLogs[$dc];
if ($serverLastLog->getLastLogId() >= $sphinxLog->getId()) {
KalturaLog::debug('Last log id [' . $serverLastLog->getLastLogId() . "] dc [{$dc}] is larger than id [" . $sphinxLog->getId() . "]");
continue;
}
} else {
$serverLastLog = new SphinxLogServer();
$serverLastLog->setServer($sphinxServer);
$serverLastLog->setDc($dc);
$lastLogs[$dc] = $serverLastLog;
}
try {
$sql = $sphinxLog->getSql();
$affected = $sphinxCon->exec($sql);
if (!$affected) {
$errorInfo = $sphinxCon->errorInfo();
// if(!preg_match('/^duplicate id/', $errorInfo[2]))
// die("No affected records [" . $sphinxCon->errorCode() . "]\n" . print_r($sphinxCon->errorInfo(), true));
}
$serverLastLog->setLastLogId($sphinxLog->getId());
$serverLastLog->save(myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_SPHINX_LOG));
} catch (Exception $e) {
KalturaLog::err($e->getMessage());
}
}
unset($sphinxCon);
SphinxLogPeer::clearInstancePool();
$sphinxLogs = SphinxLogPeer::retrieveByLastId($lastLogs);
}
KalturaLog::log('Done');
private function compareOldToNewQueries($criteria, $sphinxOldQuery, $mysqlOldQuery)
{
$pdo = DbManager::getSphinxConnection();
$successfulTest = true;
$OldSphinxQueryIds = array();
$OldQueryIds = array('just somthing to put so this is not empty');
$criteria->applyFilters();
$sphinxNewQueryids = $criteria->getFetchedIds();
try {
var_dump($sphinxOldQuery);
$stmtSphinxOldQuery = $pdo->query($sphinxOldQuery);
if (!$stmtSphinxOldQuery) {
echo "Invalid sphinx query [" . $sphinxOldQuery;
}
$OldSphinxQueryIds = $stmtSphinxOldQuery->fetchAll(PDO::FETCH_COLUMN, 2);
} catch (Exception $x) {
echo 'error in old sphinx select: ' . $x->getMessage() . PHP_EOL;
echo 'old sphinx query: ' . $sphinxOldQuery . PHP_EOL;
$successfulTest = false;
}
if ($mysqlOldQuery != null) {
try {
$con = myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2);
/* var PDOStatement $stmt*/
$stmt = $con->query($mysqlOldQuery);
$OldQueryIds = $stmt->fetchAll(PDO::FETCH_COLUMN);
$stmt->execute();
} catch (Exception $x) {
echo 'error in mysql old select: ' . $x->getMessage() . PHP_EOL;
echo 'old mysqk query: ' . ${$mysqlOldQuery} . PHP_EOL;
$successfulTest = false;
}
}
$OldQueryIds = array_slice($OldQueryIds, 0, 500);
if (!$successfulTest || is_null($mysqlOldQuery) || count(array_diff($OldQueryIds, $sphinxNewQueryids)) || count(array_diff($sphinxNewQueryids, $OldQueryIds))) {
if ($successfulTest && !(count(array_diff($OldSphinxQueryIds, $sphinxNewQueryids)) || count(array_diff($sphinxNewQueryids, $OldSphinxQueryIds)))) {
echo 'Test OK : ' . PHP_EOL;
echo 'Test Faild - but was the same in sphinx: ' . print_r($OldSphinxQueryIds, true) . print_r($sphinxNewQueryids, true) . PHP_EOL;
$successfulTest = true;
} else {
echo 'Test Faild: ' . PHP_EOL;
if ($successfulTest && ($mysqlOldQuery != null && strstr($mysqlOldQuery, 'COUNT(*)'))) {
echo 'Test ignored: ' . PHP_EOL;
$successfulTest = true;
} else {
$successfulTest = false;
}
}
}
if ($successfulTest) {
echo 'Successful Test' . PHP_EOL;
} else {
echo 'Fail Test' . PHP_EOL;
}
return $successfulTest;
}
public static function alternativeCon($con, $queryDB = kQueryCache::QUERY_DB_UNDEFINED)
{
if ($con === null) {
switch ($queryDB) {
case kQueryCache::QUERY_DB_MASTER:
$con = myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_MASTER);
break;
case kQueryCache::QUERY_DB_SLAVE:
$con = myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2);
break;
}
}
if ($con === null) {
$con = myDbHelper::alternativeCon($con);
}
if ($con === null) {
$con = Propel::getConnection(MetadataProfilePeer::DATABASE_NAME, Propel::CONNECTION_READ);
}
return $con;
}
public static function getNextJobPriority($jobType)
{
//$priorities = array(1 => 33, 2 => 27, 3 => 20, 4 => 13, 5 => 7);
$priorities = kConf::get('priority_percent');
$createdAt = time() - kConf::get('priority_time_range');
// $createdAt = kConf::get('priority_time_range');
$c = new Criteria();
$c->add(BatchJobPeer::CREATED_AT, $createdAt, Criteria::GREATER_THAN);
$c->add(BatchJobPeer::JOB_TYPE, $jobType);
$c->add(BatchJobPeer::STATUS, BatchJob::BATCHJOB_STATUS_PENDING);
$c->clearSelectColumns();
$c->addSelectColumn('MAX(' . BatchJobPeer::PRIORITY . ')');
$stmt = BatchJobPeer::doSelectStmt($c, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2));
$maxPriority = $stmt->fetchColumn();
// gets the current queues
$c = new Criteria();
$c->add(BatchJobPeer::CREATED_AT, $createdAt, Criteria::GREATER_THAN);
$c->add(BatchJobPeer::JOB_TYPE, $jobType);
$c->add(BatchJobPeer::STATUS, BatchJob::BATCHJOB_STATUS_PENDING, Criteria::GREATER_THAN);
$c->addGroupByColumn(BatchJobPeer::PRIORITY);
// To prevent stress on the master DB - use the slave for checking the queue sizes
$queues = BatchJobPeer::doCountGroupBy($c, myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_PROPEL2));
// copy the queues and calcs the total
$total = 0;
$queues_size = array();
foreach ($queues as $queue) {
$queues_size[$queue['PRIORITY']] = $queue[BatchJobPeer::COUNT];
$total += $queue[BatchJobPeer::COUNT];
}
// go over the priorities and see if its percent not used
foreach ($priorities as $priority => $top_percent) {
if ($priority > $maxPriority) {
continue;
}
if (!isset($queues_size[$priority])) {
return $priority;
}
$percent = $queues_size[$priority] / ($total / 100);
if ($percent < $top_percent) {
return $priority;
}
}
return 1;
}
