/**
* {@inheritdoc}
* @see Scalr_System_Cronjob_MultiProcess_DefaultWorker::enqueueWork()
*/
function enqueueWork($workQueue)
{
$rows = $this->db->GetAll("\n SELECT id FROM farms\n WHERE status=? AND clientid\n IN (SELECT id FROM clients WHERE status = 'Active')\n ", array(FARM_STATUS::RUNNING));
foreach ($rows as $row) {
$workQueue->put($row["id"]);
}
}
/**
* @param int $bID
*/
public function inspectBlock($bID)
{
$bID = (int) $bID;
$this->write("Block with ID {$bID}");
++$this->indent;
$b = $this->cn->GetRow('select btID from Blocks where bID = ?', array($bID));
if (empty($b)) {
$this->write('BLOCK NOT FOUND!');
} else {
$btID = (int) $b['btID'];
$blockTypes = $this->getBlockTypes();
if (!isset($blockTypes[$btID])) {
$this->write("BLOCK TYPE WITH ID {$btID} NOT FOUND!");
} else {
$this->write('Type handle: ' . $blockTypes[$btID]['handle']);
$this->write('Type name : ' . $blockTypes[$btID]['name']);
}
$cvbList = $this->cn->GetAll('select cID, cvID, arHandle from CollectionVersionBlocks where bID = ? order by cvID desc', array($bID));
if (empty($cvbList)) {
$this->write('BLOCK NOT FOUND IN ANY COLLECTION!');
} else {
$this->write('Block found here:');
++$this->indent;
foreach ($cvbList as $cvb) {
$this->inspectArea($cvb['cID'], $cvb['cvID'], $cvb['arHandle']);
}
--$this->indent;
}
}
--$this->indent;
}
function enqueueWork($workQueue)
{
$this->logger->info("Fetching farms...");
$farms = array();
$envs = $this->db->GetAll('SELECT env_id, value FROM governance WHERE enabled = 1 AND name = ?', array(Scalr_Governance::GENERAL_LEASE));
foreach ($envs as $env) {
$env['value'] = json_decode($env['value'], true);
$period = 0;
if (is_array($env['value']['notifications'])) {
foreach ($env['value']['notifications'] as $notif) {
if ($notif['period'] > $period) {
$period = $notif['period'];
}
}
$dt = new DateTime();
$dt->sub(new DateInterval('P' . $period . 'D'));
$fs = $this->db->GetAll('SELECT farmid, status FROM farm_settings
LEFT JOIN farms ON farms.id = farm_settings.farmid
WHERE farm_settings.name = ? AND status = ? AND env_id = ? AND value > ?', array(DBFarm::SETTING_LEASE_TERMINATE_DATE, FARM_STATUS::RUNNING, $env['env_id'], $dt->format('Y-m-d H:i:s')));
foreach ($fs as $f) {
if (!isset($farms[$f['farmid']])) {
$farms[$f['farmid']] = true;
$workQueue->put($f['farmid']);
}
}
}
}
$this->logger->info("Found " . count($farms) . " lease tasks");
}
function enqueueWork($workQueue)
{
$rows = $this->db->GetAll("SELECT id FROM farm_roles WHERE role_id IN (SELECT role_id FROM role_behaviors WHERE behavior IN (?,?,?,?))", array(ROLE_BEHAVIORS::POSTGRESQL, ROLE_BEHAVIORS::REDIS, ROLE_BEHAVIORS::MYSQL2, ROLE_BEHAVIORS::PERCONA));
$this->logger->info("Found " . count($rows) . " DbMsr farm roles...");
foreach ($rows as $row) {
$workQueue->put($row["id"]);
}
}
function enqueueWork($workQueue)
{
$this->logger->info("Fetching pending messages...");
$rows = $this->db->GetAll("SELECT id FROM messages\n WHERE `type`='out' AND status=? AND message_format='json' AND DATE_ADD(dtlasthandleattempt, INTERVAL handle_attempts MINUTE) < NOW() ORDER BY id DESC LIMIT 0,3000", array(MESSAGE_STATUS::PENDING));
$this->logger->info("Found " . count($rows) . " pending messages");
foreach ($rows as $row) {
$workQueue->put($row["id"]);
}
}
function enqueueWork($workQueue)
{
$this->logger->info("Fetching active farms...");
$rows = $this->db->GetAll("SELECT id FROM dm_deployment_tasks WHERE status IN ('pending','deploying')");
$this->logger->info("Found " . count($rows) . " deployment tasks");
foreach ($rows as $row) {
$workQueue->put($row["id"]);
}
}
function enqueueWork($workQueue)
{
$this->logger->info("Fetching servers...");
$farmid = $this->runOptions['getopt']->getOption('farm-id');
if ($farmid) {
$rows = $this->db->GetAll("\n SELECT distinct(m.server_id)\n FROM messages m\n INNER JOIN servers s ON m.server_id = s.server_id\n WHERE m.type = ? AND m.status = ? AND s.farm_id = ?\n ", array("in", MESSAGE_STATUS::PENDING, $farmid));
} else {
$rows = $this->db->GetAll("\n SELECT distinct(m.server_id)\n FROM messages m\n WHERE m.type = ? AND m.status = ?\n AND m.message_name NOT IN (?, ?, ?)\n ", array("in", MESSAGE_STATUS::PENDING, EVENT_TYPE::BEFORE_HOST_UP, EVENT_TYPE::HOST_INIT, EVENT_TYPE::HOST_UP));
}
$this->logger->info("Found " . count($rows) . " servers");
foreach ($rows as $row) {
$workQueue->put($row["server_id"]);
}
}
/**
* Synchronizes the account level tag value
*
* It does not verify itself whether the cost analytics service is enabled
*
* @param int $accountId The identifier of the client's account
* @param int $tagId The identifier of the clould analytics tag
* @param string $valueId The identifier of the tag's value
* @param string $valueName The name of the tag's value
*/
public function syncValue($accountId, $tagId, $valueId, $valueName)
{
if ($accountId === null) {
$accountId = 0;
}
$tag = AccountTagEntity::findPk($accountId, $tagId, $valueId);
if (!$tag instanceof AccountTagEntity) {
$tag = new AccountTagEntity();
$tag->accountId = $accountId;
$tag->tagId = $tagId;
$tag->valueId = $valueId;
$tag->valueName = $valueName;
} else {
if ($tag->valueName != $valueName) {
$tag->valueName = $valueName;
if ($tagId == TagEntity::TAG_ID_FARM) {
foreach ($this->db->GetAll("\n SELECT fr.id AS farm_role_id, fr.alias\n FROM farm_roles fr\n WHERE fr.farmid = ?\n ", [$valueId]) as $v) {
//Updates all related farm roles
$this->syncValue($accountId, TagEntity::TAG_ID_FARM_ROLE, $v['farm_role_id'], sprintf('%s', $v['alias']));
}
}
} else {
$ignoreupdate = true;
}
}
if (!isset($ignoreupdate)) {
$tag->save();
}
}
/**
* {@inheritdoc}
* @see Scalr_System_Cronjob_MultiProcess_DefaultWorker::enqueueWork()
*/
function enqueueWork($workQueue)
{
$this->logger->info("Fetching completed farms...");
$rows = $this->db->GetAll("\n SELECT farms.id, farms.status\n FROM farms\n JOIN clients ON clients.id = farms.clientid\n JOIN client_environments ON client_environments.id = farms.env_id\n WHERE clients.status='Active'\n AND client_environments.status = 'Active'\n ");
foreach ($rows as $row) {
if ($this->db->GetOne("SELECT COUNT(*) FROM servers WHERE farm_id=?", array($row['id'])) != 0) {
$workQueue->put($row['id']);
} else {
if ($row['status'] == FARM_STATUS::SYNCHRONIZING) {
$this->db->Execute('UPDATE farms SET status = ? WHERE id = ?', array(FARM_STATUS::TERMINATED, $row['id']));
}
}
}
$this->logger->info(sprintf("Found %d farms.", count($rows)));
}
public function loadByFilter($filterArgs = array(), $ufilterArgs = array())
{
$sql = array();
$args = array();
foreach ($filterArgs as $key => $value) {
if ($property = $this->findDbKeyByProperty($key)) {
if (is_array($value)) {
if (count($value)) {
foreach ($value as $vvalue) {
$args[] = $this->db->quote($vvalue);
}
$sql[] = "`{$property}` IN (" . implode(",", array_fill(0, count($value), "?")) . ")";
}
} else {
$sql[] = "`{$property}` = ?";
$args[] = $value;
}
}
}
foreach ($ufilterArgs as $key => $value) {
if ($property = $this->findDbKeyByProperty($key)) {
if (is_array($value)) {
if (count($value)) {
foreach ($value as $vvalue) {
$args[] = $this->db->quote($vvalue);
}
$sql[] = "`{$key}` NOT IN (" . implode(",", array_fill(0, count($value), "?")) . ")";
}
} else {
$sql[] = "`{$key}` != ?";
$args[] = $value;
}
}
}
$sqlString = "SELECT * FROM {$this->dbTableName}";
if (count($sql)) {
$sqlString .= " WHERE " . implode(" AND ", $sql);
}
//TODO: Return array of objects
//die("TODO");
return $this->db->GetAll($sqlString, $args);
}
/**
* {@inheritdoc}
* @see \Scalr\System\Zmq\Cron\TaskInterface::worker()
*/
public function worker($request)
{
//Warming up static DI cache
\Scalr::getContainer()->warmup();
// Reconfigure observers
\Scalr::ReconfigureObservers();
if (!isset($request->farmRoleId)) {
//This is the farm with synchronous launch of roles
try {
$DBFarm = DBFarm::LoadByID($request->farmId);
if ($DBFarm->Status != FARM_STATUS::RUNNING) {
$this->getLogger()->warn("[FarmID: %d] Farm isn't running. There is no need to scale it.", $DBFarm->ID);
return false;
}
} catch (Exception $e) {
$this->getLogger()->error("Could not load farm '%s' with ID:%d", $request->farmName, $request->farmId);
throw $e;
}
//Gets the list of the roles
$list = $DBFarm->GetFarmRoles();
} else {
//This is asynchronous lauhch
try {
$DBFarmRole = DBFarmRole::LoadByID($request->farmRoleId);
if ($DBFarmRole->getFarmStatus() != FARM_STATUS::RUNNING) {
//We don't need to handle inactive farms
return false;
}
} catch (Exception $e) {
$this->getLogger()->error("Could not load FarmRole with ID:%d", $request->farmRoleId);
throw $e;
}
$list = [$DBFarmRole];
}
$this->getLogger()->debug("Processing %s FarmRoles", count($list));
foreach ($list as $DBFarmRole) {
// Set Last polling time
$DBFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_LAST_POLLING_TIME, time(), Entity\FarmRoleSetting::TYPE_LCL);
$disabledScaling = false;
if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_ENABLED) != '1') {
if ($DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ) || $DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::VPC_ROUTER)) {
// For Mongo, RabbitMQ and VPC Router we need to launch first instance (or maintain 1 instance running)
// When 1 instance is already running, the rest is fully manual
$roleTotalInstances = $DBFarmRole->GetRunningInstancesCount() + $DBFarmRole->GetPendingInstancesCount();
if ($roleTotalInstances != 0) {
$disabledScaling = true;
}
} else {
if (!$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) {
$disabledScaling = true;
}
}
if ($disabledScaling) {
$this->getLogger()->info("[FarmID: %d] Scaling is disabled for role '%s'. Skipping...", $request->farmId, $DBFarmRole->Alias);
continue;
}
}
$farmRoleName = $DBFarmRole->Alias ? $DBFarmRole->Alias : $DBFarmRole->GetRoleObject()->name;
// Get current count of running and pending instances.
$this->getLogger()->info(sprintf("Processing role '%s'", $farmRoleName));
$scalingManager = new Scalr_Scaling_Manager($DBFarmRole);
//Replacing the logger
$scalingManager->logger = $this->getLogger();
$scalingDecision = $scalingManager->makeScalingDecision();
$scalingDecisionDetails = $scalingManager->decisonInfo;
$this->getLogger()->info(sprintf("Decision '%s' (%s)", $scalingDecision, $scalingDecisionDetails));
if ($scalingDecision == Scalr_Scaling_Decision::STOP_SCALING) {
return;
}
if ($scalingDecision == Scalr_Scaling_Decision::NOOP) {
continue;
} else {
if ($scalingDecision == Scalr_Scaling_Decision::DOWNSCALE) {
/*
Timeout instance's count decrease. Decreases instances count after scaling
resolution the spare instances are running for selected timeout interval
from scaling EditOptions
*/
// We have to check timeout limits before new scaling (downscaling) process will be initiated
if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_TIMEOUT_ENABLED)) {
// if the farm timeout is exceeded
// checking timeout interval.
$last_down_scale_data_time = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_DATETIME);
$timeout_interval = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_TIMEOUT);
// check the time interval to continue scaling or cancel it...
if (time() - $last_down_scale_data_time < $timeout_interval * 60) {
// if the launch time is too small to terminate smth in this role -> go to the next role in foreach()
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage(!empty($request->farmId) ? $request->farmId : null, sprintf("Waiting for downscaling timeout on farm %s, role %s", !empty($request->farmName) ? $request->farmName : null, !empty($DBFarmRole->Alias) ? $DBFarmRole->Alias : null), null, null, !empty($DBFarmRole->ID) ? $DBFarmRole->ID : null));
continue;
}
}
// end Timeout instance's count decrease
$sort = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_KEEP_OLDEST) == 1 ? 'DESC' : 'ASC';
$servers = $this->db->GetAll("SELECT server_id FROM servers WHERE status = ? AND farm_roleid=? ORDER BY dtadded {$sort}", array(SERVER_STATUS::RUNNING, $DBFarmRole->ID));
$got_valid_instance = false;
$ignoreFullHour = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_IGNORE_FULL_HOUR);
$useSafeShutdown = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_SAFE_SHUTDOWN);
$isRabbitMQ = $DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ);
// Select instance that will be terminated
//
// Instances ordered by uptime (oldest wil be choosen)
// Instance cannot be mysql master
// Choose the one that was rebundled recently
$DBServer = null;
while (!$got_valid_instance && count($servers) > 0) {
$item = array_shift($servers);
$DBServer = DBServer::LoadByID($item['server_id']);
if ($isRabbitMQ) {
$serverExists = $this->db->GetOne("\n SELECT EXISTS (\n SELECT 1 FROM servers\n WHERE farm_roleid = ?\n AND status NOT IN (?, ?)\n AND `index` != ?\n )\n ", [$DBServer->farmRoleId, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED, 1]);
if ($DBServer->index == 1 && $serverExists) {
continue;
}
}
if ($DBServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED)) {
continue;
}
// Exclude db master
if ($DBServer->GetProperty(SERVER_PROPERTIES::DB_MYSQL_MASTER) != 1 && $DBServer->GetProperty(Scalr_Db_Msr::REPLICATION_MASTER) != 1) {
$got_valid_instance = true;
}
//Check safe shutdown
if ($useSafeShutdown == 1) {
try {
$res = $DBServer->scalarizr->system->callAuthShutdownHook();
} catch (Exception $e) {
$res = $e->getMessage();
}
if ($res != '1') {
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBServer->farmId, sprintf("Safe shutdown enabled. Server '%s'. Script returned '%s' skipping it.", $DBServer->serverId, $res), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId));
$got_valid_instance = false;
}
}
}
// end while
if ($DBServer !== null && $got_valid_instance) {
$this->getLogger()->info(sprintf("Server '%s' selected for termination...", $DBServer->serverId));
$allow_terminate = false;
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
$aws = $DBServer->GetEnvironmentObject()->aws($DBServer);
// Shutdown an instance just before a full hour running
if (!$ignoreFullHour) {
$response = $aws->ec2->instance->describe($DBServer->GetProperty(EC2_SERVER_PROPERTIES::INSTANCE_ID))->get(0);
if ($response && count($response->instancesSet)) {
$launch_time = $response->instancesSet->get(0)->launchTime->getTimestamp();
$time = 3600 - (time() - $launch_time) % 3600;
// Terminate instance in < 10 minutes for full hour.
if ($time <= 600) {
$allow_terminate = true;
} else {
$timeout = round(($time - 600) / 60, 1);
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling down (%s). Server '%s' will be terminated in %s minutes. Launch time: %s", $DBServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $DBServer->serverId, $timeout, $response->instancesSet->get(0)->launchTime->format('c')), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId));
}
}
} else {
$allow_terminate = true;
}
//Releases memory
$DBServer->GetEnvironmentObject()->getContainer()->release('aws');
unset($aws);
} else {
$allow_terminate = true;
}
if ($allow_terminate) {
$terminateStrategy = $DBFarmRole->GetSetting(Scalr_Role_Behavior::ROLE_BASE_TERMINATE_STRATEGY);
if (!$terminateStrategy) {
$terminateStrategy = 'terminate';
}
try {
if ($terminateStrategy == 'terminate') {
$DBServer->terminate(DBServer::TERMINATE_REASON_SCALING_DOWN, false);
$DBFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_DATETIME, time(), Entity\FarmRoleSetting::TYPE_LCL);
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling down (%s). Server '%s' marked as 'Pending terminate' and will be fully terminated in 3 minutes.", $DBServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $DBServer->serverId), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId));
} else {
$DBServer->suspend('SCALING_DOWN', false);
$DBFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_DOWNSCALE_DATETIME, time(), Entity\FarmRoleSetting::TYPE_LCL);
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling down (%s). Server '%s' marked as 'Pending suspend' and will be fully suspended in 3 minutes.", $DBServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $DBServer->serverId), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId));
}
} catch (Exception $e) {
$this->getLogger()->fatal(sprintf("Cannot %s %s: %s", $terminateStrategy, $request->farmId, $DBServer->serverId));
}
}
} else {
$this->getLogger()->warn(sprintf("[FarmID: %s] Scalr unable to determine what instance it should terminate (FarmRoleID: %s). Skipping...", $request->farmId, $DBFarmRole->ID));
}
//break;
} elseif ($scalingDecision == Scalr_Scaling_Decision::UPSCALE) {
/*
Timeout instance's count increase. Increases instance's count after
scaling resolution 'need more instances' for selected timeout interval
from scaling EditOptions
*/
if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_UPSCALE_TIMEOUT_ENABLED)) {
// if the farm timeout is exceeded
// checking timeout interval.
$last_up_scale_data_time = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_UPSCALE_DATETIME);
$timeout_interval = $DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_UPSCALE_TIMEOUT);
// check the time interval to continue scaling or cancel it...
if (time() - $last_up_scale_data_time < $timeout_interval * 60) {
// if the launch time is too small to terminate smth in this role -> go to the next role in foreach()
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(sprintf("Waiting for upscaling timeout on farm %s, role %s", $request->farmName, $DBFarmRole->Alias));
continue;
}
}
// end Timeout instance's count increase
//Check DBMsr. Do not start slave during slave2master process
$isDbMsr = $DBFarmRole->GetRoleObject()->getDbMsrBehavior();
if ($isDbMsr) {
if ($DBFarmRole->GetSetting(Scalr_Db_Msr::SLAVE_TO_MASTER)) {
$runningServers = $DBFarmRole->GetRunningInstancesCount();
if ($runningServers > 0) {
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($request->farmId, sprintf("Role is in slave2master promotion process. Do not launch new slaves while there is no active slaves")));
continue;
} else {
$DBFarmRole->SetSetting(Scalr_Db_Msr::SLAVE_TO_MASTER, 0, Entity\FarmRoleSetting::TYPE_LCL);
}
}
}
if ($DBFarmRole->GetSetting(Entity\FarmRoleSetting::SCALING_ONE_BY_ONE) == 1) {
$pendingInstances = $DBFarmRole->GetPendingInstancesCount();
if ($pendingInstances > 0) {
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("There are %s pending intances of %s role on % farm. Waiting...", $pendingInstances, $DBFarmRole->Alias, $request->farmName)));
continue;
}
}
$fstatus = $this->db->GetOne("SELECT status FROM farms WHERE id=? LIMIT 1", array($request->farmId));
if ($fstatus != FARM_STATUS::RUNNING) {
$this->getLogger()->warn("[FarmID: {$request->farmId}] Farm terminated. There is no need to scale it.");
return;
}
$terminateStrategy = $DBFarmRole->GetSetting(Scalr_Role_Behavior::ROLE_BASE_TERMINATE_STRATEGY);
if (!$terminateStrategy) {
$terminateStrategy = 'terminate';
}
$suspendedServer = null;
if ($terminateStrategy == 'suspend') {
$suspendedServers = $DBFarmRole->GetServersByFilter(array('status' => SERVER_STATUS::SUSPENDED));
if (count($suspendedServers) > 0) {
$suspendedServer = array_shift($suspendedServers);
}
}
if ($terminateStrategy == 'suspend' && $suspendedServer) {
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling up (%s). Found server to resume. ServerID = %s.", $suspendedServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $suspendedServer->serverId), $suspendedServer->serverId, $suspendedServer->envId, $suspendedServer->farmRoleId));
}
if ($terminateStrategy == 'terminate' || !$suspendedServer || !PlatformFactory::isOpenstack($suspendedServer->platform) && $suspendedServer->platform != SERVER_PLATFORMS::EC2 && $suspendedServer->platform != SERVER_PLATFORMS::GCE) {
$ServerCreateInfo = new ServerCreateInfo($DBFarmRole->Platform, $DBFarmRole);
try {
$DBServer = \Scalr::LaunchServer($ServerCreateInfo, null, false, DBServer::LAUNCH_REASON_SCALING_UP);
$DBFarmRole->SetSetting(Entity\FarmRoleSetting::SCALING_UPSCALE_DATETIME, time(), Entity\FarmRoleSetting::TYPE_LCL);
\Scalr::getContainer()->logger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($request->farmId, sprintf("Role '%s' scaling up (%s). Starting new instance. ServerID = %s.", $DBServer->GetFarmRoleObject()->Alias, $scalingDecisionDetails, $DBServer->serverId), $DBServer->serverId, $DBServer->envId, $DBServer->farmRoleId));
} catch (Exception $e) {
\Scalr::getContainer()->logger(LOG_CATEGORY::SCALING)->error($e->getMessage());
}
} else {
$platform = PlatformFactory::NewPlatform($suspendedServer->platform);
$platform->ResumeServer($suspendedServer);
}
}
}
}
return true;
}
protected function buildResponseFromSql($sql, $filterFields = array(), $groupSQL = "", $simpleQuery = true, $noLimit = false)
{
$this->request->defineParams(array('start' => array('type' => 'int', 'default' => 0), 'limit' => array('type' => 'int', 'default' => 20)));
if (is_array($groupSQL)) {
return $this->buildResponseFromSql2($sql, $filterFields, $groupSQL, is_array($simpleQuery) ? $simpleQuery : array(), $noLimit);
}
if ($this->getParam('query') && count($filterFields) > 0) {
$filter = $this->db->qstr('%' . trim($this->getParam('query')) . '%');
foreach ($filterFields as $field) {
if ($simpleQuery) {
$likes[] = "`{$field}` LIKE {$filter}";
} else {
$likes[] = "{$field} LIKE {$filter}";
}
}
$sql .= " AND (";
$sql .= implode(" OR ", $likes);
$sql .= ")";
}
if ($groupSQL) {
$sql .= "{$groupSQL}";
}
if (!$noLimit) {
$response['total'] = $this->db->GetOne('SELECT COUNT(*) FROM (' . $sql . ') c_sub');
}
// @TODO replace with simple code (legacy code)
$s = $this->getParam('sort');
if (!is_array($s)) {
$s = json_decode($this->getParam('sort'), true);
}
if (is_array($s)) {
$sorts = array();
if (count($s) && (!isset($s[0]) || !is_array($s[0]))) {
$s = array($s);
}
foreach ($s as $param) {
$sort = preg_replace("/[^A-Za-z0-9_]+/", "", $param['property']);
$dir = in_array(strtolower($param['direction']), array('asc', 'desc')) ? $param['direction'] : 'ASC';
if ($sort && $dir) {
$sorts[] = "`{$sort}` {$dir}";
}
}
if (count($sorts) > 0) {
$sql .= " ORDER BY " . implode($sorts, ',');
}
} else {
if ($this->getParam('sort')) {
$sort = preg_replace("/[^A-Za-z0-9_]+/", "", $this->getParam('sort'));
$dir = in_array(strtolower($this->getParam('dir')), array('asc', 'desc')) ? $this->getParam('dir') : 'ASC';
$sql .= " ORDER BY `{$sort}` {$dir}";
}
}
if (!$noLimit) {
$start = intval($this->getParam('start'));
if ($start > $response["total"]) {
$start = 0;
}
$limit = intval($this->getParam('limit'));
$sql .= " LIMIT {$start}, {$limit}";
}
//$response['sql'] = $sql;
$response["success"] = true;
$response["data"] = $this->db->GetAll($sql);
return $response;
}
/**
* Sets ACL roles to this user
*
* This method modifies resords of two tables
* `account_team_users` and `account_team_user_acls`.
*
* Attention! It expects full list of the ACL roles relations for user.
* All missing relations will be removed.
*
* @param int $userId The ID of the user
* @param array $data ACL roles array which looks like
* array(teamId => array(accountRoleId1, accountRoleId2, ...))
* @param int $accountId optional The ID of the account. Restricts queries to the
* specified account.
*/
public function setAllRolesForUser($userId, array $data = array(), $accountId = null)
{
$tu = array();
$rs = $this->db->Execute("\n SELECT tu.`id`, tu.`team_id` FROM `account_team_users` tu WHERE tu.`user_id` = ?\n ", array($userId));
while ($rec = $rs->FetchRow()) {
$tu[$rec['team_id']] = $rec['id'];
}
//Useless relations between teems
$toRemove = array_diff(array_keys($tu), array_keys($data));
if (!empty($toRemove)) {
$this->db->Execute("\n DELETE FROM `account_team_users`\n WHERE `user_id` = ?\n AND `team_id` IN (" . rtrim(str_repeat("?,", count($toRemove)), ',') . ")\n ", array_merge(array($userId), $toRemove));
}
foreach ($data as $teamId => $roles) {
if (empty($roles)) {
$roles = array();
}
if (!isset($tu[$teamId])) {
//Relation between user and team has to be created
$this->db->Execute("\n INSERT IGNORE `account_team_users` (`user_id`, `team_id`) VALUES (?, ?)\n ", array($userId, $teamId));
$tu[$teamId] = $this->db->Insert_ID();
$tua = array();
} else {
$tua = array_map(function ($value) {
return $value['account_role_id'];
}, $this->db->GetAll("\n SELECT account_role_id FROM `account_team_user_acls` WHERE `account_team_user_id` = ?\n ", array($tu[$teamId])));
}
//Unnecessary relations with roles
$toRemove = array_diff($tua, array_values($roles));
if (!empty($toRemove)) {
$this->db->Execute("\n DELETE FROM `account_team_user_acls`\n WHERE `account_team_user_id` = ?\n AND `account_role_id` IN (" . rtrim(str_repeat("?,", count($toRemove)), ',') . ")\n ", array_merge(array($tu[$teamId]), $toRemove));
}
if ($c = count($roles)) {
//INSERT-SELECT approach avoids missing foreign keys assertions
$this->db->Execute("\n INSERT IGNORE `account_team_user_acls` (`account_team_user_id`, `account_role_id`)\n SELECT '" . $tu[$teamId] . "', `account_role_id` FROM `acl_account_roles`\n WHERE `account_role_id` IN (" . rtrim(str_repeat("?,", $c), ',') . ")\n " . (!empty($accountId) ? " AND `account_id` = " . intval($accountId) : "") . "\n ", array_values($roles));
}
}
}
private function loadRecords()
{
$this->records = $this->db->GetAll("SELECT * FROM dns_zone_records WHERE zone_id=?", array($this->id));
}
function handleWork($farmId)
{
$DBFarm = DBFarm::LoadByID($farmId);
$GLOBALS["SUB_TRANSACTIONID"] = abs(crc32(posix_getpid() . $farmId));
$GLOBALS["LOGGER_FARMID"] = $farmId;
if ($DBFarm->Status != FARM_STATUS::RUNNING) {
$this->logger->warn("[FarmID: {$DBFarm->ID}] Farm terminated. There is no need to scale it.");
return;
}
foreach ($DBFarm->GetFarmRoles() as $DBFarmRole) {
for ($i = 0; $i < 10; $i++) {
if ($DBFarmRole->NewRoleID != '') {
$this->logger->warn("[FarmID: {$DBFarm->ID}] Role '{$DBFarmRole->GetRoleObject()->name}' being synchronized. This role will not be scalled.");
continue 2;
}
if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_ENABLED) != '1' && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB) && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ) && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::VPC_ROUTER)) {
$this->logger->info("[FarmID: {$DBFarm->ID}] Scaling disabled for role '{$DBFarmRole->GetRoleObject()->name}'. Skipping...");
continue 2;
}
// Get polling interval in seconds
$polling_interval = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_POLLING_INTERVAL) * 60;
$dt_last_polling = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_LAST_POLLING_TIME);
if ($dt_last_polling && $dt_last_polling + $polling_interval > time() && $i == 0) {
$this->logger->info("Polling interval: every {$polling_interval} seconds");
continue;
}
// Set Last polling time
$DBFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_LAST_POLLING_TIME, time(), DBFarmRole::TYPE_LCL);
// Get current count of running and pending instances.
$this->logger->info(sprintf("Processing role '%s'", $DBFarmRole->GetRoleObject()->name));
$scalingManager = new Scalr_Scaling_Manager($DBFarmRole);
$scalingDecision = $scalingManager->makeScalingDecition();
if ($scalingDecision == Scalr_Scaling_Decision::STOP_SCALING) {
return;
}
if ($scalingDecision == Scalr_Scaling_Decision::NOOP) {
continue 2;
} elseif ($scalingDecision == Scalr_Scaling_Decision::DOWNSCALE) {
/*
Timeout instance's count decrease. Decreases instance�s count after scaling
resolution the spare instances are running�g for selected timeout interval
from scaling EditOptions
*/
// We have to check timeout limits before new scaling (downscaling) process will be initiated
if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_DOWNSCALE_TIMEOUT_ENABLED)) {
// if the farm timeout is exceeded
// checking timeout interval.
$last_down_scale_data_time = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_DOWNSCALE_DATETIME);
$timeout_interval = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_DOWNSCALE_TIMEOUT);
// check the time interval to continue scaling or cancel it...
if (time() - $last_down_scale_data_time < $timeout_interval * 60) {
// if the launch time is too small to terminate smth in this role -> go to the next role in foreach()
Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Waiting for downscaling timeout on farm %s, role %s", $DBFarm->Name, $DBFarmRole->GetRoleObject()->name)));
continue 2;
}
}
// end Timeout instance's count decrease
$sort = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_KEEP_OLDEST) == 1 ? 'DESC' : 'ASC';
$servers = $this->db->GetAll("SELECT server_id FROM servers WHERE status = ? AND farm_roleid=? ORDER BY dtadded {$sort}", array(SERVER_STATUS::RUNNING, $DBFarmRole->ID));
$got_valid_instance = false;
// Select instance that will be terminated
//
// * Instances ordered by uptime (oldest wil be choosen)
// * Instance cannot be mysql master
// * Choose the one that was rebundled recently
while (!$got_valid_instance && count($servers) > 0) {
$item = array_shift($servers);
$DBServer = DBServer::LoadByID($item['server_id']);
if ($DBServer->GetFarmRoleObject()->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::RABBITMQ)) {
$serversCount = count($DBServer->GetFarmRoleObject()->GetServersByFilter(array(), array('status' => array(SERVER_STATUS::TERMINATED, SERVER_STATUS::TROUBLESHOOTING))));
if ($DBServer->index == 1 && $serversCount > 1) {
continue;
}
}
if ($DBServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED)) {
continue;
}
// Exclude db master
if ($DBServer->GetProperty(SERVER_PROPERTIES::DB_MYSQL_MASTER) != 1 && $DBServer->GetProperty(Scalr_Db_Msr::REPLICATION_MASTER) != 1) {
// We do not want to delete the most recently synced instance. Because of LA fluctuation.
// I.e. LA may skyrocket during sync and drop dramatically after sync.
if ($DBServer->dateLastSync != 0) {
$chk_sync_time = $this->db->GetOne("\n SELECT server_id FROM servers\n WHERE dtlastsync > {$DBServer->dateLastSync}\n AND farm_roleid='{$DBServer->farmRoleId}'\n AND status NOT IN('" . SERVER_STATUS::TERMINATED . "', '" . SERVER_STATUS::TROUBLESHOOTING . "')\n LIMIT 1\n ");
if ($chk_sync_time) {
$got_valid_instance = true;
}
} else {
$got_valid_instance = true;
}
}
}
if ($DBServer && $got_valid_instance) {
$this->logger->info(sprintf("Server '%s' selected for termination...", $DBServer->serverId));
$allow_terminate = false;
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
$aws = $DBServer->GetEnvironmentObject()->aws($DBServer);
// Shutdown an instance just before a full hour running
if (!$DBServer->GetFarmRoleObject()->GetSetting(DBFarmRole::SETTING_SCALING_IGNORE_FULL_HOUR)) {
$response = $aws->ec2->instance->describe($DBServer->GetProperty(EC2_SERVER_PROPERTIES::INSTANCE_ID))->get(0);
if ($response && count($response->instancesSet)) {
$launch_time = $response->instancesSet->get(0)->launchTime->getTimestamp();
$time = 3600 - (time() - $launch_time) % 3600;
// Terminate instance in < 10 minutes for full hour.
if ($time <= 600) {
$allow_terminate = true;
} else {
$timeout = round(($time - 600) / 60, 1);
Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Farm %s, role %s scaling down. Server '%s' will be terminated in %s minutes. Launch time: %s", $DBFarm->Name, $DBServer->GetFarmRoleObject()->GetRoleObject()->name, $DBServer->serverId, $timeout, $response->instancesSet->get(0)->launchTime->format('c'))));
}
}
} else {
$allow_terminate = true;
}
//Releases memory
$DBServer->GetEnvironmentObject()->getContainer()->release('aws');
unset($aws);
} else {
$allow_terminate = true;
}
if ($allow_terminate) {
//Check safe shutdown
if ($DBServer->GetFarmRoleObject()->GetSetting(DBFarmRole::SETTING_SCALING_SAFE_SHUTDOWN) == 1) {
if ($DBServer->IsSupported('0.11.3')) {
try {
$port = $DBServer->GetProperty(SERVER_PROPERTIES::SZR_API_PORT);
if (!$port) {
$port = 8010;
}
$szrClient = Scalr_Net_Scalarizr_Client::getClient($DBServer, Scalr_Net_Scalarizr_Client::NAMESPACE_SYSTEM, $port);
$res = $szrClient->callAuthShutdownHook();
} catch (Exception $e) {
$res = $e->getMessage();
}
} else {
Logger::getLogger(LOG_CATEGORY::FARM)->error(new FarmLogMessage($DBFarm->ID, sprintf("Safe shutdown enabled, but not supported by scalarizr installed on server '%s'. Ignoring.", $DBServer->serverId)));
}
if ($res != '1') {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Safe shutdown enabled. Server '%s'. Script returned '%s', server won't be terminated while return value not '1'", $DBServer->serverId, $res)));
break;
}
}
try {
$DBServer->terminate('SCALING_DOWN', false);
$DBFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_DOWNSCALE_DATETIME, time(), DBFarmRole::TYPE_LCL);
Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Farm %s, role %s scaling down. Server '%s' marked as 'Pending terminate' and will be fully terminated in 3 minutes.", $DBFarm->Name, $DBServer->GetFarmRoleObject()->GetRoleObject()->name, $DBServer->serverId)));
} catch (Exception $e) {
$this->logger->fatal(sprintf("Cannot terminate %s: %s", $DBFarm->ID, $DBServer->serverId, $e->getMessage()));
}
}
} else {
$this->logger->warn(sprintf("[FarmID: %s] Scalr unable to determine what instance it should terminate (FarmRoleID: %s). Skipping...", $DBFarm->ID, $DBFarmRole->ID));
}
break;
} elseif ($scalingDecision == Scalr_Scaling_Decision::UPSCALE) {
/*
Timeout instance's count increase. Increases instance's count after
scaling resolution �need more instances� for selected timeout interval
from scaling EditOptions
*/
if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_UPSCALE_TIMEOUT_ENABLED)) {
// if the farm timeout is exceeded
// checking timeout interval.
$last_up_scale_data_time = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_UPSCALE_DATETIME);
$timeout_interval = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_UPSCALE_TIMEOUT);
// check the time interval to continue scaling or cancel it...
if (time() - $last_up_scale_data_time < $timeout_interval * 60) {
// if the launch time is too small to terminate smth in this role -> go to the next role in foreach()
Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Waiting for upscaling timeout on farm %s, role %s", $DBFarm->Name, $DBFarmRole->GetRoleObject()->name)));
continue 2;
}
}
// end Timeout instance's count increase
//Check DBMsr. Do not start slave during slave2master process
$isDbMsr = $DBFarmRole->GetRoleObject()->getDbMsrBehavior();
if ($isDbMsr) {
if ($DBFarmRole->GetSetting(Scalr_Db_Msr::SLAVE_TO_MASTER)) {
$runningServers = $DBFarmRole->GetRunningInstancesCount();
if ($runningServers > 0) {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Role is in slave2master promotion process. Do not launch new slaves while there is no active slaves")));
continue 2;
} else {
$DBFarmRole->SetSetting(Scalr_Db_Msr::SLAVE_TO_MASTER, 0, DBFarmRole::TYPE_LCL);
}
}
}
if ($DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_ONE_BY_ONE) == 1) {
$pendingInstances = $DBFarmRole->GetPendingInstancesCount();
if ($pendingInstances > 0) {
Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("There are %s pending intances of %s role on % farm. Waiting...", $pendingInstances, $DBFarmRole->GetRoleObject()->name, $DBFarm->Name)));
continue 2;
}
}
$fstatus = $this->db->GetOne("SELECT status FROM farms WHERE id=? LIMIT 1", array($DBFarm->ID));
if ($fstatus != FARM_STATUS::RUNNING) {
$this->logger->warn("[FarmID: {$DBFarm->ID}] Farm terminated. There is no need to scale it.");
return;
}
$ServerCreateInfo = new ServerCreateInfo($DBFarmRole->Platform, $DBFarmRole);
try {
$DBServer = Scalr::LaunchServer($ServerCreateInfo, null, false, "Scaling up");
$DBFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_UPSCALE_DATETIME, time(), DBFarmRole::TYPE_LCL);
Logger::getLogger(LOG_CATEGORY::FARM)->info(new FarmLogMessage($DBFarm->ID, sprintf("Farm %s, role %s scaling up. Starting new instance. ServerID = %s.", $DBFarm->Name, $DBServer->GetFarmRoleObject()->GetRoleObject()->name, $DBServer->serverId)));
} catch (Exception $e) {
Logger::getLogger(LOG_CATEGORY::SCALING)->error($e->getMessage());
}
}
}
}
}
