function modify_hostgroup_in_db($final_array)
{
global $server_cfg, $game;
foreach ($final_array as $regex => $config) {
if (!isset($config["class"])) {
$final_array[$regex] = "";
}
}
$conf_all = parse_ini_file($server_cfg["rs_conf_file"], true);
$conf = $conf_all["DB"];
$db_server = $conf["host"];
$db_user = $conf["user"];
$db_pass = $conf["password"];
$db_name = $conf["database"];
$mysql_pdo = new PDO("mysql:host={$db_server};dbname={$db_name}", $db_user, $db_pass);
if (!$mysql_pdo) {
print "Failed to create new mysql PDO\n";
return 1;
}
$game_cfg = load_game_config($game);
$deploy_id = $game_cfg["deployIDs"][0];
foreach ($final_array as $regex => $metric) {
$mysql_regex = str_replace(".*", "%", $regex);
$hostgroup = $metric['hostgroup'];
$query = "update instances set hostgroup='{$hostgroup}' where hostname like '{$mysql_regex}' and deploy_id={$deploy_id}";
$stmt = $mysql_pdo->prepare($query);
$stmt->execute();
}
}
function main($server_cfg)
{
$options = get_options();
if (isset($options['g']) && $options['g'] !== '') {
$game_names = explode(",", $options['g']);
} else {
$game_names = $server_cfg['game_list'];
}
foreach ($game_names as $game) {
zpm_preamble($game);
$game_cfg = load_game_config($game);
$retval = null;
// refs will start failing in 5.3.x if not declared
$cleanup = "/usr/local/zperfmon/bin/clean.sh -g " . $game_cfg['name'] . " > /dev/null ";
$output = system($cleanup, $retval);
if ($retval != 0) {
error_log("Couldn`t cleanup game {$game}", 3, sprintf($server_cfg['log_file'], $game_cfg['name']));
continue;
}
$arrays_to_id = get_array_id_map($server_cfg, $game_cfg);
foreach ($arrays_to_id as $array => $id) {
$game_cfg = load_game_config($game, $id);
$cleanup = "/usr/local/zperfmon/bin/clean.sh -g " . $game_cfg['name'] . " > /dev/null ";
$output = system($cleanup, $retval);
if ($retval != 0) {
error_log("Couldn`t cleanup game {$game}", 3, sprintf($server_cfg['log_file'], $game_cfg['name']));
continue;
}
}
zpm_postamble($game);
}
}
function main($server_cfg)
{
if (isset($_GET['game'])) {
$game = $_GET['game'];
} else {
echo "Game name is invalid or not set \n";
exit;
}
$game_cfg = load_game_config($game);
$deploy_id = $game_cfg["deployIDs"][0];
# Getting the distinct hostgroups for the inputed game
$conf_all = parse_ini_file($server_cfg["rs_conf_file"], true);
$conf = $conf_all["DB"];
$db_server = $conf["host"];
$db_user = $conf["user"];
$db_pass = $conf["password"];
$db_name = $conf["database"];
$mysql_pdo = new PDO("mysql:host={$db_server};dbname={$db_name}", $db_user, $db_pass);
if (!$mysql_pdo) {
print "Failed to create new mysql PDO\n";
return 1;
}
$regex = $_GET["regex"];
$regex = str_replace(".*", '%', $regex);
$query = "select hostname from instances where hostname like '{$regex}' and deploy_id={$deploy_id}";
$stmt = $mysql_pdo->prepare($query);
$stmt->execute();
$hosts = $stmt->fetchAll(PDO::FETCH_ASSOC);
$count_hostnames = array();
foreach ($hosts as $value) {
$count_hostnames["hosts"][] = $value["hostname"];
}
$count_hostnames["count"] = count($count_hostnames["hosts"]);
echo json_encode($count_hostnames);
}
function extract_functions($files, $game_name, $run_id, $aggregate_dir)
{
// XXX: xhprof depends on this global for doing entry counts
global $display_calls;
$display_calls = true;
$functions_to_extract = array("MC::set", "MC::get", "ApcManager::get", "ApcManager::set", "serialize", "unserialize", "AMFBaseSerializer::serialize", "AMFBaseDeserializer::deserialize");
if ($game_cfg = load_game_config($game_name)) {
//$game_cfg = $game_cfg[$game_name];
if (isset($game_cfg["tracked_functions"])) {
$functions_to_extract = $game_cfg["tracked_functions"];
}
}
$prof_obj = new XHProfRuns_Default();
$runs = xhprof_aggregate_runs_list($prof_obj, $files);
if ($runs['raw']) {
$aggregate = $runs['raw'];
} else {
return;
}
# keep this in sync with the aggregate_files
# $aggregate_file_name = "{$run_id}.xhprof";
$overall_totals = null;
$flattened_profile = xhprof_compute_flat_info($aggregate, $overall_totals);
$interesting_funcs = extract_interesting_functions($flattened_profile, $functions_to_extract);
$top_funcs = get_top_functions($flattened_profile, 5);
$xhprof = array("interesting" => $interesting_funcs, "top functions" => $top_funcs, "files" => $files);
file_put_contents("{$aggregate_dir}/{$run_id}.extract", serialize($xhprof));
}
function get_pie($game_name, $period = "day", $tstamp = null, $metrics = array("excl_wt" => "Exclusive Wall time", "excl_cpu" => "Exclusive CPU time"), $how_many = 6, $profile = null)
{
global $server_cfg;
$game_cfg = load_game_config($game_name);
//$game_cfg = $game_cfg["$game_name"];
if ($profile != null) {
$period = "absolute";
}
switch ($period) {
case "day":
$prof_path = sprintf($server_cfg["daily_profile"], $game_cfg["name"]);
break;
case "30min":
$prof_path = sprintf($server_cfg["daily_upload_directory"], $game_cfg["name"]);
break;
case "absolute":
break;
default:
echo "Unsupported period \\'{$period}\\' for get_pie()";
error_log("Unsupported period \\'{$period}\\' for get_pie()");
return null;
}
if ($period != "absolute") {
// Find all aggregate profiles in dir and take the last
$profile_list = glob("{$prof_path}/*all.xhprof");
if (empty($profile_list)) {
echo "no profile in {$period} {$prof_path}/*.all.xhprof";
return null;
}
$profile = end($profile_list);
}
return get_top_functions($profile, $metrics, $how_many);
}
function main($server_cfg)
{
$options = get_options($server_cfg);
foreach ($options['games'] as $game) {
zpm_preamble($game);
$game_cfg = load_game_config($game);
if (!$game_cfg) {
error_log("Daily job: Failed to load config for {$game}\n", 3, sprintf($server_cfg['log_file'], $game));
continue;
}
// log_file and logger object are used by all child functions.
$game_cfg['log_file'] = sprintf($server_cfg['log_file'], $game);
$game_cfg['logger'] = new Logger($server_cfg, $game_cfg);
$job = $options['job'];
if ($job == "both" or $job == "daily") {
process_daily($server_cfg, $game_cfg, $options['timeslot'], $options['period']);
}
//performance report
/*
if ($job == "both" or $job == "report") {
generate_report($server_cfg, $game_cfg,
$options['timeslot'], $options['period']);
}*/
zpm_postamble($game);
}
}
function main($server_cfg)
{
$options = get_options($server_cfg);
foreach ($options['games'] as $game) {
zpm_preamble($game);
$game_cfg = load_game_config($game);
if (!$game_cfg) {
error_log("Daily job: Failed to load config for {$game}\n", 3, sprintf($server_cfg['log_file'], $game));
continue;
}
// log_file and logger object are used by all child functions.
$game_cfg['log_file'] = sprintf($server_cfg['log_file'], $game);
$game_cfg['logger'] = new Logger($server_cfg, $game_cfg);
$job = $options['job'];
//instance detail report per game
if ($job == "both" or $job == "report") {
InstanceReport::generate_detail_report($server_cfg, $game_cfg, $options['timeslot'], $options['period']);
}
zpm_postamble($game);
}
//instance summary report for all games
$job = $options['job'];
if ($job == "both" or $job == "report") {
InstanceReport::generate_summary_report($server_cfg, $options['timeslot'], $options['period']);
}
}
public function get_config_file($game)
{
$game_cfg = load_game_config($game);
if (isset($game_cfg['cloud_name']) && $game_cfg['cloud_name'] == 'zcloud') {
return $this->server_cfg['common_config_zcloud_file'];
} else {
return $this->server_cfg['common_config_file'];
}
}
function scan_game_cfgs($server_cfg)
{
$map = array();
foreach ($server_cfg["game_list"] as $game) {
$gcfg = load_game_config($game);
$deploy_id = $gcfg['deployIDs'][0];
$map[$deploy_id] = $game;
}
return $map;
}
private function get_ips()
{
$this->game_cfg = load_game_config($this->game);
$deploy_id = $this->game_cfg['deployIDs'][0];
$query = "select array_id, private_ip from instances where deploy_id={$deploy_id} and array_id != 0";
$stmt = $this->prepare($query);
if ($stmt) {
$result = $this->fetchAll($stmt, array());
foreach ($result as $item) {
$array_id = $item['array_id'];
$this->ip_list[$array_id][] = $item['private_ip'];
}
}
}
function splitarraywise($server_cfg, $ip_list, $game_name, $time_slot)
{
$game_cfg = load_game_config($game_name);
$rs = new RightScale($server_cfg, $game_cfg);
$mapping = $rs->get_array_ip_mapping();
$return = array();
foreach ($ip_list as $ip) {
$dir_base_path = sprintf($server_cfg["root_upload_directory"], $game_name);
$base_ip_dir = $dir_base_path . $time_slot . "/" . $server_cfg['profile_upload_directory'];
$ip_dir = $base_ip_dir . "/" . $ip;
//$filelist = scandir($ip_dir);
$array_id = $mapping[$ip];
// To prevent game_ folder to be created due to 5 minute processing
if (!isset($array_id) || $array_id == '') {
continue;
}
$return[$array_id][] = $ip;
$dir_base_path_array = sprintf($server_cfg["root_upload_directory"], $game_name . "_" . $array_id);
//creating the timeslot directory
if (!is_dir($dir_base_path_array . $time_slot)) {
mkdir($dir_base_path_array . $time_slot, 0777, true);
}
//creating the xhprof directory
if (!is_dir($dir_base_path_array . $time_slot . "/" . $server_cfg['profile_upload_directory'])) {
mkdir($dir_base_path_array . $time_slot . "/" . $server_cfg['profile_upload_directory'], 0777, true);
}
$base_ip_dir_array = $dir_base_path_array . $time_slot . "/" . $server_cfg['profile_upload_directory'];
$ip_directory_array = $base_ip_dir_array . "/" . $ip;
// Check if symlink exists already. In case of frequent call of the function in a timeslot
if (!is_dir($ip_directory_array)) {
symlink($ip_dir, $ip_directory_array);
}
if (!file_exists($base_ip_dir_array . "/" . ".profiles")) {
// put the ip uploading ip addresses which should be segragated while massaging.
file_put_contents("{$base_ip_dir_array}/.profiles", $ip, FILE_APPEND | LOCK_EX);
//touch(($base_ip_dir_array."/".".profiles"));
error_log(".profiles created" . "\n" . sprintf($server_cfg['log_file'], $game_cfg['name']));
}
if (!file_exists($base_ip_dir_array . "/" . ".slowpages")) {
touch($base_ip_dir_array . "/" . ".slowpages");
error_log(".slowpages created" . "\n" . sprintf($server_cfg['log_file'], $game_cfg['name']));
}
if (!file_exists($base_ip_dir_array . "/" . ".apache_stats")) {
touch($base_ip_dir_array . "/" . ".apache_stats");
error_log(".apache_stats created" . "\n" . sprintf($server_cfg['log_file'], $game_cfg['name']));
}
}
return $return;
}
public function __construct($server_cfg, $game, $slack_now = false)
{
$cfg = get_eu_cfg($server_cfg['eu_conf_file']);
$db_server = $cfg["db_host"];
$db_user = $cfg["db_user"];
$db_pass = $cfg["db_pass"];
$db_name = $cfg["db_name"];
$this->table = $cfg["table"];
$this->game = $game;
$this->game_cfg = load_game_config($game);
$this->server_cfg = $server_cfg;
parent::__construct($db_server, $db_user, $db_pass, $db_name);
$this->max_metric = $this->get_max_metric($slack_now);
$this->rs = new Rightscale($server_cfg, $game);
}
function main($server_cfg)
{
$options = get_options();
if (isset($options['g']) && $options['g'] !== '') {
$game_names = explode(",", $options['g']);
} else {
$game_names = $server_cfg['game_list'];
}
foreach ($game_names as $game_name) {
zpm_preamble($game_name);
try {
$game_cfg = load_game_config($game_name);
//$game_cfg = $game_cfg[$game_name];
$now = time();
$query_period = $now - 5 * 60;
// 5 minutes ago
if (isset($game_cfg['splunk_url'])) {
$splunk_url = $game_cfg['splunk_url'];
$splunk_user = $game_cfg['splunk_user'];
$splunk_pass = $game_cfg['splunk_password'];
$splunk_queries = $game_cfg['splunk_queries'];
$splunk_collector = new SplunkCollector($splunk_url, $splunk_user, $splunk_pass);
$splunk_store = new SplunkAdapter($game_cfg);
foreach ($splunk_queries as $query_name => $query) {
$results = $splunk_collector->run_query($query, $query_period);
if (is_array($results)) {
$count = $results[0]["count"];
$splunk_store->store_splunk_count($now, $query_name, $count);
} else {
echo "Could not fetch {$query_name} for {$game_name}\n";
}
}
}
} catch (Exception $e) {
}
zpm_postamble($game_name);
}
}
function main($server_cfg)
{
$options = getopt("g:");
$game = $options['g'];
$game_cfg = load_game_config($game);
$hostConfigObj = new HostgroupConfig($server_cfg, $game);
$rsObj = new RightScale($server_cfg, $game_cfg);
$array_id_name = array_values($rsObj->get_array_to_arrayid_mapping());
$config_col_names = $hostConfigObj->get_config_column_names();
$xhprofModelObj = new XhprofModel($server_cfg, $game_cfg, false);
// add columns for both the tables
$tables = array("vertica_stats_30min", "vertica_stats_daily");
echo "adding columns for {$game}:\n";
foreach ($tables as $table) {
$result = add_missing_columns($config_col_names, $table, $xhprofModelObj);
}
foreach ($array_id_name as $array_id) {
echo "adding columns for {$game}:{$array_id}:\n";
$game_cfg = load_game_config($game, $array_id);
$xhprofModelObj = new XhprofModel($server_cfg, $game_cfg, false);
foreach ($tables as $table) {
$result = add_missing_columns($config_col_names, $table, $xhprofModelObj);
}
}
}
function move_slow_page($server_cfg, $game_name, $time_slot = '*', &$top5)
{
$output = array();
$root_upload_directory = sprintf($server_cfg['root_upload_directory'], $game_name);
//checking if the input game_name corresponds to a array of a parent game
try {
$game_cfg = array();
if (!in_array($game_name, $server_cfg["game_list"])) {
/*$game_split = split("_", $game_name);
$game_name_array = array();
for ( $counter = 0; $counter < count($game_split)-1; $counter ++) {
array_push($game_name_array, $game_split[$counter]);
}
$game_name = implode($game_name_array);
$array_id = $game_split[count($game_split)-1];*/
$last_tok_index = strrpos($game_name, "_");
$array_id = substr($game_name, $last_tok_index + 1, strlen($game_name) - $last_tok_index);
$game_name = substr($game_name, 0, $last_tok_index);
$game_cfg = load_game_config($game_name, $array_id);
} else {
$game_cfg = load_game_config($game_name);
}
$game_name = $game_cfg["name"];
if (!$game_cfg) {
error_log("configuration for " . $game_name . " is not loaded\n", 3, sprintf($server_cfg['log_file'], $game_cfg['name']));
return $output;
}
} catch (Exception $e) {
error_log("configuration loading for {$game_name} failed\n", 3, sprintf($server_cfg['log_file'], $game_cfg['name']));
error_log("Exception says: " . $e->getMessage() . "\n", 3, sprintf($server_cfg['log_file'], $game_cfg['name']));
}
$slowpage_markers = glob("{$root_upload_directory}{$time_slot}/" . $server_cfg['profile_upload_directory'] . "/.slowpages", GLOB_BRACE);
if (empty($slowpage_markers)) {
error_log("No slowpage for {$time_slot}\n", 3, sprintf($server_cfg['log_file'], $game_name));
return $output;
}
foreach ($slowpage_markers as $marker) {
$profile_upload_directory = dirname($marker);
$timestamp = (int) (basename(dirname(dirname($marker))) * 1800);
error_log("slow pages for {$profile_upload_directory}\n", 3, sprintf($server_cfg['log_file'], $game_name));
foreach (glob("{$profile_upload_directory}/*php") as $page_dir) {
if (!is_dir($page_dir)) {
continue;
}
scan($server_cfg, $game_name, $game_cfg["slow_page_threshold"], $page_dir, $time_slot, $output, $top5);
}
}
error_log(sprintf("in move_slow_page top5 list has %d lines\n", count($top5)), 3, sprintf($server_cfg['log_file'], $game_name));
return $output;
}
function main($server_cfg)
{
$options = get_options();
if (isset($options['g']) && $options['g'] !== '') {
$game_names = explode(",", $options['g']);
} else {
$game_names = $server_cfg['game_list'];
}
$time_slots = null;
if (!empty($options['t'])) {
$time_slots = $options['t'];
}
$timestamp = $_SERVER['REQUEST_TIME'];
$current_time_slot = (int) ($timestamp / (30 * 60));
foreach ($game_names as $game_name) {
zpm_preamble($game_name);
// MODIFIED!!!! Create and mount next two timeslots !!!
$game_cfg = load_game_config($game_name);
if (!$game_cfg) {
error_log("configuration for " . $game_name . " is not loaded\n", 3, sprintf($server_cfg['log_file'], $game_cfg['name']));
continue;
}
$target_dir = sprintf($server_cfg['root_upload_directory'], $game_cfg["name"]);
$target_dir = $target_dir . "/%s/" . $server_cfg["profile_upload_directory"];
$dir_name_array = create_directory($target_dir, $current_time_slot, $server_cfg, $game_cfg);
if ($dir_name_array === null) {
error_log("Directory creation failed for the game " . $game_cfg['name'], 3, sprintf($server_cfg['log_file'], $game_cfg['name']));
continue;
}
if ($dir_name_array === 0) {
error_log("No Ram disk Exist!!!" . $game_cfg['name'], 3, sprintf($server_cfg['log_file'], $game_cfg['name']));
}
//creating new games and getting the list of web arrays to process data for
$rs = new RightScale($server_cfg, load_game_config($game_name));
$rs->make_array_games();
$array_ids = $rs->get_arrays_to_serve();
//
// Wrap processing for each game inside a file lock
//
$game_lock_file = sprintf($server_cfg["zperfmon_lock_file"], $game_name);
$game_lock_handle = grab_lock($game_lock_file, $server_cfg, $game_cfg);
if (!$game_lock_handle) {
error_log("Could not get exclusive lock for \"{$game_name}\"\n", 3, sprintf($server_cfg['log_file'], $game_cfg['name']));
delete_ramdisk_data($game_name, $time_slots, $timestamp, $server_cfg, $game_cfg);
continue;
}
//processing parent game
try {
//
// process uploaded data
//
run_cron_for_game($server_cfg, $game_cfg, $time_slots);
} catch (Exception $e) {
error_log("Upload processing for {$game_name} failed\n", 3, sprintf($server_cfg['log_file'], $game_cfg['name']));
error_log("Exception says: " . $e->getMessage() . "\n", 3, sprintf($server_cfg['log_file'], $game_cfg['name']));
}
// loop to process data for each web array
foreach ($array_ids as $array) {
try {
$game_cfg = load_game_config($game_name, $array);
if (!$game_cfg) {
error_log("configuration for " . $game_name . " is not loaded\n", 3, sprintf($server_cfg['log_file'], $game_cfg['name']));
continue;
}
//
// process uploaded data
//
run_cron_for_game($server_cfg, $game_cfg, $time_slots);
} catch (Exception $e) {
error_log("Upload processing for {$game_name} failed\n", 3, sprintf($server_cfg['log_file'], $game_cfg['name']));
error_log("Exception says: " . $e->getMessage() . "\n", 3, sprintf($server_cfg['log_file'], $game_cfg['name']));
}
}
// compress for parent game
// This implementation forces us to always pass a timeslot
// while calling this script. otherwise compression will fail.
$game_cfg = load_game_config($game_name);
compress_unziped_profiles($server_cfg, $game_cfg, $time_slots);
// Now compress the array games
foreach ($array_ids as $array) {
$game_cfg = load_game_config($game_name, $array);
compress_unziped_profiles($server_cfg, $game_cfg, $time_slots);
}
// cleanup the lock
drop_lock($game_lock_handle, $game_lock_file);
zpm_postamble($game_name);
//MODIFIED!!!!
if ($dir_name_array != 0) {
delete_ramdisk_data($game_name, $time_slots, $timestamp, $server_cfg, $game_cfg);
}
}
}
}
$profile = end($profile_list);
}
/* now create an array whih contains data,
* for all the pages and return an object,
* having page name as key and data as their value
*/
$top_fn = array();
$page_data = array();
if ($pages != null) {
foreach ($pages as $page) {
$profile_list = glob("{$prof_path}/*{$page}.xhprof");
if (empty($profile_list)) {
//echo "no profile in $period $prof_path/*.all.xhprof";
continue;
}
$profile = end($profile_list);
$page_data[$page] = get_top_functions($profile, $metrics, $how_many);
$top_fn = array_merge($top_fn, $page_data);
}
return $top_fn;
} else {
return get_top_functions($profile, $metrics, $how_many);
}
}
//$test_mode = true;
if (isset($test_mode) && $test_mode) {
$game_cfg = load_game_config("fluid");
//$game_cfg = $game_cfg["fluid"];
print_r(get_pie($server_cfg, $game_cfg));
}
function partition_dynamics($game_names, $server_cfg, $con, $log_file)
{
//Month to Partition map
$partition_drop_map = array('p001', 'p002', 'p003', 'p004', 'p005', 'p006', 'p001', 'p002', 'p003', 'p004', 'p005', 'p006');
//Current month to retention month p_tag map.
$cursor_retain_months = array(11, 12, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10);
$month_index = date('m') - 1;
$array_drop_index = $month_index - 2;
//If array drop index is negative go to the end of array
if ($array_drop_index < 0) {
$array_drop_index = 12 + $array_drop_index;
}
$partition_drop_array_games = $partition_drop_map[$array_drop_index];
foreach ($game_names as $game) {
$db = "zprf_" . $game;
report_partitions_state($db, $con, $log_file);
$cursor_retain = $cursor_retain_months[$month_index];
$cursor_delete = 0 - $cursor_retain;
//Deleting rows in the ratio 1:4
delete_rows($cursor_delete, $db, $con, $log_file);
//Doing an alter table
crunch_space($db, $con, $log_file);
//Fetching the retention time from game_cfg file
$game_cfg = load_game_config($game);
$retention_time = $game_cfg["xhprof_retention_time"];
//Maximum retention time = 4 months
//If retention time not specified setting the default retention time as 4 months
if ($retention_time == NULL or $retention_time > 4) {
$retention_time = 4;
}
$partition_drop_index = $month_index - $retention_time - 1;
if ($partition_drop_index < 0) {
$partition_drop_index = 12 + $partition_drop_index;
}
$partition_drop = $partition_drop_map[$partition_drop_index];
$rs = new RightScale($server_cfg, load_game_config($game));
$array_ids = $rs->get_arrays_to_serve();
drop_partition_data($partition_drop, $partition_drop_array_games, $array_ids, $db, $con, $log_file);
}
}
break;
case 'POST':
$game = $_POST["game"];
$arrayid = @$_POST["array"];
break;
default:
$game = null;
$arrayid = null;
break;
}
} else {
$options = getopt("g:");
if (isset($options["g"])) {
$game = $options["g"];
}
}
} catch (Exception $e) {
$game = null;
}
if (!isset($game) || !$game) {
echo "<body>Error: expected a game name parameter, eg.: <b>\"?game=fish\"</b></body></html>";
exit(1);
}
if (isset($arrayid) && $arrayid == "all") {
$arrayid = null;
}
$game_cfg = load_game_config($game, $arrayid);
if (!$game_cfg) {
echo "<body>Error: invalid game name: <b>\"{$game}\"</b></body></html>";
exit(1);
}
function insert_event($xhProfModelObject, $server_cfg = null, $game_cfg = null, $timestamp, $type, $text)
{
if (!$xhProfModelObject) {
return false;
}
$query_name = "event_insert";
$result = null;
$result = $xhProfModelObject->generic_execute_get_query($query_name, array('start' => $timestamp, 'type' => $type, 'text' => $text), false, false);
//if its a array, just return, else if its a game ,fill the array dbs as well
if (isset($game_cfg['parent'])) {
//its an array, so our work here is done, return
return $result;
}
$result_array = null;
$arrays = get_array_id_map($server_cfg, $game_cfg);
//for loop to put events in each array db
foreach ($arrays as $array => $array_id) {
$game_array_cfg = load_game_config($game_cfg['name'], $array_id);
$xhProfModelObject = new XhProfModel($server_cfg, $game_array_cfg, false);
$result_array = $xhProfModelObject->generic_execute_get_query($query_name, array('start' => $timestamp, 'type' => $type, 'text' => $text), false, false);
}
return $result;
}
public function getCloudIdGame($game)
{
$game_cfg = load_game_config($game);
$deploy_id = $game_cfg['deployIDs'][0];
$result = $this->xhprofModelObject->generic_execute_get_query("get_cloud_id", array('deploy_id' => $deploy_id));
return $result[0]['cloud_id'];
}
function run_upload_processing($server_cfg)
{
date_default_timezone_set('UTC');
$current_timestamp = $_SERVER['REQUEST_TIME'];
$options = getopt("g:t:a");
if (isset($options['g']) && $options['g'] !== '') {
$game_names = explode(",", $options['g']);
} else {
$game_names = $server_cfg['game_list'];
}
$time_slots = null;
if (empty($options['t'])) {
$time_slots = (int) ($current_timestamp / 1800);
$time_slots = "{" . $time_slots . "}";
} else {
$time_slots = $options['t'];
}
foreach ($game_names as $game_name) {
zpm_preamble($game_name);
$game_cfg = load_game_config($game_name);
// Process each array of the game
$rsObj = new RightScale($server_cfg, $game_cfg);
$rsObj->make_array_games();
process_uploads($server_cfg, $game_cfg, $time_slots, $rsObj);
zpm_postamble($game_name);
}
}
function main($server_cfg)
{
$timestamp = time();
$options = getopt("g:");
if (isset($options['g']) && $options !== '') {
$game_names = explode(",", $options['g']);
} else {
$game_names = $server_cfg['game_list'];
}
foreach ($game_names as $game_name) {
$game_cfg = load_game_config($game_name);
get_rightscale_data($server_cfg, $game_cfg, $timestamp);
}
}
function main($server_cfg)
{
global $mysql_pdo, $pool_name, $game_name, $deregistered_ips;
$relative_path = realpath(dirname(__FILE__));
$options = get_options();
if (isset($options['a']) && $options['a'] !== '') {
$array_id = $options['a'];
} else {
die("Array ID not provided\n");
}
$query = "select distinct hostgroup,deploy_id,private_ip from instances where array_id={$array_id}";
$stmt = $mysql_pdo->prepare($query);
$stmt->execute();
$rows = $stmt->fetchAll(PDO::FETCH_ASSOC);
if ($rows) {
$ip_list = array();
$row = $rows[0];
$pool_name = $row['hostgroup'];
$deploy_id = $row['deploy_id'];
foreach ($rows as $key => $row) {
$ip_list[] = $row['private_ip'];
}
} else {
die("Could not map array id to hostgroup\n");
}
//Fetching the game name from the game_deploy_map.json file
$game_deploy_map = get_game_deploy_map($server_cfg);
if (isset($game_deploy_map[$deploy_id])) {
$game_name = $game_deploy_map[$deploy_id];
} else {
die("Unable to map game deployment id to game name\n");
}
$game_cfg = load_game_config($game_name);
// Getting the ip of loadbalancer to be hit
$f5_credentials = $game_cfg["f5_credentials"][$pool_name];
$f5_vip = $f5_credentials["f5_vip"];
//Fetching the target slack
$target_slack = $game_cfg["target_slack"][$pool_name];
if ($target_slack <= 0) {
die("Unable to proceed as the target slack is less than or equal to zero\n");
}
$pool_data = get_instance_count_slack_offset($game_name, $pool_name);
//Delete any previous rollback file existing
unlink_rollback_file($server_cfg, $game_name, $pool_name);
$instance_count = count($ip_list);
$slack_offset = $pool_data['slack'];
$opt_inst_count = $pool_data['optimal_instance_count'];
//Getting the current slack via api from the zmon machine.
$current_slack = get_current_slack($game_name, $pool_name, $instance_count);
if ($current_slack < $slack_offset) {
die("Unable to proceed as the current slack is less than the 1 day lowest slack.\n");
}
//Setting the target slack relative to the slack offset and finding the optimal instance count for acheving it.
$target_slack = $target_slack + ($current_slack - $slack_offset);
if ($target_slack >= $current_slack) {
die("Unable to proceed as the target slack is greater than current slack\n");
}
//Calculating the optimal instance count for the target slack
$opt_inst_count_target = $target_slack * ($instance_count - $opt_inst_count) / $slack_offset + $opt_inst_count;
$opt_inst_count_target = round($opt_inst_count_target);
if ($opt_inst_count_target <= 0) {
die("Unable to proceed as the optimal instance count required for target slack is less then or equal to 0\n");
}
echo "Current slack = {$current_slack}%\nCurrent Instance Count = {$instance_count}\nTarget slack = {$target_slack}%\nOptimal count for target slack = {$opt_inst_count_target}\n\n";
$iteration_count = 0;
/* FETCH THE INITIAL PRT from all the nodes and store it */
echo "Calculating initial page response times...\n";
$initial_page_pdt_array = get_pdt_per_page($f5_vip, count($ip_list));
$prev_pdt = $initial_page_pdt_array;
foreach ($initial_page_pdt_array as $page => $pdt_data) {
echo "Page Delivery time for page {$page} is " . $pdt_data[2] . " ms\n";
}
echo "\n";
//Fetching the tolerable pdt limit percent from the game cfg file
$pdt_diff_tol_percent = $game_cfg["pdt_diff_tol_percent"];
while ($iteration_count < 5) {
$instance_count = count($ip_list);
$count_overflow = $instance_count - $opt_inst_count_target;
$count_deregister = (int) ($count_overflow / 2);
//Deregister the first (count_overflow) / 2 nodes from the ip_list and wait for 5 minutes
if ($count_overflow > 2) {
echo "Deregistering {$count_deregister} nodes from F5...\n";
$deregister_ips = array_slice($ip_list, 0, $count_deregister);
$deregister_ip_list = json_encode($deregister_ips);
$deregister_ip_list = substr($deregister_ip_list, 1, -1);
//Deregister $count_deregister nodes from F5 parallelly
system("php {$relative_path}/deregister_node.php -g {$game_name} -p {$pool_name} -i {$deregister_ip_list}", $exit_code);
if ($exit_code == 0) {
echo "Failed to complete the deregistration process. Exiting.\n";
exit(0);
}
$ip_list = array_slice($ip_list, $count_deregister);
$deregistered_ips = array_merge($deregistered_ips, $deregister_ips);
update_zmon_machines($game_cfg, $deregistered_ips);
} else {
echo "The current instance count ({$instance_count}) is almost optimal for the target slack.\n";
$current_slack = get_current_slack($game_name, $pool_name, $instance_count);
print_final_result($current_slack, count($ip_list), $deregistered_ips);
}
//Wait for 5 minutes for the changes to take place
wait_for_change(5);
//Calculate the current slack
$current_slack_after_dereg = get_current_slack($game_name, $pool_name, count($ip_list));
/* FETCH THE PRT from all the nodes and store it. Also Compare the PRT */
//Flag to check if the pdt diff is huge so as to terminate the workflow in this iteration
$pdt_stop_iteration = false;
while (true) {
$pdt_huge_diff = false;
$current_page_pdt_array = get_pdt_per_page($f5_vip, count($ip_list), $prev_pdt);
$prev_pdt = $current_page_pdt_array;
foreach ($current_page_pdt_array as $page => $pdt_data) {
$pdt_diff = $pdt_data[2] - $initial_page_pdt_array[$page][2];
$diff_tol = $pdt_diff_tol_percent / 100 * $initial_page_pdt_array[$page][2];
echo "Checking PDT for {$page} .... (" . $pdt_data[2] . " ms) .... ";
if ($pdt_diff > $diff_tol) {
echo "\nPDT (" . $pdt_data[2] . " ms) increased more than {$pdt_diff_tol_percent} % for page {$page}.\n\n";
$pdt_stop_iteration = true;
$pdt_huge_diff = true;
} else {
echo "OK\n";
}
}
//If the pdt difference is intolerable then register $count_deregister/2 nodes wait for 5 mins and check again.
if ($pdt_huge_diff) {
$count_register = ceil($count_deregister / 2);
if (count($deregistered_ips) < $count_register) {
$count_register = count($deregistered_ips);
}
if (empty($deregistered_ips)) {
echo "All the nodes have been registerd.Still pdt difference is high...\n";
print_final_result($current_slack_after_dereg, count($ip_list), $deregistered_ips);
}
echo "Registering {$count_register} nodes .... \n";
$register_ips = array_slice($deregistered_ips, 0, $count_register);
$register_ip_list = json_encode($register_ips);
$register_ip_list = substr($register_ip_list, 1, -1);
system("php {$relative_path}/register_node.php -g {$game_name} -p {$pool_name} -i {$register_ip_list}", $exit_code);
if ($exit_code == 0) {
echo "Failed to complete the Registration process. Exiting.\n";
print_final_result($current_slack_after_dereg, count($ip_list), $deregistered_ips, false);
}
$ip_list = array_merge($ip_list, $register_ips);
$deregistered_ips = array_slice($deregistered_ips, $count_register);
update_zmon_machines($game_cfg, $deregistered_ips);
//Wait for 5 minutes for the changes to take place
wait_for_change(5);
} else {
break;
}
}
$current_slack_after_dereg = get_current_slack($game_name, $pool_name, count($ip_list));
if ($current_slack_after_dereg > $target_slack) {
echo "Current slack ({$current_slack_after_dereg} %) is greater than target slack ({$target_slack} %). continuing .....\n";
} else {
if ($current_slack_after_dereg == $target_slack) {
echo "Current slack ({$current_slack_after_dereg} %) is equal to target slack. Slack reduction completed successfully...\n";
print_final_result($current_slack_after_dereg, count($ip_list), $deregistered_ips);
} else {
// Current slack is less than the target slack - add $count_deregister/2 nodes, wait for 5 mins and check slack again.
echo "Current slack ({$current_slack_after_dereg} %) is less than target slack ({$target_slack} %)\n";
$count_register = ceil($count_deregister / 2);
while ($current_slack_after_dereg < $target_slack) {
if (count($deregistered_ips) < $count_register) {
$count_register = count($deregistered_ips);
}
if (empty($deregistered_ips)) {
echo "All the ips have been registered...\n";
break;
}
echo "Registering {$count_register} nodes .... \n";
$register_ips = array_slice($deregistered_ips, 0, $count_register);
$register_ip_list = json_encode($register_ips);
$register_ip_list = substr($register_ip_list, 1, -1);
system("php {$relative_path}/register_node.php -g {$game_name} -p {$pool_name} -i {$register_ip_list}", $exit_code);
if ($exit_code == 0) {
echo "Failed to complete the Registration process. Exiting.\n";
print_final_result($current_slack_after_dereg, count($ip_list), $deregistered_ips, false);
}
$ip_list = array_merge($ip_list, $register_ips);
$deregistered_ips = array_slice($deregistered_ips, $count_register);
update_zmon_machines($game_cfg, $deregistered_ips);
//Wait for 5 minutes for the changes to take place
wait_for_change(5);
$current_slack_after_dereg = get_current_slack($game_name, $pool_name, count($ip_list));
}
print_final_result($current_slack_after_dereg, count($ip_list), $deregistered_ips);
}
}
if ($pdt_stop_iteration) {
print_final_result($current_slack_after_dereg, count($ip_list), $deregistered_ips);
}
$iteration_count++;
}
if ($iteration_count >= $limit) {
echo "{$iteration_count} iterations completed!\n";
print_final_result($current_slack_after_dereg, count($ip_list), $deregistered_ips);
}
}
}
if (isset($_GET['hostgroup'])) {
$hostgroup = $_GET['hostgroup'];
}
return array('game' => $game, 'array_id' => $array_id, "hostgroup" => $hostgroup);
}
function ProcessRequest($server_cfg, $game_cfg, $params)
{
$query = $_GET["query"];
if (!$query) {
$query = 'eu_web_chart_range';
}
if (!$query) {
return json_encode("Illegal query.");
}
$xhprofModelObject = new XhProfModel($server_cfg, $game_cfg);
date_default_timezone_set('UTC');
$end_time = time();
$start_time = time() - 12 * 7 * 24 * 60 * 60;
# 12 weeks ago
$chart_result = $xhprofModelObject->generic_execute_get_query_detail($query, array('table' => "vertica_stats_30min", 'end_time' => $end_time, 'start_time' => $start_time, 'prefix' => str_replace("-", "_", $params['hostgroup'])));
$tags = $xhprofModelObject->generic_execute_get_query("get_tag_range", array('table' => "events", 'end_time' => $end_time, 'start_time' => $start_time, 'extra_params' => ""));
$chart_result["tags"] = $tags;
echo json_encode($chart_result);
}
$params = safe_get_params();
$game_cfg = load_game_config($params['game'], $params['array_id']);
ProcessRequest($server_cfg, $game_cfg, $params);
?>
function main($server_cfg)
{
$game_cfg = null;
$options = getopt("g:n:p:");
//
// parameters needed to read zrt varaibles;
// such as XHPRO_ENABLE_IPLIST
//
$parameters['product'] = $options['p'];
$parameters['namespace'] = $options['n'];
if (isset($options['g'])) {
$game_cfg = load_game_config($options['g']);
} else {
echo "pass game name as parameter with option -g";
exit(Monitor::UNKNOWN);
}
if (!isset($game_cfg)) {
echo "game configuration is not loaded";
exit(Monitor::UNKNOWN);
}
$time_slot = (int) (time() / 1800);
$monitor = new Monitor($server_cfg, $game_cfg);
$result = $monitor->monitor_metrics($time_slot - 1, $parameters);
exit($result);
}
function generate_ini($server_cfg)
{
global $ip;
$game = get_game();
$game_cfg = load_game_config($game);
if ($game == null or empty($game_cfg) or !in_array($game, $server_cfg["auto_ini_games"]) or !ip_in_candidates($ip, $server_cfg, $game)) {
send_fail("ini generation criteria not satisfied");
return;
}
$ini_file_path = sprintf($server_cfg['ini_file_path'], $game);
$ini = file_get_contents($ini_file_path);
if (empty($ini)) {
send_fail("Could not read ini file at {$ini_file_path}");
return;
}
$md5_digest = md5($ini);
$ini_header = sprintf(INI_HEADER_TEMPLATE, $md5_digest);
send_ini($ini_header, $ini);
}
<?php # # Copyright 2013 Zynga Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # include 'game_config.php'; $game_cfg = load_game_config($argv[1]); echo $game_cfg[$argv[2]];
{
include_once '/usr/local/zperfmon/bin/rightscale.php';
$rs = new RightScale($server_cfg, $game_cfg);
$array_map = $rs->get_array_to_arrayid_mapping();
$ret = array("all" => "all");
$ret = array_merge($ret, $array_map);
return $ret;
}
$game_array_id_name = array();
$key = "gameidmapping";
define('TTL', 12 * 60 * 60);
//cache expiry time for apc is 12 hrs
$game_array_id_name = apc_fetch($key, $success);
if (!$success) {
foreach ($server_cfg['game_list'] as $game) {
$game_cfg = load_game_config($game);
$array_list = get_array_map($server_cfg, $game_cfg);
$game_array_id_name[$game] = $array_list;
}
apc_add($key, $game_array_id_name, TTL);
}
$game_array_id_json = json_encode($game_array_id_name);
$game = "";
if (isset($_REQUEST["game"])) {
$game = $_REQUEST["game"];
}
if (isset($_REQUEST["array"]) && $_REQUEST['array'] !== 'all') {
$array = $_REQUEST['array'];
} else {
$array = "all";
}
function main($server_cfg)
{
$options = getopt("g:");
if (isset($options['g']) && $options['g'] !== '') {
$game = $options['g'];
} else {
echo "Input not correct: Use <script> -g <game_name> \n";
exit;
}
$game_cfg = load_game_config($game);
$deploy_id = $game_cfg["deployIDs"][0];
# Getting the distinct hostgroups for the inputed game
$conf_all = parse_ini_file($server_cfg["rs_conf_file"], true);
$conf = $conf_all["DB"];
$db_server = $conf["host"];
$db_user = $conf["user"];
$db_pass = $conf["password"];
$db_name = $conf["database"];
$mysql_pdo = new PDO("mysql:host={$db_server};dbname={$db_name}", $db_user, $db_pass);
if (!$mysql_pdo) {
print "Failed to create new mysql PDO\n";
return 1;
}
$query = "select distinct hostgroup from instances where deploy_id = {$deploy_id} and array_id = 0";
$stmt = $mysql_pdo->prepare($query);
$stmt->execute();
$hostgroups = $stmt->fetchAll(PDO::FETCH_ASSOC);
$query = "select distinct hostgroup from instances where deploy_id = {$deploy_id} and array_id != 0";
$stmt = $mysql_pdo->prepare($query);
$stmt->execute();
$web_hostgroups = $stmt->fetchAll(PDO::FETCH_ASSOC);
# Regular expressions to find the class
$re = "(spare|old|bad|consumer|mb|gh-mqs|mq|db|mc|proxy|nagios|msched|gib)";
$re_gh = "(spare|old|bad|consumer|mb|db|mq-|mc|proxy|nagios|msched|gib)";
$re_bad = "(spare|old|bad)";
$yml = "";
foreach ($web_hostgroups as $hostgroup) {
$yml = $yml . "'" . $hostgroup['hostgroup'] . ".*':\n";
$yml = $yml . " <<: *common_web_eu\n";
$yml = $yml . " class: web\n";
$yml = $yml . " hostgroup: " . $hostgroup['hostgroup'] . "\n\n";
}
foreach ($hostgroups as $hostgroup) {
preg_match($re_bad, $hostgroup['hostgroup'], $matches_bad);
preg_match($re, $hostgroup['hostgroup'], $matches);
if (count($matches_bad) > 0) {
$yml = $yml . "'" . $hostgroup['hostgroup'] . ".*':\n\n";
} else {
if (count($matches) == 0) {
$yml = $yml . "'" . $hostgroup['hostgroup'] . ".*':\n\n";
echo "Please define a class for {$hostgroup['hostgroup']}\n";
} else {
$class = $matches[0];
# for greyhound class
if ($class == "gh-mqs") {
preg_match($re_gh, $hostgroup['hostgroup'], $matches);
$class = $matches[0];
} else {
if ($class == "mq") {
$class = "mqueue";
}
}
$host = explode("-", $hostgroup['hostgroup']);
array_shift($host);
$hostgroup_name = implode("-", $host);
$yml = $yml . "'" . $hostgroup['hostgroup'] . ".*':\n";
$yml = $yml . " <<: *common_" . $class . "_eu\n";
$yml = $yml . " class: " . $class . "\n";
$yml = $yml . " hostgroup: " . $hostgroup_name . "\n\n";
}
}
}
$yml_path = sprintf($server_cfg["hostgroups_config"], $game_cfg['name']);
file_put_contents($yml_path, $yml);
}
