function get_array_id_map($server_cfg, $game_cfg) { include_once '/usr/local/zperfmon/bin/rightscale.php'; $rs = new RightScale($server_cfg, $game_cfg); $array_map = $rs->get_array_to_arrayid_mapping(); return $array_map; }
function insert_zmonitor_data($server_cfg, $game_cfg, $time_slots = '*') { $game_name = $game_cfg["name"]; $table = $game_cfg['zmonitor_table']; $array_name = null; // Check if passed game is an array game. If yes change parameters for get_eu accordingly if (isset($game_cfg['id'])) { $array_id = $game_cfg['id']; $rsObj = new RightScale($server_cfg, $game_cfg); $array_id_name = $rsObj->get_array_id_name(); $game_name = $game_cfg['parent']; if (!empty($array_id_name)) { $array_name = $array_id_name[$array_id]; } } // Fetch curent eu from eu database $euObj = new EUAdapter($server_cfg); $data_array = $euObj->get_current_eu($game_name, $array_name); $query = createQuery($table, $data_array); echo "{$query}\n"; $query_res = execute_queries($server_cfg, $game_cfg, array($query)); if ($query_res == 0) { $game_cfg['logger']->log("insert_zmonitor_data", "zmonitor data for " . $game_cfg['name'] . " successfully inserted", Logger::INFO); } else { $game_cfg['logger']->log("insert_zmonitor_data", "zmonitor data for " . $game_cfg['name'] . " not inserted", Logger::ERR); } }
function splitarraywise($server_cfg, $ip_list, $game_name, $time_slot) { $game_cfg = load_game_config($game_name); $rs = new RightScale($server_cfg, $game_cfg); $mapping = $rs->get_array_ip_mapping(); $return = array(); foreach ($ip_list as $ip) { $dir_base_path = sprintf($server_cfg["root_upload_directory"], $game_name); $base_ip_dir = $dir_base_path . $time_slot . "/" . $server_cfg['profile_upload_directory']; $ip_dir = $base_ip_dir . "/" . $ip; //$filelist = scandir($ip_dir); $array_id = $mapping[$ip]; // To prevent game_ folder to be created due to 5 minute processing if (!isset($array_id) || $array_id == '') { continue; } $return[$array_id][] = $ip; $dir_base_path_array = sprintf($server_cfg["root_upload_directory"], $game_name . "_" . $array_id); //creating the timeslot directory if (!is_dir($dir_base_path_array . $time_slot)) { mkdir($dir_base_path_array . $time_slot, 0777, true); } //creating the xhprof directory if (!is_dir($dir_base_path_array . $time_slot . "/" . $server_cfg['profile_upload_directory'])) { mkdir($dir_base_path_array . $time_slot . "/" . $server_cfg['profile_upload_directory'], 0777, true); } $base_ip_dir_array = $dir_base_path_array . $time_slot . "/" . $server_cfg['profile_upload_directory']; $ip_directory_array = $base_ip_dir_array . "/" . $ip; // Check if symlink exists already. In case of frequent call of the function in a timeslot if (!is_dir($ip_directory_array)) { symlink($ip_dir, $ip_directory_array); } if (!file_exists($base_ip_dir_array . "/" . ".profiles")) { // put the ip uploading ip addresses which should be segragated while massaging. file_put_contents("{$base_ip_dir_array}/.profiles", $ip, FILE_APPEND | LOCK_EX); //touch(($base_ip_dir_array."/".".profiles")); error_log(".profiles created" . "\n" . sprintf($server_cfg['log_file'], $game_cfg['name'])); } if (!file_exists($base_ip_dir_array . "/" . ".slowpages")) { touch($base_ip_dir_array . "/" . ".slowpages"); error_log(".slowpages created" . "\n" . sprintf($server_cfg['log_file'], $game_cfg['name'])); } if (!file_exists($base_ip_dir_array . "/" . ".apache_stats")) { touch($base_ip_dir_array . "/" . ".apache_stats"); error_log(".apache_stats created" . "\n" . sprintf($server_cfg['log_file'], $game_cfg['name'])); } } return $return; }
function insert_rightscale_data($server_cfg, $game_cfg, $time_slots = '*') { // TODO take the names from config and expand table as and when new machine class is added to config $machine_class_default = array('web_count' => 0, 'mb_count' => 0, 'mc_count' => 0, 'db_count' => 0, 'queue_count' => 0, 'proxy_count' => 0, 'admin_count' => 0); $game_name = $game_cfg["name"]; $table = $game_cfg['db_stats_table']; $gid = $game_cfg['gameid']; // to query for the dau_5min table and to insert into stats_30min; $snid = $game_cfg['snid'] ? $game_cfg['snid'] : null; $cid = $game_cfg['cid'] ? $game_cfg['cid'] : null; $deploy_array = $game_cfg['deployIDs']; $deploy_id = $deploy_array[0]; // // get the max value of the supplied timeslots // expects that time_slots are given as follows // {ts1,ts2} // preg_match("/\\{(.*)\\}/", $time_slots, $m); $timeslot = max(explode(",", $m[1])); if (empty($timeslot)) { $game_cfg['logger']->log("insert_bd_metric", "null is given as timeslot", Logger::ERR); error_log("null is given as timeslots", 3, sprintf($server_cfg['log_file'], $game_cfg['name'])); return; } $timestamp = $timeslot * 1800; $dauObj = new DAUAdapter(); $dau = $dauObj->get_timed_dau($timestamp, $gid, $snid, $cid); $rsObj = new RightScale($server_cfg, $game_cfg); $web_count = null; if (isset($game_cfg['id'])) { $web_count = $rsObj->get_host_count_per_pool($game_cfg['id'], $deploy_id); $game_name = $game_cfg['parent']; // to query other machine counts } $machine_counts = $rsObj->get_host_count_per_class($deploy_id, $game_name); $counts_array = get_counts($machine_class_default, $machine_counts, $web_count); $query = create_query($table, $timestamp, $dau, $counts_array, $gid); $queries = array($query); $query_res = execute_queries($server_cfg, $game_cfg, $queries); if ($query_res == 0) { $game_cfg['logger']->log("insert_bd_metrics", "bd_metrics metrics for " . $game_cfg['name'] . " successfully inserted", Logger::INFO); error_log("bd_metrics metrics for {$game_cfg['name']} successfully inserted", 3, sprintf($server_cfg['log_file'], $game_cfg['name'])); } else { $game_cfg['logger']->log("insert_bd_metrics", "bd_metrics metrics for " . $game_cfg['name'] . " not inserted", Logger::ERR); error_log("bd_metrics metrics for {$game_cfg['name']} successfully inserted", 3, sprintf($server_cfg['log_file'], $game_cfg['name'])); } }
function main($server_cfg) { $options = getopt("g:"); $game = $options['g']; $game_cfg = load_game_config($game); $hostConfigObj = new HostgroupConfig($server_cfg, $game); $rsObj = new RightScale($server_cfg, $game_cfg); $array_id_name = array_values($rsObj->get_array_to_arrayid_mapping()); $config_col_names = $hostConfigObj->get_config_column_names(); $xhprofModelObj = new XhprofModel($server_cfg, $game_cfg, false); // add columns for both the tables $tables = array("vertica_stats_30min", "vertica_stats_daily"); echo "adding columns for {$game}:\n"; foreach ($tables as $table) { $result = add_missing_columns($config_col_names, $table, $xhprofModelObj); } foreach ($array_id_name as $array_id) { echo "adding columns for {$game}:{$array_id}:\n"; $game_cfg = load_game_config($game, $array_id); $xhprofModelObj = new XhprofModel($server_cfg, $game_cfg, false); foreach ($tables as $table) { $result = add_missing_columns($config_col_names, $table, $xhprofModelObj); } } }
function main($server_cfg) { $options = get_options(); if (isset($options['g']) && $options['g'] !== '') { $game_names = explode(",", $options['g']); } else { $game_names = $server_cfg['game_list']; } $time_slots = null; if (!empty($options['t'])) { $time_slots = $options['t']; } $timestamp = $_SERVER['REQUEST_TIME']; $current_time_slot = (int) ($timestamp / (30 * 60)); foreach ($game_names as $game_name) { zpm_preamble($game_name); // MODIFIED!!!! Create and mount next two timeslots !!! $game_cfg = load_game_config($game_name); if (!$game_cfg) { error_log("configuration for " . $game_name . " is not loaded\n", 3, sprintf($server_cfg['log_file'], $game_cfg['name'])); continue; } $target_dir = sprintf($server_cfg['root_upload_directory'], $game_cfg["name"]); $target_dir = $target_dir . "/%s/" . $server_cfg["profile_upload_directory"]; $dir_name_array = create_directory($target_dir, $current_time_slot, $server_cfg, $game_cfg); if ($dir_name_array === null) { error_log("Directory creation failed for the game " . $game_cfg['name'], 3, sprintf($server_cfg['log_file'], $game_cfg['name'])); continue; } if ($dir_name_array === 0) { error_log("No Ram disk Exist!!!" . $game_cfg['name'], 3, sprintf($server_cfg['log_file'], $game_cfg['name'])); } //creating new games and getting the list of web arrays to process data for $rs = new RightScale($server_cfg, load_game_config($game_name)); $rs->make_array_games(); $array_ids = $rs->get_arrays_to_serve(); // // Wrap processing for each game inside a file lock // $game_lock_file = sprintf($server_cfg["zperfmon_lock_file"], $game_name); $game_lock_handle = grab_lock($game_lock_file, $server_cfg, $game_cfg); if (!$game_lock_handle) { error_log("Could not get exclusive lock for \"{$game_name}\"\n", 3, sprintf($server_cfg['log_file'], $game_cfg['name'])); delete_ramdisk_data($game_name, $time_slots, $timestamp, $server_cfg, $game_cfg); continue; } //processing parent game try { // // process uploaded data // run_cron_for_game($server_cfg, $game_cfg, $time_slots); } catch (Exception $e) { error_log("Upload processing for {$game_name} failed\n", 3, sprintf($server_cfg['log_file'], $game_cfg['name'])); error_log("Exception says: " . $e->getMessage() . "\n", 3, sprintf($server_cfg['log_file'], $game_cfg['name'])); } // loop to process data for each web array foreach ($array_ids as $array) { try { $game_cfg = load_game_config($game_name, $array); if (!$game_cfg) { error_log("configuration for " . $game_name . " is not loaded\n", 3, sprintf($server_cfg['log_file'], $game_cfg['name'])); continue; } // // process uploaded data // run_cron_for_game($server_cfg, $game_cfg, $time_slots); } catch (Exception $e) { error_log("Upload processing for {$game_name} failed\n", 3, sprintf($server_cfg['log_file'], $game_cfg['name'])); error_log("Exception says: " . $e->getMessage() . "\n", 3, sprintf($server_cfg['log_file'], $game_cfg['name'])); } } // compress for parent game // This implementation forces us to always pass a timeslot // while calling this script. otherwise compression will fail. $game_cfg = load_game_config($game_name); compress_unziped_profiles($server_cfg, $game_cfg, $time_slots); // Now compress the array games foreach ($array_ids as $array) { $game_cfg = load_game_config($game_name, $array); compress_unziped_profiles($server_cfg, $game_cfg, $time_slots); } // cleanup the lock drop_lock($game_lock_handle, $game_lock_file); zpm_postamble($game_name); //MODIFIED!!!! if ($dir_name_array != 0) { delete_ramdisk_data($game_name, $time_slots, $timestamp, $server_cfg, $game_cfg); } } }
function run_upload_processing($server_cfg) { date_default_timezone_set('UTC'); $current_timestamp = $_SERVER['REQUEST_TIME']; $options = getopt("g:t:a"); if (isset($options['g']) && $options['g'] !== '') { $game_names = explode(",", $options['g']); } else { $game_names = $server_cfg['game_list']; } $time_slots = null; if (empty($options['t'])) { $time_slots = (int) ($current_timestamp / 1800); $time_slots = "{" . $time_slots . "}"; } else { $time_slots = $options['t']; } foreach ($game_names as $game_name) { zpm_preamble($game_name); $game_cfg = load_game_config($game_name); // Process each array of the game $rsObj = new RightScale($server_cfg, $game_cfg); $rsObj->make_array_games(); process_uploads($server_cfg, $game_cfg, $time_slots, $rsObj); zpm_postamble($game_name); } }
function partition_dynamics($game_names, $server_cfg, $con, $log_file) { //Month to Partition map $partition_drop_map = array('p001', 'p002', 'p003', 'p004', 'p005', 'p006', 'p001', 'p002', 'p003', 'p004', 'p005', 'p006'); //Current month to retention month p_tag map. $cursor_retain_months = array(11, 12, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10); $month_index = date('m') - 1; $array_drop_index = $month_index - 2; //If array drop index is negative go to the end of array if ($array_drop_index < 0) { $array_drop_index = 12 + $array_drop_index; } $partition_drop_array_games = $partition_drop_map[$array_drop_index]; foreach ($game_names as $game) { $db = "zprf_" . $game; report_partitions_state($db, $con, $log_file); $cursor_retain = $cursor_retain_months[$month_index]; $cursor_delete = 0 - $cursor_retain; //Deleting rows in the ratio 1:4 delete_rows($cursor_delete, $db, $con, $log_file); //Doing an alter table crunch_space($db, $con, $log_file); //Fetching the retention time from game_cfg file $game_cfg = load_game_config($game); $retention_time = $game_cfg["xhprof_retention_time"]; //Maximum retention time = 4 months //If retention time not specified setting the default retention time as 4 months if ($retention_time == NULL or $retention_time > 4) { $retention_time = 4; } $partition_drop_index = $month_index - $retention_time - 1; if ($partition_drop_index < 0) { $partition_drop_index = 12 + $partition_drop_index; } $partition_drop = $partition_drop_map[$partition_drop_index]; $rs = new RightScale($server_cfg, load_game_config($game)); $array_ids = $rs->get_arrays_to_serve(); drop_partition_data($partition_drop, $partition_drop_array_games, $array_ids, $db, $con, $log_file); } }