function get_array_id_map($server_cfg, $game_cfg)
{
include_once '/usr/local/zperfmon/bin/rightscale.php';
$rs = new RightScale($server_cfg, $game_cfg);
$array_map = $rs->get_array_to_arrayid_mapping();
return $array_map;
}
function insert_zmonitor_data($server_cfg, $game_cfg, $time_slots = '*')
{
$game_name = $game_cfg["name"];
$table = $game_cfg['zmonitor_table'];
$array_name = null;
// Check if passed game is an array game. If yes change parameters for get_eu accordingly
if (isset($game_cfg['id'])) {
$array_id = $game_cfg['id'];
$rsObj = new RightScale($server_cfg, $game_cfg);
$array_id_name = $rsObj->get_array_id_name();
$game_name = $game_cfg['parent'];
if (!empty($array_id_name)) {
$array_name = $array_id_name[$array_id];
}
}
// Fetch curent eu from eu database
$euObj = new EUAdapter($server_cfg);
$data_array = $euObj->get_current_eu($game_name, $array_name);
$query = createQuery($table, $data_array);
echo "{$query}\n";
$query_res = execute_queries($server_cfg, $game_cfg, array($query));
if ($query_res == 0) {
$game_cfg['logger']->log("insert_zmonitor_data", "zmonitor data for " . $game_cfg['name'] . " successfully inserted", Logger::INFO);
} else {
$game_cfg['logger']->log("insert_zmonitor_data", "zmonitor data for " . $game_cfg['name'] . " not inserted", Logger::ERR);
}
}
function splitarraywise($server_cfg, $ip_list, $game_name, $time_slot)
{
$game_cfg = load_game_config($game_name);
$rs = new RightScale($server_cfg, $game_cfg);
$mapping = $rs->get_array_ip_mapping();
$return = array();
foreach ($ip_list as $ip) {
$dir_base_path = sprintf($server_cfg["root_upload_directory"], $game_name);
$base_ip_dir = $dir_base_path . $time_slot . "/" . $server_cfg['profile_upload_directory'];
$ip_dir = $base_ip_dir . "/" . $ip;
//$filelist = scandir($ip_dir);
$array_id = $mapping[$ip];
// To prevent game_ folder to be created due to 5 minute processing
if (!isset($array_id) || $array_id == '') {
continue;
}
$return[$array_id][] = $ip;
$dir_base_path_array = sprintf($server_cfg["root_upload_directory"], $game_name . "_" . $array_id);
//creating the timeslot directory
if (!is_dir($dir_base_path_array . $time_slot)) {
mkdir($dir_base_path_array . $time_slot, 0777, true);
}
//creating the xhprof directory
if (!is_dir($dir_base_path_array . $time_slot . "/" . $server_cfg['profile_upload_directory'])) {
mkdir($dir_base_path_array . $time_slot . "/" . $server_cfg['profile_upload_directory'], 0777, true);
}
$base_ip_dir_array = $dir_base_path_array . $time_slot . "/" . $server_cfg['profile_upload_directory'];
$ip_directory_array = $base_ip_dir_array . "/" . $ip;
// Check if symlink exists already. In case of frequent call of the function in a timeslot
if (!is_dir($ip_directory_array)) {
symlink($ip_dir, $ip_directory_array);
}
if (!file_exists($base_ip_dir_array . "/" . ".profiles")) {
// put the ip uploading ip addresses which should be segragated while massaging.
file_put_contents("{$base_ip_dir_array}/.profiles", $ip, FILE_APPEND | LOCK_EX);
//touch(($base_ip_dir_array."/".".profiles"));
error_log(".profiles created" . "\n" . sprintf($server_cfg['log_file'], $game_cfg['name']));
}
if (!file_exists($base_ip_dir_array . "/" . ".slowpages")) {
touch($base_ip_dir_array . "/" . ".slowpages");
error_log(".slowpages created" . "\n" . sprintf($server_cfg['log_file'], $game_cfg['name']));
}
if (!file_exists($base_ip_dir_array . "/" . ".apache_stats")) {
touch($base_ip_dir_array . "/" . ".apache_stats");
error_log(".apache_stats created" . "\n" . sprintf($server_cfg['log_file'], $game_cfg['name']));
}
}
return $return;
}
function insert_rightscale_data($server_cfg, $game_cfg, $time_slots = '*')
{
// TODO take the names from config and expand table as and when new machine class is added to config
$machine_class_default = array('web_count' => 0, 'mb_count' => 0, 'mc_count' => 0, 'db_count' => 0, 'queue_count' => 0, 'proxy_count' => 0, 'admin_count' => 0);
$game_name = $game_cfg["name"];
$table = $game_cfg['db_stats_table'];
$gid = $game_cfg['gameid'];
// to query for the dau_5min table and to insert into stats_30min;
$snid = $game_cfg['snid'] ? $game_cfg['snid'] : null;
$cid = $game_cfg['cid'] ? $game_cfg['cid'] : null;
$deploy_array = $game_cfg['deployIDs'];
$deploy_id = $deploy_array[0];
//
// get the max value of the supplied timeslots
// expects that time_slots are given as follows
// {ts1,ts2}
//
preg_match("/\\{(.*)\\}/", $time_slots, $m);
$timeslot = max(explode(",", $m[1]));
if (empty($timeslot)) {
$game_cfg['logger']->log("insert_bd_metric", "null is given as timeslot", Logger::ERR);
error_log("null is given as timeslots", 3, sprintf($server_cfg['log_file'], $game_cfg['name']));
return;
}
$timestamp = $timeslot * 1800;
$dauObj = new DAUAdapter();
$dau = $dauObj->get_timed_dau($timestamp, $gid, $snid, $cid);
$rsObj = new RightScale($server_cfg, $game_cfg);
$web_count = null;
if (isset($game_cfg['id'])) {
$web_count = $rsObj->get_host_count_per_pool($game_cfg['id'], $deploy_id);
$game_name = $game_cfg['parent'];
// to query other machine counts
}
$machine_counts = $rsObj->get_host_count_per_class($deploy_id, $game_name);
$counts_array = get_counts($machine_class_default, $machine_counts, $web_count);
$query = create_query($table, $timestamp, $dau, $counts_array, $gid);
$queries = array($query);
$query_res = execute_queries($server_cfg, $game_cfg, $queries);
if ($query_res == 0) {
$game_cfg['logger']->log("insert_bd_metrics", "bd_metrics metrics for " . $game_cfg['name'] . " successfully inserted", Logger::INFO);
error_log("bd_metrics metrics for {$game_cfg['name']} successfully inserted", 3, sprintf($server_cfg['log_file'], $game_cfg['name']));
} else {
$game_cfg['logger']->log("insert_bd_metrics", "bd_metrics metrics for " . $game_cfg['name'] . " not inserted", Logger::ERR);
error_log("bd_metrics metrics for {$game_cfg['name']} successfully inserted", 3, sprintf($server_cfg['log_file'], $game_cfg['name']));
}
}
function main($server_cfg)
{
$options = getopt("g:");
$game = $options['g'];
$game_cfg = load_game_config($game);
$hostConfigObj = new HostgroupConfig($server_cfg, $game);
$rsObj = new RightScale($server_cfg, $game_cfg);
$array_id_name = array_values($rsObj->get_array_to_arrayid_mapping());
$config_col_names = $hostConfigObj->get_config_column_names();
$xhprofModelObj = new XhprofModel($server_cfg, $game_cfg, false);
// add columns for both the tables
$tables = array("vertica_stats_30min", "vertica_stats_daily");
echo "adding columns for {$game}:\n";
foreach ($tables as $table) {
$result = add_missing_columns($config_col_names, $table, $xhprofModelObj);
}
foreach ($array_id_name as $array_id) {
echo "adding columns for {$game}:{$array_id}:\n";
$game_cfg = load_game_config($game, $array_id);
$xhprofModelObj = new XhprofModel($server_cfg, $game_cfg, false);
foreach ($tables as $table) {
$result = add_missing_columns($config_col_names, $table, $xhprofModelObj);
}
}
}
function main($server_cfg)
{
$options = get_options();
if (isset($options['g']) && $options['g'] !== '') {
$game_names = explode(",", $options['g']);
} else {
$game_names = $server_cfg['game_list'];
}
$time_slots = null;
if (!empty($options['t'])) {
$time_slots = $options['t'];
}
$timestamp = $_SERVER['REQUEST_TIME'];
$current_time_slot = (int) ($timestamp / (30 * 60));
foreach ($game_names as $game_name) {
zpm_preamble($game_name);
// MODIFIED!!!! Create and mount next two timeslots !!!
$game_cfg = load_game_config($game_name);
if (!$game_cfg) {
error_log("configuration for " . $game_name . " is not loaded\n", 3, sprintf($server_cfg['log_file'], $game_cfg['name']));
continue;
}
$target_dir = sprintf($server_cfg['root_upload_directory'], $game_cfg["name"]);
$target_dir = $target_dir . "/%s/" . $server_cfg["profile_upload_directory"];
$dir_name_array = create_directory($target_dir, $current_time_slot, $server_cfg, $game_cfg);
if ($dir_name_array === null) {
error_log("Directory creation failed for the game " . $game_cfg['name'], 3, sprintf($server_cfg['log_file'], $game_cfg['name']));
continue;
}
if ($dir_name_array === 0) {
error_log("No Ram disk Exist!!!" . $game_cfg['name'], 3, sprintf($server_cfg['log_file'], $game_cfg['name']));
}
//creating new games and getting the list of web arrays to process data for
$rs = new RightScale($server_cfg, load_game_config($game_name));
$rs->make_array_games();
$array_ids = $rs->get_arrays_to_serve();
//
// Wrap processing for each game inside a file lock
//
$game_lock_file = sprintf($server_cfg["zperfmon_lock_file"], $game_name);
$game_lock_handle = grab_lock($game_lock_file, $server_cfg, $game_cfg);
if (!$game_lock_handle) {
error_log("Could not get exclusive lock for \"{$game_name}\"\n", 3, sprintf($server_cfg['log_file'], $game_cfg['name']));
delete_ramdisk_data($game_name, $time_slots, $timestamp, $server_cfg, $game_cfg);
continue;
}
//processing parent game
try {
//
// process uploaded data
//
run_cron_for_game($server_cfg, $game_cfg, $time_slots);
} catch (Exception $e) {
error_log("Upload processing for {$game_name} failed\n", 3, sprintf($server_cfg['log_file'], $game_cfg['name']));
error_log("Exception says: " . $e->getMessage() . "\n", 3, sprintf($server_cfg['log_file'], $game_cfg['name']));
}
// loop to process data for each web array
foreach ($array_ids as $array) {
try {
$game_cfg = load_game_config($game_name, $array);
if (!$game_cfg) {
error_log("configuration for " . $game_name . " is not loaded\n", 3, sprintf($server_cfg['log_file'], $game_cfg['name']));
continue;
}
//
// process uploaded data
//
run_cron_for_game($server_cfg, $game_cfg, $time_slots);
} catch (Exception $e) {
error_log("Upload processing for {$game_name} failed\n", 3, sprintf($server_cfg['log_file'], $game_cfg['name']));
error_log("Exception says: " . $e->getMessage() . "\n", 3, sprintf($server_cfg['log_file'], $game_cfg['name']));
}
}
// compress for parent game
// This implementation forces us to always pass a timeslot
// while calling this script. otherwise compression will fail.
$game_cfg = load_game_config($game_name);
compress_unziped_profiles($server_cfg, $game_cfg, $time_slots);
// Now compress the array games
foreach ($array_ids as $array) {
$game_cfg = load_game_config($game_name, $array);
compress_unziped_profiles($server_cfg, $game_cfg, $time_slots);
}
// cleanup the lock
drop_lock($game_lock_handle, $game_lock_file);
zpm_postamble($game_name);
//MODIFIED!!!!
if ($dir_name_array != 0) {
delete_ramdisk_data($game_name, $time_slots, $timestamp, $server_cfg, $game_cfg);
}
}
}
function run_upload_processing($server_cfg)
{
date_default_timezone_set('UTC');
$current_timestamp = $_SERVER['REQUEST_TIME'];
$options = getopt("g:t:a");
if (isset($options['g']) && $options['g'] !== '') {
$game_names = explode(",", $options['g']);
} else {
$game_names = $server_cfg['game_list'];
}
$time_slots = null;
if (empty($options['t'])) {
$time_slots = (int) ($current_timestamp / 1800);
$time_slots = "{" . $time_slots . "}";
} else {
$time_slots = $options['t'];
}
foreach ($game_names as $game_name) {
zpm_preamble($game_name);
$game_cfg = load_game_config($game_name);
// Process each array of the game
$rsObj = new RightScale($server_cfg, $game_cfg);
$rsObj->make_array_games();
process_uploads($server_cfg, $game_cfg, $time_slots, $rsObj);
zpm_postamble($game_name);
}
}
function partition_dynamics($game_names, $server_cfg, $con, $log_file)
{
//Month to Partition map
$partition_drop_map = array('p001', 'p002', 'p003', 'p004', 'p005', 'p006', 'p001', 'p002', 'p003', 'p004', 'p005', 'p006');
//Current month to retention month p_tag map.
$cursor_retain_months = array(11, 12, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10);
$month_index = date('m') - 1;
$array_drop_index = $month_index - 2;
//If array drop index is negative go to the end of array
if ($array_drop_index < 0) {
$array_drop_index = 12 + $array_drop_index;
}
$partition_drop_array_games = $partition_drop_map[$array_drop_index];
foreach ($game_names as $game) {
$db = "zprf_" . $game;
report_partitions_state($db, $con, $log_file);
$cursor_retain = $cursor_retain_months[$month_index];
$cursor_delete = 0 - $cursor_retain;
//Deleting rows in the ratio 1:4
delete_rows($cursor_delete, $db, $con, $log_file);
//Doing an alter table
crunch_space($db, $con, $log_file);
//Fetching the retention time from game_cfg file
$game_cfg = load_game_config($game);
$retention_time = $game_cfg["xhprof_retention_time"];
//Maximum retention time = 4 months
//If retention time not specified setting the default retention time as 4 months
if ($retention_time == NULL or $retention_time > 4) {
$retention_time = 4;
}
$partition_drop_index = $month_index - $retention_time - 1;
if ($partition_drop_index < 0) {
$partition_drop_index = 12 + $partition_drop_index;
}
$partition_drop = $partition_drop_map[$partition_drop_index];
$rs = new RightScale($server_cfg, load_game_config($game));
$array_ids = $rs->get_arrays_to_serve();
drop_partition_data($partition_drop, $partition_drop_array_games, $array_ids, $db, $con, $log_file);
}
}
