private function parseDatasize()
{
$values = Utils::get_GET_intArray('datasize');
$this->filters['datasize']['currentChoice'] = $values;
foreach ($values as $value) {
$definition = $this->filters['datasize'];
$DBreference = $definition['table'] != 'mixed' ? "{$definition['table']}Alias." : '';
$DBreference .= isset($definition['field']) ? $definition['field'] : 'datasize';
$errorMargin = $this->getErrorMargin($value);
$maxValue = $value + $errorMargin;
$minValue = $value - $errorMargin;
$this->whereClause .= " AND {$DBreference} >= {$minValue} AND {$DBreference} <= {$maxValue}";
}
}
public function performanceChartsAction()
{
$exec_rows = null;
$id_exec_rows = null;
$dbUtil = $this->container->getDBUtils();
$this->buildFilters(array('perf_details' => array('default' => 1)));
$charts = array();
$clusters = array();
try {
//TODO fix, initialize variables
$dbUtil->get_exec_details('1', 'id_exec', $exec_rows, $id_exec_rows);
//check the URL
$execs = Utils::get_GET_intArray('execs');
if (empty($execs)) {
$whereClause = $this->filters->getWhereClause();
$query = "SELECT e.id_exec FROM aloja2.execs e JOIN aloja2.clusters c USING (id_cluster)\n LEFT JOIN aloja_ml.predictions p USING (id_exec)\n WHERE 1 " . DBUtils::getFilterExecs() . "{$whereClause} ";
$query .= isset($_GET['random']) ? '' : 'LIMIT 1';
$idExecs = $dbUtil->get_rows($query);
if (isset($_GET['random'])) {
$execs = array($idExecs[rand(0, sizeof($idExecs) - 1)]['id_exec']);
} else {
$execs[] = $idExecs[0]['id_exec'];
}
}
if (Utils::get_GET_string('random') && !$execs) {
$keys = array_keys($exec_rows);
$execs = array_unique(array($keys[array_rand($keys)], $keys[array_rand($keys)]));
}
if (Utils::get_GET_string('hosts')) {
$hosts = Utils::get_GET_string('hosts');
} else {
$hosts = 'Slaves';
}
if (Utils::get_GET_string('metric')) {
$metric = Utils::get_GET_string('metric');
} else {
$metric = 'CPU';
}
if (Utils::get_GET_string('aggr')) {
$aggr = Utils::get_GET_string('aggr');
} else {
$aggr = 'AVG';
}
if (Utils::get_GET_string('detail')) {
$detail = Utils::get_GET_int('detail');
} else {
$detail = 10;
}
if ($aggr == 'AVG') {
$aggr_text = "Average";
} elseif ($aggr == 'SUM') {
$aggr_text = "SUM";
} else {
throw new \Exception("Aggregation type '{$aggr}' is not valid.");
}
if ($hosts == 'Slaves') {
$selectedHosts = $dbUtil->get_rows("SELECT h.host_name from execs e inner join hosts h where e.id_exec IN (" . implode(", ", $execs) . ") AND h.id_cluster = e.id_cluster AND h.role='slave'");
$selected_hosts = array();
foreach ($selectedHosts as $host) {
array_push($selected_hosts, $host['host_name']);
}
} elseif ($hosts == 'Master') {
$selectedHosts = $dbUtil->get_rows("SELECT h.host_name from execs e inner join hosts h where e.id_exec IN (" . implode(", ", $execs) . ") AND h.id_cluster = e.id_cluster AND h.role='master' AND h.host_name != ''");
$selected_hosts = array();
foreach ($selectedHosts as $host) {
array_push($selected_hosts, $host['host_name']);
}
} else {
$selected_hosts = array($hosts);
}
$charts = array();
$exec_details = array();
$chart_details = array();
$clusters = array();
foreach ($execs as $exec) {
//do a security check
$tmp = filter_var($exec, FILTER_SANITIZE_NUMBER_INT);
if (!is_numeric($tmp) || !($tmp > 0)) {
unset($execs[$exec]);
continue;
}
$exec_title = $dbUtil->get_exec_details($exec, 'exec', $exec_rows, $id_exec_rows);
$pos_name = strpos($exec_title, '/');
$exec_title = ' ' . strtoupper(substr($exec_title, $pos_name + 1)) . ' ' . (strpos($exec_title, '_az') > 0 ? 'AZURE' : 'LOCAL') . " ID_{$exec} " . substr($exec_title, 21, strlen($exec_title) - $pos_name - (strpos($exec_title, '_az') > 0 ? 21 : 18));
$exec_details[$exec]['time'] = $dbUtil->get_exec_details($exec, 'exe_time', $exec_rows, $id_exec_rows);
$exec_details[$exec]['start_time'] = $dbUtil->get_exec_details($exec, 'start_time', $exec_rows, $id_exec_rows);
$exec_details[$exec]['end_time'] = $dbUtil->get_exec_details($exec, 'end_time', $exec_rows, $id_exec_rows);
$id_cluster = $dbUtil->get_exec_details($exec, 'id_cluster', $exec_rows, $id_exec_rows);
if (!in_array($id_cluster, $clusters)) {
$clusters[] = $id_cluster;
}
//$end_time = get_exec_details($exec, 'init_time');
$date_where = " AND date BETWEEN '{$exec_details[$exec]['start_time']}' and '{$exec_details[$exec]['end_time']}' ";
$where = " WHERE id_exec = '{$exec}' AND host IN ('" . join("','", $selected_hosts) . "') {$date_where}";
$where_BWM = " WHERE id_exec = '{$exec}' AND host IN ('" . join("','", $selected_hosts) . "') ";
$where_VMSTATS = " WHERE id_exec = '{$exec}' AND host IN ('" . join("','", $selected_hosts) . "') ";
$where_sampling = "round(time/{$detail})";
$group_by = " GROUP BY {$where_sampling} ORDER by time";
$group_by_vmstats = " GROUP BY {$where_sampling} ORDER by time";
$where_sampling_BWM = "round(unix_timestamp/{$detail})";
$group_by_BWM = " GROUP BY {$where_sampling_BWM} ORDER by unix_timestamp";
$charts[$exec] = array('job_status' => array('metric' => "ALL", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time,\n maps map,shuffle,merge,reduce,waste FROM aloja_logs.JOB_status\n WHERE id_exec = '{$exec}' {$date_where} GROUP BY job_name, date ORDER by job_name, time;", 'fields' => array('map', 'shuffle', 'reduce', 'waste', 'merge'), 'title' => "Job execution history {$exec_title} ", 'group_title' => 'Job execution history (number of running Hadoop processes)', 'percentage' => false, 'stacked' => false, 'negative' => false), 'cpu' => array('metric' => "CPU", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(`%user`) `%user`, {$aggr}(`%system`) `%system`, {$aggr}(`%steal`) `%steal`, {$aggr}(`%iowait`)\n `%iowait`, {$aggr}(`%nice`) `%nice` FROM aloja_logs.SAR_cpu {$where} {$group_by};", 'fields' => array('%user', '%system', '%steal', '%iowait', '%nice'), 'title' => "CPU Utilization ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'CPU Utilization ' . "({$aggr_text}, {$hosts})", 'percentage' => $aggr == 'SUM' ? '300' : 100, 'stacked' => true, 'negative' => false), 'load' => array('metric' => "CPU", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(`ldavg-1`) `ldavg-1`, {$aggr}(`ldavg-5`) `ldavg-5`, {$aggr}(`ldavg-15`) `ldavg-15`\n FROM aloja_logs.SAR_load {$where} {$group_by};", 'fields' => array('ldavg-15', 'ldavg-5', 'ldavg-1'), 'title' => "CPU Load Average ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'CPU Load Average ' . "({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'load_queues' => array('metric' => "CPU", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(`runq-sz`) `runq-sz`, {$aggr}(`blocked`) `blocked`\n FROM aloja_logs.SAR_load {$where} {$group_by};", 'fields' => array('runq-sz', 'blocked'), 'title' => "CPU Queues ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'CPU Queues ' . "({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'load_tasks' => array('metric' => "CPU", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(`plist-sz`) `plist-sz` FROM aloja_logs.SAR_load {$where} {$group_by};", 'fields' => array('plist-sz'), 'title' => "Number of tasks for CPUs ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'Number of tasks for CPUs ' . "({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'switches' => array('metric' => "CPU", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(`proc/s`) `proc/s`, {$aggr}(`cswch/s`) `cswch/s` FROM aloja_logs.SAR_switches {$where} {$group_by};", 'fields' => array('proc/s', 'cswch/s'), 'title' => "CPU Context Switches ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'CPU Context Switches' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'interrupts' => array('metric' => "CPU", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(`intr/s`) `intr/s` FROM aloja_logs.SAR_interrupts {$where} {$group_by};", 'fields' => array('intr/s'), 'title' => "CPU Interrupts ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'CPU Interrupts ' . "({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'memory_util' => array('metric' => "Memory", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(kbmemfree)*1024 kbmemfree, {$aggr}(kbmemused)*1024 kbmemused\n FROM aloja_logs.SAR_memory_util {$where} {$group_by};", 'fields' => array('kbmemfree', 'kbmemused'), 'title' => "Memory Utilization ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'Memory Utilization' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => true, 'negative' => false), 'memory_util_det' => array('metric' => "Memory", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(kbbuffers)*1024 kbbuffers, {$aggr}(kbcommit)*1024 kbcommit, {$aggr}(kbcached)*1024 kbcached,\n {$aggr}(kbactive)*1024 kbactive, {$aggr}(kbinact)*1024 kbinact\n FROM aloja_logs.SAR_memory_util {$where} {$group_by};", 'fields' => array('kbcached', 'kbbuffers', 'kbinact', 'kbcommit', 'kbactive'), 'title' => "Memory Utilization Details ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'Memory Utilization Details' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => true, 'negative' => false), 'memory' => array('metric' => "Memory", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(`frmpg/s`) `frmpg/s`, {$aggr}(`bufpg/s`) `bufpg/s`, {$aggr}(`campg/s`) `campg/s`\n FROM aloja_logs.SAR_memory {$where} {$group_by};", 'fields' => array('frmpg/s', 'bufpg/s', 'campg/s'), 'title' => "Memory Stats ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'Memory Stats' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'io_pagging_disk' => array('metric' => "Memory", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(`pgpgin/s`)*1024 `pgpgin/s`, {$aggr}(`pgpgout/s`)*1024 `pgpgout/s`\n FROM aloja_logs.SAR_io_paging {$where} {$group_by};", 'fields' => array('pgpgin/s', 'pgpgout/s'), 'title' => "I/O Paging IN/OUT to disk ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'I/O Paging IN/OUT to disk' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'io_pagging' => array('metric' => "Memory", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(`fault/s`) `fault/s`, {$aggr}(`majflt/s`) `majflt/s`, {$aggr}(`pgfree/s`) `pgfree/s`,\n {$aggr}(`pgscank/s`) `pgscank/s`, {$aggr}(`pgscand/s`) `pgscand/s`, {$aggr}(`pgsteal/s`) `pgsteal/s`\n FROM aloja_logs.SAR_io_paging {$where} {$group_by};", 'fields' => array('fault/s', 'majflt/s', 'pgfree/s', 'pgscank/s', 'pgscand/s', 'pgsteal/s'), 'title' => "I/O Paging ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'I/O Paging' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'io_pagging_vmeff' => array('metric' => "Memory", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(`%vmeff`) `%vmeff` FROM aloja_logs.SAR_io_paging {$where} {$group_by};", 'fields' => array('%vmeff'), 'title' => "I/O Paging %vmeff ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'I/O Paging %vmeff' . " ({$aggr_text}, {$hosts})", 'percentage' => $aggr == 'SUM' ? '300' : 100, 'stacked' => false, 'negative' => false), 'io_transactions' => array('metric' => "Disk", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(`tps`) `tp/s`, {$aggr}(`rtps`) `read tp/s`, {$aggr}(`wtps`) `write tp/s`\n FROM aloja_logs.SAR_io_rate {$where} {$group_by};", 'fields' => array('tp/s', 'read tp/s', 'write tp/s'), 'title' => "I/O Transactions/s ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'I/O Transactions/s' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'io_bytes' => array('metric' => "Disk", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(`bread/s`)/(1024) `KB_read/s`, {$aggr}(`bwrtn/s`)/(1024) `KB_wrtn/s`\n FROM aloja_logs.SAR_io_rate {$where} {$group_by};", 'fields' => array('KB_read/s', 'KB_wrtn/s'), 'title' => "KB R/W ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'KB R/W' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'block_devices_util' => array('metric' => "Disk", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(`%util_SUM`) `%util_SUM`, {$aggr}(`%util_MAX`) `%util_MAX`\n FROM (\n select\n id_exec, host, date,\n sum(`%util`) `%util_SUM`,\n max(`%util`) `%util_MAX`\n from aloja_logs.SAR_block_devices d WHERE id_exec = '{$exec}'\n GROUP BY date, host\n ) t {$where} {$group_by};", 'fields' => array('%util_SUM', '%util_MAX'), 'title' => "Disk Uitlization percentage (All DEVs, {$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'Disk Uitlization percentage' . " (All DEVs, {$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'block_devices_await' => array('metric' => "Disk", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(`await_SUM`) `await_SUM`, {$aggr}(`await_MAX`) `await_MAX`\n FROM (\n select\n id_exec, host, date,\n sum(`await`) `await_SUM`,\n max(`await`) `await_MAX`\n from aloja_logs.SAR_block_devices d WHERE id_exec = '{$exec}'\n GROUP BY date, host\n ) t {$where} {$group_by};", 'fields' => array('await_SUM', 'await_MAX'), 'title' => "Disk request wait time in ms (All DEVs, {$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'Disk request wait time in ms' . " (All DEVs, {$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'block_devices_svctm' => array('metric' => "Disk", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(`svctm_SUM`) `svctm_SUM`, {$aggr}(`svctm_MAX`) `svctm_MAX`\n FROM (\n select\n id_exec, host, date,\n sum(`svctm`) `svctm_SUM`,\n max(`svctm`) `svctm_MAX`\n from aloja_logs.SAR_block_devices d WHERE id_exec = '{$exec}'\n GROUP BY date, host\n ) t {$where} {$group_by};", 'fields' => array('svctm_SUM', 'svctm_MAX'), 'title' => "Disk service time in ms (All DEVs, {$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'Disk service time in ms' . " (All DEVs, {$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'block_devices_queues' => array('metric' => "Disk", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(`avgrq-sz`) `avg-req-size`, {$aggr}(`avgqu-sz`) `avg-queue-size`\n FROM (\n select\n id_exec, host, date,\n max(`avgrq-sz`) `avgrq-sz`,\n max(`avgqu-sz`) `avgqu-sz`\n from aloja_logs.SAR_block_devices d WHERE id_exec = '{$exec}'\n GROUP BY date, host\n ) t {$where} {$group_by};", 'fields' => array('avg-req-size', 'avg-queue-size'), 'title' => "Disk req and queue sizes ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'Disk req and queue sizes' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'vmstats_io' => array('metric' => "Disk", 'query' => "SELECT time, {$aggr}(`bi`)/(1024) `KB_IN`, {$aggr}(`bo`)/(1024) `KB_OUT`\n FROM aloja_logs.VMSTATS {$where_VMSTATS} {$group_by_vmstats};", 'fields' => array('KB_IN', 'KB_OUT'), 'title' => "VMSTATS KB I/O ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'VMSTATS KB I/O' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'vmstats_rb' => array('metric' => "CPU", 'query' => "SELECT time, {$aggr}(`r`) `runnable procs`, {$aggr}(`b`) `sleep procs` FROM aloja_logs.VMSTATS {$where_VMSTATS} {$group_by_vmstats};", 'fields' => array('runnable procs', 'sleep procs'), 'title' => "VMSTATS Processes (r-b) ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'VMSTATS Processes (r-b)' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'vmstats_memory' => array('metric' => "Memory", 'query' => "SELECT time, {$aggr}(`buff`) `buff`,\n {$aggr}(`cache`) `cache`,\n {$aggr}(`free`) `free`,\n {$aggr}(`swpd`) `swpd`\n FROM aloja_logs.VMSTATS {$where_VMSTATS} {$group_by_vmstats};", 'fields' => array('buff', 'cache', 'free', 'swpd'), 'title' => "VMSTATS Processes (r-b) ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'VMSTATS Processes (r-b)' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => true, 'negative' => false), 'net_devices_kbs' => array('metric' => "Network", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(if(IFACE != 'lo', `rxkB/s`, NULL))/1024 `rxMB/s_NET`, {$aggr}(if(IFACE != 'lo', `txkB/s`, NULL))/1024 `txMB/s_NET`\n FROM aloja_logs.SAR_net_devices {$where} AND IFACE not IN ('') {$group_by};", 'fields' => array('rxMB/s_NET', 'txMB/s_NET'), 'title' => "MB/s received and transmitted ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'MB/s received and transmitted' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'net_devices_kbs_local' => array('metric' => "Network", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(if(IFACE = 'lo', `rxkB/s`, NULL))/1024 `rxMB/s_LOCAL`, {$aggr}(if(IFACE = 'lo', `txkB/s`, NULL))/1024 `txMB/s_LOCAL`\n FROM aloja_logs.SAR_net_devices {$where} AND IFACE not IN ('') {$group_by};", 'fields' => array('rxMB/s_LOCAL', 'txMB/s_LOCAL'), 'title' => "MB/s received and transmitted LOCAL ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'MB/s received and transmitted LOCAL' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'net_devices_pcks' => array('metric' => "Network", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(if(IFACE != 'lo', `rxpck/s`, NULL))/1024 `rxpck/s_NET`, {$aggr}(if(IFACE != 'lo', `txkB/s`, NULL))/1024 `txpck/s_NET`\n FROM aloja_logs.SAR_net_devices {$where} AND IFACE not IN ('') {$group_by};", 'fields' => array('rxpck/s_NET', 'txpck/s_NET'), 'title' => "Packets/s received and transmitted ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'Packets/s received and transmitted' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'net_devices_pcks_local' => array('metric' => "Network", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(if(IFACE = 'lo', `rxkB/s`, NULL))/1024 `rxpck/s_LOCAL`, {$aggr}(if(IFACE = 'lo', `txkB/s`, NULL))/1024 `txpck/s_LOCAL`\n FROM aloja_logs.SAR_net_devices {$where} AND IFACE not IN ('') {$group_by};", 'fields' => array('rxpck/s_LOCAL', 'txpck/s_LOCAL'), 'title' => "Packets/s received and transmitted LOCAL ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'Packets/s received and transmitted LOCAL' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'net_sockets_pcks' => array('metric' => "Network", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(`totsck`) `totsck`,\n {$aggr}(`tcpsck`) `tcpsck`,\n {$aggr}(`udpsck`) `udpsck`,\n {$aggr}(`rawsck`) `rawsck`,\n {$aggr}(`ip-frag`) `ip-frag`,\n {$aggr}(`tcp-tw`) `tcp-time-wait`\n FROM aloja_logs.SAR_net_sockets {$where} {$group_by};", 'fields' => array('totsck', 'tcpsck', 'udpsck', 'rawsck', 'ip-frag', 'tcp-time-wait'), 'title' => "Packets/s received and transmitted ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'Packets/s received and transmitted' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'net_erros' => array('metric' => "Network", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(`rxerr/s`) `rxerr/s`,\n {$aggr}(`txerr/s`) `txerr/s`,\n {$aggr}(`coll/s`) `coll/s`,\n {$aggr}(`rxdrop/s`) `rxdrop/s`,\n {$aggr}(`txdrop/s`) `txdrop/s`,\n {$aggr}(`txcarr/s`) `txcarr/s`,\n {$aggr}(`rxfram/s`) `rxfram/s`,\n {$aggr}(`rxfifo/s`) `rxfifo/s`,\n {$aggr}(`txfifo/s`) `txfifo/s`\n FROM aloja_logs.SAR_net_errors {$where} {$group_by};", 'fields' => array('rxerr/s', 'txerr/s', 'coll/s', 'rxdrop/s', 'txdrop/s', 'txcarr/s', 'rxfram/s', 'rxfifo/s', 'txfifo/s'), 'title' => "Network errors ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'Network errors' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'bwm_in_out_total' => array('metric' => "Network", 'query' => "SELECT time_to_sec(timediff(FROM_UNIXTIME(unix_timestamp),'{$exec_details[$exec]['start_time']}')) time,\n {$aggr}(`bytes_in`)/(1024*1024) `MB_in`,\n {$aggr}(`bytes_out`)/(1024*1024) `MB_out`\n FROM aloja_logs.BWM2 {$where_BWM} AND iface_name = 'total' {$group_by_BWM};", 'fields' => array('MB_in', 'MB_out'), 'title' => "BW Monitor NG Total Bytes IN/OUT ({$aggr_text}, {$hosts}) {$exec_title}", 'group_title' => 'BW Monitor NG Total Bytes IN/OUT' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'bwm_packets_total' => array('metric' => "Network", 'query' => "SELECT time_to_sec(timediff(FROM_UNIXTIME(unix_timestamp),'{$exec_details[$exec]['start_time']}')) time,\n {$aggr}(`packets_in`) `packets_in`,\n {$aggr}(`packets_out`) `packets_out`\n FROM aloja_logs.BWM2 {$where_BWM} AND iface_name = 'total' {$group_by_BWM};", 'fields' => array('packets_in', 'packets_out'), 'title' => "BW Monitor NG Total packets IN/OUT ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'BW Monitor NG Total packets IN/OUT' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'bwm_errors_total' => array('metric' => "Network", 'query' => "SELECT time_to_sec(timediff(FROM_UNIXTIME(unix_timestamp),'{$exec_details[$exec]['start_time']}')) time,\n {$aggr}(`errors_in`) `errors_in`,\n {$aggr}(`errors_out`) `errors_out`\n FROM aloja_logs.BWM2 {$where_BWM} AND iface_name = 'total' {$group_by_BWM};", 'fields' => array('errors_in', 'errors_out'), 'title' => "BW Monitor NG Total errors IN/OUT ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'BW Monitor NG Total errors IN/OUT' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false));
$has_records = false;
//of any chart
foreach ($charts[$exec] as $key_type => $chart) {
if ($chart['metric'] == 'ALL' || $metric == $chart['metric']) {
$charts[$exec][$key_type]['chart'] = new HighCharts();
$charts[$exec][$key_type]['chart']->setTitle($chart['title']);
$charts[$exec][$key_type]['chart']->setPercentage($chart['percentage']);
$charts[$exec][$key_type]['chart']->setStacked($chart['stacked']);
$charts[$exec][$key_type]['chart']->setFields($chart['fields']);
$charts[$exec][$key_type]['chart']->setNegativeValues($chart['negative']);
list($rows, $max, $min) = Utils::minimize_exec_rows($dbUtil->get_rows($chart['query']), $chart['stacked']);
if (!isset($chart_details[$key_type]['max']) || $max > $chart_details[$key_type]['max']) {
$chart_details[$key_type]['max'] = $max;
}
if (!isset($chart_details[$key_type]['min']) || $min < $chart_details[$key_type]['min']) {
$chart_details[$key_type]['min'] = $min;
}
//$charts[$exec][$key_type]['chart']->setMax($max);
//$charts[$exec][$key_type]['chart']->setMin($min);
if (count($rows) > 0) {
$has_records = true;
$charts[$exec][$key_type]['chart']->setRows($rows);
}
}
}
}
if ($exec_details) {
$max_time = null;
foreach ($exec_details as $exec => $exe_time) {
if (!$max_time || $exe_time['time'] > $max_time) {
$max_time = $exe_time['time'];
}
}
foreach ($exec_details as $exec => $exe_time) {
#if (!$max_time) throw new Exception('Missing MAX time');
$exec_details[$exec]['size'] = round($exe_time['time'] / $max_time * 100, 2);
//TODO improve
$exec_details[$exec]['max_time'] = $max_time;
}
}
if (isset($has_records)) {
} else {
throw new \Exception("No results for query!");
}
} catch (\Exception $e) {
if (empty($execs)) {
$this->container->getTwig()->addGlobal('message', "No results for query!\n");
} else {
$this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n");
}
}
$chartsJS = '';
if ($charts) {
reset($charts);
$current_chart = current($charts);
foreach ($current_chart as $chart_type => $chart) {
foreach ($execs as $exec) {
if (isset($charts[$exec][$chart_type]['chart'])) {
//make Y axis all the same when comparing
$charts[$exec][$chart_type]['chart']->setMax($chart_details[$chart_type]['max']);
//the same for max X (plus 10%)
$charts[$exec][$chart_type]['chart']->setMaxX($exec_details[$exec]['max_time'] * 1.007);
//print the JS
$chartsJS .= $charts[$exec][$chart_type]['chart']->getChartJS() . "\n\n";
}
}
}
}
if (!isset($exec)) {
$exec = '';
}
return $this->render('perfDetailsViews/perfcharts.html.twig', array('title' => 'Hadoop Job/s Execution details and System Performance Charts', 'chartsJS' => $chartsJS, 'charts' => $charts, 'metric' => $metric, 'execs' => $execs, 'aggr' => $aggr, 'hosts' => $hosts, 'host_rows' => $dbUtil->get_hosts($clusters), 'detail' => $detail));
}
public static function generateCostsFilters($dbConnection)
{
$clustersInfo = $dbConnection->get_rows("SELECT id_cluster,cost_hour,cost_remote,cost_IB,cost_SSD FROM clusters");
foreach ($clustersInfo as $row) {
$costsHour[$row['id_cluster']] = $row['cost_hour'];
$costsRemote[$row['id_cluster']] = $row['cost_remote'];
$costsSSD[$row['id_cluster']] = $row['cost_IB'];
$costsIB[$row['id_cluster']] = $row['cost_SSD'];
}
//If form submitted, get given values and change those
if (isset($_GET['cost_hour'])) {
foreach (Utils::get_GET_intArray('cost_hour') as $idCluster => $value) {
$costsHour[$idCluster] = $value;
}
foreach (Utils::get_GET_intArray('cost_remote') as $idCluster => $value) {
$costsRemote[$idCluster] = $value;
}
foreach (Utils::get_GET_intArray('cost_IB') as $idCluster => $value) {
$costsSSD[$idCluster] = $value;
}
foreach (Utils::get_GET_intArray('cost_SSD') as $idCluster => $value) {
$costsIB[$idCluster] = $value;
}
}
return array('costsHour' => $costsHour, 'costsRemote' => $costsRemote, 'costsSSD' => $costsSSD, 'costsIB' => $costsIB);
}
public function hdp2CountersAction()
{
try {
$db = $this->container->getDBUtils();
$this->buildFilters(array('bench' => array('default' => null)));
$whereClause = $this->filters->getWhereClause();
$benchOptions = $db->get_rows("SELECT DISTINCT bench FROM aloja2.execs e JOIN aloja2.HDI_JOB_details USING (id_exec) WHERE valid = 1");
$discreteOptions = array();
$discreteOptions['bench'][] = 'All';
foreach ($benchOptions as $option) {
$discreteOptions['bench'][] = array_shift($option);
}
$dbUtil = $this->container->getDBUtils();
$message = null;
//check the URL
$execs = Utils::get_GET_intArray('execs');
if (Utils::get_GET_string('pageTab')) {
$type = Utils::get_GET_string('pageTab');
} else {
$type = 'SUMMARY';
}
$join = "JOIN aloja2.execs e using (id_exec) JOIN aloja2.clusters USING (id_cluster) LEFT JOIN aloja_ml.predictions p USING (id_exec) WHERE job_name NOT IN\n ('TeraGen', 'random-text-writer', 'mahout-examples-0.7-job.jar', 'Create pagerank nodes', 'Create pagerank links') {$whereClause}" . ($execs ? ' AND id_exec IN (' . join(',', $execs) . ') ' : '');
if (isset($_GET['jobid'])) {
$join .= " AND JOB_ID = '" . Utils::get_GET_string('jobid') . "' ";
}
$join .= " LIMIT 10000";
$query = "";
if ($type == 'SUMMARY') {
$query = "SELECT e.bench, e.exe_time, c.id_exec, c.JOB_ID, c.job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME, c.TOTAL_MAPS, c.FAILED_MAPS, c.FINISHED_MAPS, c.TOTAL_REDUCES, c.FAILED_REDUCES, c.job_name as CHARTS\n \t\t\tFROM aloja2.HDI_JOB_details c {$join}";
} else {
if ($type == "MAP") {
$query = "SELECT e.bench, e.exe_time, c.id_exec, JOB_ID, job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME, c.TOTAL_MAPS, c.FAILED_MAPS, c.FINISHED_MAPS, `TOTAL_LAUNCHED_MAPS`,\n \t\t\t`RACK_LOCAL_MAPS`,\n \t\t\t`SPILLED_RECORDS`,\n \t\t\t`MAP_INPUT_RECORDS`,\n \t\t\t`MAP_OUTPUT_RECORDS`,\n \t\t\t`MAP_OUTPUT_BYTES`,\n \t\t\t`MAP_OUTPUT_MATERIALIZED_BYTES`\n \t\t\tFROM aloja2.HDI_JOB_details c {$join}";
} else {
if ($type == 'REDUCE') {
$query = "SELECT e.bench, e.exe_time, c.id_exec, c.JOB_ID, c.job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME, c.TOTAL_REDUCES, c.FAILED_REDUCES,\n \t\t\t`TOTAL_LAUNCHED_REDUCES`,\n \t\t\t`REDUCE_INPUT_GROUPS`,\n \t\t\t`REDUCE_INPUT_RECORDS`,\n \t\t\t`REDUCE_OUTPUT_RECORDS`,\n \t\t\t`REDUCE_SHUFFLE_BYTES`,\n \t\t\t`COMBINE_INPUT_RECORDS`,\n \t\t\t`COMBINE_OUTPUT_RECORDS`\n \t\t\tFROM aloja2.HDI_JOB_details c {$join}";
} else {
if ($type == 'FILE-IO') {
$query = "SELECT e.bench, e.exe_time, c.id_exec, c.JOB_ID, c.job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME,\n \t\t\t`SLOTS_MILLIS_MAPS`,\n \t\t\t`SLOTS_MILLIS_REDUCES`,\n \t\t\t`SPLIT_RAW_BYTES`,\n \t\t\t`FILE_BYTES_WRITTEN`,\n \t\t\t`FILE_BYTES_READ`,\n \t\t\t`WASB_BYTES_WRITTEN`,\n \t\t\t`WASB_BYTES_READ`,\n \t\t\t`BYTES_READ`,\n \t\t\t`BYTES_WRITTEN`\n \t\t\tFROM aloja2.HDI_JOB_details c {$join}";
} else {
if ($type == 'DETAIL') {
$query = "SELECT e.bench, e.exe_time, c.* FROM aloja_logs.aloja2 c {$join}";
} else {
if ($type == "TASKS") {
$query = "SELECT e.bench, e.exe_time, j.job_name, c.* FROM aloja_logs.HDI_JOB_tasks c\n \t\t\tJOIN aloja2.HDI_JOB_details j USING(id_exec,JOB_ID) {$join} ";
// $taskStatusOptions = $db->get_rows("SELECT DISTINCT TASK_STATUS FROM HDI_JOB_tasks JOIN aloja2.execs USING (id_exec) WHERE valid = 1");
// $typeOptions = $db->get_rows("SELECT DISTINCT TASK_TYPE FROM HDI_JOB_tasks JOIN aloja2.execs USING (id_exec) WHERE valid = 1");
//
// $discreteOptions['TASK_STATUS'][] = 'All';
// $discreteOptions['TASK_TYPE'][] = 'All';
// foreach($taskStatusOptions as $option) {
// $discreteOptions['TASK_STATUS'][] = array_shift($option);
// }
// foreach($typeOptions as $option) {
// $discreteOptions['TASK_TYPE'][] = array_shift($option);
// }
$discreteOptions['TASK_STATUS'] = array('All', 'SUCCEEDED');
$discreteOptions['TASK_TYPE'] = array('All', 'MAP', 'REDUCE');
} else {
throw new \Exception('Unknown type!');
}
}
}
}
}
}
$exec_rows = $dbUtil->get_rows($query);
if (count($exec_rows) > 0) {
$show_in_result_counters = array('id_exec' => 'ID', 'JOB_ID' => 'JOBID', 'bench' => 'Bench', 'job_name' => 'JOBNAME');
$show_in_result_counters = Utils::generate_show($show_in_result_counters, $exec_rows, 4);
}
} catch (\Exception $e) {
$this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n");
}
return $this->render('repositoryViews/hdp2counters.html.twig', array('theaders' => isset($show_in_result_counters) ? $show_in_result_counters : array(), 'message' => $message, 'title' => 'Hadoop Jobs and Tasks Execution Counters', 'type' => $type, 'execs' => $execs, 'execsParam' => isset($_GET['execs']) ? Utils::get_GET_intArray('execs') : '', 'discreteOptions' => $discreteOptions, 'hdp2' => true));
}
public function hdp2CountersDataAction()
{
$db = $this->container->getDBUtils();
$this->buildFilters(array('bench' => array('table' => 'execs', 'default' => null, 'type' => 'selectMultiple')));
$whereClause = $this->filters->getWhereClause();
try {
//check the URL
$execs = Utils::get_GET_intArray('execs');
if (!($type = Utils::get_GET_string('pageTab'))) {
$type = 'SUMMARY';
}
$join = "JOIN aloja2.execs e using (id_exec) LEFT JOIN aloja_ml.predictions p USING (id_exec) WHERE e.valid = 1 AND job_name NOT IN\n ('TeraGen', 'random-text-writer', 'mahout-examples-0.7-job.jar', 'Create pagerank nodes', 'Create pagerank links') {$whereClause}" . ($execs ? ' AND id_exec IN (' . join(',', $execs) . ') ' : '') . " LIMIT 10000";
if ($type == 'SUMMARY') {
$query = "SELECT e.bench, e.exe_time, c.id_exec, c.JOB_ID, c.job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME, c.TOTAL_MAPS, c.FAILED_MAPS, c.FINISHED_MAPS, c.TOTAL_REDUCES, c.FAILED_REDUCES, c.job_name as CHARTS,\n \t\t\te.perf_details\n \t\t\tFROM aloja2.HDI_JOB_details c {$join}";
} elseif ($type == 'MAP') {
$query = "SELECT e.bench, e.exe_time, c.id_exec, JOB_ID, job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME, c.TOTAL_MAPS, c.FAILED_MAPS, c.FINISHED_MAPS, `TOTAL_LAUNCHED_MAPS`,\n \t\t\t`RACK_LOCAL_MAPS`,\n \t\t\t`SPILLED_RECORDS`,\n \t\t\t`MAP_INPUT_RECORDS`,\n \t\t\t`MAP_OUTPUT_RECORDS`,\n \t\t\t`MAP_OUTPUT_BYTES`,\n \t\t\t`MAP_OUTPUT_MATERIALIZED_BYTES`,\n \t\t\te.perf_details\n \t\t\tFROM aloja2.HDI_JOB_details c {$join}";
} elseif ($type == 'REDUCE') {
$query = "SELECT e.bench, e.exe_time, c.id_exec, c.JOB_ID, c.job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME, c.TOTAL_REDUCES, c.FAILED_REDUCES,\n \t\t\t`TOTAL_LAUNCHED_REDUCES`,\n \t\t\t`REDUCE_INPUT_GROUPS`,\n \t\t\t`REDUCE_INPUT_RECORDS`,\n \t\t\t`REDUCE_OUTPUT_RECORDS`,\n \t\t\t`REDUCE_SHUFFLE_BYTES`,\n \t\t\t`COMBINE_INPUT_RECORDS`,\n \t\t\t`COMBINE_OUTPUT_RECORDS`,\n \t\t\te.perf_details\n \t\t\tFROM aloja2.HDI_JOB_details c {$join}";
} elseif ($type == 'FILE-IO') {
$query = "SELECT e.bench, e.exe_time, c.id_exec, c.JOB_ID, c.job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME,\n \t\t\t`SLOTS_MILLIS_MAPS`,\n \t\t\t`SLOTS_MILLIS_REDUCES`,\n \t\t\t`SPLIT_RAW_BYTES`,\n \t\t\t`FILE_BYTES_WRITTEN`,\n \t\t\t`FILE_BYTES_READ`,\n \t\t\t`WASB_BYTES_WRITTEN`,\n \t\t\t`WASB_BYTES_READ`,\n \t\t\t`BYTES_READ`,\n \t\t\t`BYTES_WRITTEN`,\n \t\t\te.perf_details\n \t\t\tFROM aloja2.HDI_JOB_details c {$join}";
} elseif ($type == 'DETAIL') {
$query = "SELECT e.bench, e.exe_time, c.*,e.perf_details FROM aloja2.HDI_JOB_details c {$join}";
} elseif ($type == 'TASKS') {
$query = "SELECT e.bench, e.exe_time, j.job_name, c.*,e.perf_details FROM aloja_logs.HDI_JOB_tasks c\n \t\t\tJOIN aloja2.HDI_JOB_details j USING(id_exec,JOB_ID) {$join} ";
} else {
throw new \Exception('Unknown type!');
}
$exec_rows = $db->get_rows($query);
if (count($exec_rows) > 0) {
$show_in_result_counters = array('id_exec' => 'ID', 'JOB_ID' => 'JOBID', 'bench' => 'Bench', 'job_name' => 'JOBNAME');
$show_in_result_counters = Utils::generate_show($show_in_result_counters, $exec_rows, 4);
$jsonData = Utils::generateJSONTable($exec_rows, $show_in_result_counters, 0, 'COUNTER');
header('Content-Type: application/json');
echo json_encode(array('aaData' => $jsonData));
// if (count($exec_rows) > 10000) {
// $message .= 'WARNING, large resulset, please limit the query! Rows: '.count($exec_rows);
// }
} else {
echo 'No data available';
}
} catch (\Exception $e) {
exit($e->getMessage());
echo 'No data available';
/*$noData = array();
for($i = 0; $i<=sizeof($show_in_result); ++$i)
$noData[] = 'error';
echo json_encode(array('aaData' => array($noData)));*/
}
}