public function getFilterOptions($dbUtils) { $options['bench'] = $dbUtils->get_rows("SELECT DISTINCT bench FROM aloja2.execs e WHERE valid = 1 AND filter = 0 " . DBUtils::getFilterExecs() . " ORDER BY bench ASC"); $options['net'] = $dbUtils->get_rows("SELECT DISTINCT net FROM aloja2.execs e WHERE valid = 1 AND filter = 0 " . DBUtils::getFilterExecs() . " ORDER BY net ASC"); $options['disk'] = $dbUtils->get_rows("SELECT DISTINCT disk FROM aloja2.execs e WHERE valid = 1 AND filter = 0 " . DBUtils::getFilterExecs() . " ORDER BY disk ASC"); $options['blk_size'] = $dbUtils->get_rows("SELECT DISTINCT blk_size FROM aloja2.execs e WHERE valid = 1 AND filter = 0 " . DBUtils::getFilterExecs() . " ORDER BY blk_size ASC"); $options['comp'] = $dbUtils->get_rows("SELECT DISTINCT comp FROM aloja2.execs e WHERE valid = 1 AND filter = 0 " . DBUtils::getFilterExecs() . " ORDER BY comp ASC"); $options['id_cluster'] = $dbUtils->get_rows("select distinct id_cluster,CONCAT_WS('/',LPAD(id_cluster,2,0),c.vm_size,CONCAT(c.datanodes,'Dn')) as name from aloja2.execs e JOIN aloja2.clusters c using (id_cluster) WHERE valid = 1 AND filter = 0 " . DBUtils::getFilterExecs() . " ORDER BY c.name ASC"); $options['maps'] = $dbUtils->get_rows("SELECT DISTINCT maps FROM aloja2.execs e WHERE valid = 1 AND filter = 0 " . DBUtils::getFilterExecs() . " ORDER BY maps ASC"); $options['replication'] = $dbUtils->get_rows("SELECT DISTINCT replication FROM aloja2.execs e WHERE valid = 1 AND filter = 0 " . DBUtils::getFilterExecs() . " ORDER BY replication ASC"); $options['iosf'] = $dbUtils->get_rows("SELECT DISTINCT iosf FROM aloja2.execs e WHERE valid = 1 AND filter = 0 " . DBUtils::getFilterExecs() . " ORDER BY iosf ASC"); $options['iofilebuf'] = $dbUtils->get_rows("SELECT DISTINCT iofilebuf FROM aloja2.execs e WHERE valid = 1 AND filter = 0 " . DBUtils::getFilterExecs() . " ORDER BY iofilebuf ASC"); $options['datanodes'] = $dbUtils->get_rows("SELECT DISTINCT datanodes FROM aloja2.execs e JOIN aloja2.clusters USING (id_cluster) WHERE valid = 1 AND filter = 0 " . DBUtils::getFilterExecs() . " ORDER BY datanodes ASC"); $options['benchtype'] = $dbUtils->get_rows("SELECT DISTINCT bench_type FROM aloja2.execs e WHERE valid = 1 AND filter = 0 " . DBUtils::getFilterExecs() . " ORDER BY bench_type ASC"); $options['vm_size'] = $dbUtils->get_rows("SELECT DISTINCT vm_size FROM aloja2.execs e JOIN aloja2.clusters c USING (id_cluster) WHERE valid = 1 AND filter = 0 " . DBUtils::getFilterExecs() . " ORDER BY vm_size ASC"); $options['vm_cores'] = $dbUtils->get_rows("SELECT DISTINCT vm_cores FROM aloja2.execs e JOIN aloja2.clusters c USING (id_cluster) WHERE valid = 1 AND filter = 0 " . DBUtils::getFilterExecs() . " ORDER BY vm_cores ASC"); $options['vm_RAM'] = $dbUtils->get_rows("SELECT DISTINCT vm_RAM FROM aloja2.execs e JOIN aloja2.clusters c USING (id_cluster) WHERE valid = 1 AND filter = 0 " . DBUtils::getFilterExecs() . " ORDER BY vm_RAM ASC"); $options['hadoop_version'] = $dbUtils->get_rows("SELECT DISTINCT hadoop_version FROM aloja2.execs e WHERE valid = 1 AND filter = 0 " . DBUtils::getFilterExecs() . " ORDER BY hadoop_version ASC"); $options['type'] = $dbUtils->get_rows("SELECT DISTINCT type FROM aloja2.execs e JOIN aloja2.clusters c USING (id_cluster) WHERE valid = 1 AND filter = 0 " . DBUtils::getFilterExecs() . " ORDER BY type ASC"); $options['presets'] = $dbUtils->get_rows("SELECT * FROM aloja2.filter_presets ORDER BY short_name DESC"); $options['provider'] = $dbUtils->get_rows("SELECT DISTINCT provider FROM aloja2.execs e JOIN aloja2.clusters c USING (id_cluster) WHERE valid = 1 AND filter = 0 " . DBUtils::getFilterExecs() . " ORDER BY provider DESC;"); $options['vm_OS'] = $dbUtils->get_rows("SELECT DISTINCT vm_OS FROM aloja2.execs e JOIN aloja2.clusters c USING (id_cluster) WHERE valid = 1 AND filter = 0 " . DBUtils::getFilterExecs() . " ORDER BY vm_OS DESC;"); $options['datasize'] = $dbUtils->get_rows("SELECT DISTINCT datasize FROM aloja2.execs e JOIN aloja2.clusters c USING (id_cluster) WHERE valid = 1 AND filter = 0 " . DBUtils::getFilterExecs() . " ORDER BY datasize ASC;"); $options['scale_factor'] = $dbUtils->get_rows("SELECT DISTINCT scale_factor FROM aloja2.execs e JOIN aloja2.clusters c USING (id_cluster) WHERE valid = 1 AND filter = 0 " . DBUtils::getFilterExecs() . " ORDER BY scale_factor ASC;"); return $options; }
public function get_execs($filter_execs = null) { if ($filter_execs === null) { $filter_execs = DBUtils::getFilterExecs(); } $query = "SELECT e.*, (exe_time/3600)*(cost_hour) cost, name cluster_name, datanodes FROM execs e\n join clusters c USING (id_cluster)\n WHERE bench_type not like 'HDI-prep%' AND bench not like 'prep_%' AND valid = 1 AND filter = 0 {$filter_execs};"; return $this->get_rows($query); }
public function get_execs($filter_execs = null) { if ($filter_execs === null) { $filter_execs = DBUtils::getFilterExecs(); } $query = "SELECT e.*, (exe_time/3600)*(cost_hour) cost, name cluster_name, datanodes FROM aloja2.execs e\n JOIN aloja2.clusters c USING (id_cluster)\n WHERE 1 {$filter_execs};"; return $this->get_rows($query); }
public function clustersAction() { $clusterNameSelected = null; if (isset($_GET['cluster_name'])) { $clusterNameSelected = Utils::get_GET_string('cluster_name'); } $filter_execs = DBUtils::getFilterExecs(); $db = $this->container->getDBUtils(); $clusters = $db->get_rows("SELECT * FROM aloja2.clusters c WHERE id_cluster IN (SELECT distinct(id_cluster) FROM aloja2.execs e WHERE 1 {$filter_execs});"); echo $this->container->getTwig()->render('defaultViews/clusters.html.twig', array('selected' => 'Clusters', 'clusters' => $clusters, 'clusterNameSelected' => $clusterNameSelected, 'title' => 'ALOJA Clusters')); }
private function processExtraData() { //Getting option to tell JS what to filter on rendering $benchsDatasize = $this->dbConnection->get_rows("SELECT DISTINCT bench_type,bench,datasize FROM aloja2.execs e WHERE 1 AND valid = 1 AND filter = 0 " . DBUtils::getFilterExecs() . " GROUP BY bench_type,bench,datasize ORDER BY bench ASC "); $dataBenchs = array(); $availDatasizes = array(); foreach ($benchsDatasize as $row) { $datasize = $this->roundDatasize($row['datasize']); if (!isset($availDatasizes[$row['bench_type']]) || !isset($availDatasizes[$row['bench_type']][$row['bench']]) || !in_array($datasize, $availDatasizes[$row['bench_type']][$row['bench']])) { $dataBenchs[$row['bench_type']][$row['bench']][] = $row['datasize']; $availDatasizes[$row['bench_type']][$row['bench']][] = $datasize; } } $this->additionalFilters['datasizesInfo'] = json_encode($dataBenchs); //Getting scale factors per bench $scaleFactors = array(); $benchsScaleFactors = $this->dbConnection->get_rows("SELECT DISTINCT bench_type,bench,scale_factor FROM aloja2.execs e WHERE 1 AND valid = 1 AND filter = 0 " . DBUtils::getFilterExecs() . " GROUP BY bench_type,bench,datasize ORDER BY bench ASC "); foreach ($benchsScaleFactors as $row) { $scaleFactor = $row['scale_factor']; $scaleFactors[$row['bench_type']][$row['bench']][] = $scaleFactor; } $this->additionalFilters['scaleFactorsInfo'] = json_encode($scaleFactors); //Getting providers / clusters $providerClusters = array(); $clusters = $this->dbConnection->get_rows("SELECT provider,id_cluster FROM aloja2.clusters ORDER BY provider DESC "); foreach ($clusters as $row) { $providerClusters[$row['provider']][] = $row['id_cluster']; } $this->additionalFilters['providerClusters'] = json_encode($providerClusters); }
public function performanceChartsAction() { $exec_rows = null; $id_exec_rows = null; $dbUtil = $this->container->getDBUtils(); $this->buildFilters(array('perf_details' => array('default' => 1))); $charts = array(); $clusters = array(); try { //TODO fix, initialize variables $dbUtil->get_exec_details('1', 'id_exec', $exec_rows, $id_exec_rows); //check the URL $execs = Utils::get_GET_intArray('execs'); if (empty($execs)) { $whereClause = $this->filters->getWhereClause(); $query = "SELECT e.id_exec FROM aloja2.execs e JOIN aloja2.clusters c USING (id_cluster)\n LEFT JOIN aloja_ml.predictions p USING (id_exec)\n WHERE 1 " . DBUtils::getFilterExecs() . "{$whereClause} "; $query .= isset($_GET['random']) ? '' : 'LIMIT 1'; $idExecs = $dbUtil->get_rows($query); if (isset($_GET['random'])) { $execs = array($idExecs[rand(0, sizeof($idExecs) - 1)]['id_exec']); } else { $execs[] = $idExecs[0]['id_exec']; } } if (Utils::get_GET_string('random') && !$execs) { $keys = array_keys($exec_rows); $execs = array_unique(array($keys[array_rand($keys)], $keys[array_rand($keys)])); } if (Utils::get_GET_string('hosts')) { $hosts = Utils::get_GET_string('hosts'); } else { $hosts = 'Slaves'; } if (Utils::get_GET_string('metric')) { $metric = Utils::get_GET_string('metric'); } else { $metric = 'CPU'; } if (Utils::get_GET_string('aggr')) { $aggr = Utils::get_GET_string('aggr'); } else { $aggr = 'AVG'; } if (Utils::get_GET_string('detail')) { $detail = Utils::get_GET_int('detail'); } else { $detail = 10; } if ($aggr == 'AVG') { $aggr_text = "Average"; } elseif ($aggr == 'SUM') { $aggr_text = "SUM"; } else { throw new \Exception("Aggregation type '{$aggr}' is not valid."); } if ($hosts == 'Slaves') { $selectedHosts = $dbUtil->get_rows("SELECT h.host_name from execs e inner join hosts h where e.id_exec IN (" . implode(", ", $execs) . ") AND h.id_cluster = e.id_cluster AND h.role='slave'"); $selected_hosts = array(); foreach ($selectedHosts as $host) { array_push($selected_hosts, $host['host_name']); } } elseif ($hosts == 'Master') { $selectedHosts = $dbUtil->get_rows("SELECT h.host_name from execs e inner join hosts h where e.id_exec IN (" . implode(", ", $execs) . ") AND h.id_cluster = e.id_cluster AND h.role='master' AND h.host_name != ''"); $selected_hosts = array(); foreach ($selectedHosts as $host) { array_push($selected_hosts, $host['host_name']); } } else { $selected_hosts = array($hosts); } $charts = array(); $exec_details = array(); $chart_details = array(); $clusters = array(); foreach ($execs as $exec) { //do a security check $tmp = filter_var($exec, FILTER_SANITIZE_NUMBER_INT); if (!is_numeric($tmp) || !($tmp > 0)) { unset($execs[$exec]); continue; } $exec_title = $dbUtil->get_exec_details($exec, 'exec', $exec_rows, $id_exec_rows); $pos_name = strpos($exec_title, '/'); $exec_title = ' ' . strtoupper(substr($exec_title, $pos_name + 1)) . ' ' . (strpos($exec_title, '_az') > 0 ? 'AZURE' : 'LOCAL') . " ID_{$exec} " . substr($exec_title, 21, strlen($exec_title) - $pos_name - (strpos($exec_title, '_az') > 0 ? 21 : 18)); $exec_details[$exec]['time'] = $dbUtil->get_exec_details($exec, 'exe_time', $exec_rows, $id_exec_rows); $exec_details[$exec]['start_time'] = $dbUtil->get_exec_details($exec, 'start_time', $exec_rows, $id_exec_rows); $exec_details[$exec]['end_time'] = $dbUtil->get_exec_details($exec, 'end_time', $exec_rows, $id_exec_rows); $id_cluster = $dbUtil->get_exec_details($exec, 'id_cluster', $exec_rows, $id_exec_rows); if (!in_array($id_cluster, $clusters)) { $clusters[] = $id_cluster; } //$end_time = get_exec_details($exec, 'init_time'); $date_where = " AND date BETWEEN '{$exec_details[$exec]['start_time']}' and '{$exec_details[$exec]['end_time']}' "; $where = " WHERE id_exec = '{$exec}' AND host IN ('" . join("','", $selected_hosts) . "') {$date_where}"; $where_BWM = " WHERE id_exec = '{$exec}' AND host IN ('" . join("','", $selected_hosts) . "') "; $where_VMSTATS = " WHERE id_exec = '{$exec}' AND host IN ('" . join("','", $selected_hosts) . "') "; $where_sampling = "round(time/{$detail})"; $group_by = " GROUP BY {$where_sampling} ORDER by time"; $group_by_vmstats = " GROUP BY {$where_sampling} ORDER by time"; $where_sampling_BWM = "round(unix_timestamp/{$detail})"; $group_by_BWM = " GROUP BY {$where_sampling_BWM} ORDER by unix_timestamp"; $charts[$exec] = array('job_status' => array('metric' => "ALL", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time,\n maps map,shuffle,merge,reduce,waste FROM aloja_logs.JOB_status\n WHERE id_exec = '{$exec}' {$date_where} GROUP BY job_name, date ORDER by job_name, time;", 'fields' => array('map', 'shuffle', 'reduce', 'waste', 'merge'), 'title' => "Job execution history {$exec_title} ", 'group_title' => 'Job execution history (number of running Hadoop processes)', 'percentage' => false, 'stacked' => false, 'negative' => false), 'cpu' => array('metric' => "CPU", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(`%user`) `%user`, {$aggr}(`%system`) `%system`, {$aggr}(`%steal`) `%steal`, {$aggr}(`%iowait`)\n `%iowait`, {$aggr}(`%nice`) `%nice` FROM aloja_logs.SAR_cpu {$where} {$group_by};", 'fields' => array('%user', '%system', '%steal', '%iowait', '%nice'), 'title' => "CPU Utilization ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'CPU Utilization ' . "({$aggr_text}, {$hosts})", 'percentage' => $aggr == 'SUM' ? '300' : 100, 'stacked' => true, 'negative' => false), 'load' => array('metric' => "CPU", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(`ldavg-1`) `ldavg-1`, {$aggr}(`ldavg-5`) `ldavg-5`, {$aggr}(`ldavg-15`) `ldavg-15`\n FROM aloja_logs.SAR_load {$where} {$group_by};", 'fields' => array('ldavg-15', 'ldavg-5', 'ldavg-1'), 'title' => "CPU Load Average ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'CPU Load Average ' . "({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'load_queues' => array('metric' => "CPU", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(`runq-sz`) `runq-sz`, {$aggr}(`blocked`) `blocked`\n FROM aloja_logs.SAR_load {$where} {$group_by};", 'fields' => array('runq-sz', 'blocked'), 'title' => "CPU Queues ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'CPU Queues ' . "({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'load_tasks' => array('metric' => "CPU", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(`plist-sz`) `plist-sz` FROM aloja_logs.SAR_load {$where} {$group_by};", 'fields' => array('plist-sz'), 'title' => "Number of tasks for CPUs ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'Number of tasks for CPUs ' . "({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'switches' => array('metric' => "CPU", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(`proc/s`) `proc/s`, {$aggr}(`cswch/s`) `cswch/s` FROM aloja_logs.SAR_switches {$where} {$group_by};", 'fields' => array('proc/s', 'cswch/s'), 'title' => "CPU Context Switches ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'CPU Context Switches' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'interrupts' => array('metric' => "CPU", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(`intr/s`) `intr/s` FROM aloja_logs.SAR_interrupts {$where} {$group_by};", 'fields' => array('intr/s'), 'title' => "CPU Interrupts ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'CPU Interrupts ' . "({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'memory_util' => array('metric' => "Memory", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(kbmemfree)*1024 kbmemfree, {$aggr}(kbmemused)*1024 kbmemused\n FROM aloja_logs.SAR_memory_util {$where} {$group_by};", 'fields' => array('kbmemfree', 'kbmemused'), 'title' => "Memory Utilization ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'Memory Utilization' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => true, 'negative' => false), 'memory_util_det' => array('metric' => "Memory", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(kbbuffers)*1024 kbbuffers, {$aggr}(kbcommit)*1024 kbcommit, {$aggr}(kbcached)*1024 kbcached,\n {$aggr}(kbactive)*1024 kbactive, {$aggr}(kbinact)*1024 kbinact\n FROM aloja_logs.SAR_memory_util {$where} {$group_by};", 'fields' => array('kbcached', 'kbbuffers', 'kbinact', 'kbcommit', 'kbactive'), 'title' => "Memory Utilization Details ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'Memory Utilization Details' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => true, 'negative' => false), 'memory' => array('metric' => "Memory", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(`frmpg/s`) `frmpg/s`, {$aggr}(`bufpg/s`) `bufpg/s`, {$aggr}(`campg/s`) `campg/s`\n FROM aloja_logs.SAR_memory {$where} {$group_by};", 'fields' => array('frmpg/s', 'bufpg/s', 'campg/s'), 'title' => "Memory Stats ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'Memory Stats' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'io_pagging_disk' => array('metric' => "Memory", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(`pgpgin/s`)*1024 `pgpgin/s`, {$aggr}(`pgpgout/s`)*1024 `pgpgout/s`\n FROM aloja_logs.SAR_io_paging {$where} {$group_by};", 'fields' => array('pgpgin/s', 'pgpgout/s'), 'title' => "I/O Paging IN/OUT to disk ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'I/O Paging IN/OUT to disk' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'io_pagging' => array('metric' => "Memory", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(`fault/s`) `fault/s`, {$aggr}(`majflt/s`) `majflt/s`, {$aggr}(`pgfree/s`) `pgfree/s`,\n {$aggr}(`pgscank/s`) `pgscank/s`, {$aggr}(`pgscand/s`) `pgscand/s`, {$aggr}(`pgsteal/s`) `pgsteal/s`\n FROM aloja_logs.SAR_io_paging {$where} {$group_by};", 'fields' => array('fault/s', 'majflt/s', 'pgfree/s', 'pgscank/s', 'pgscand/s', 'pgsteal/s'), 'title' => "I/O Paging ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'I/O Paging' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'io_pagging_vmeff' => array('metric' => "Memory", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(`%vmeff`) `%vmeff` FROM aloja_logs.SAR_io_paging {$where} {$group_by};", 'fields' => array('%vmeff'), 'title' => "I/O Paging %vmeff ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'I/O Paging %vmeff' . " ({$aggr_text}, {$hosts})", 'percentage' => $aggr == 'SUM' ? '300' : 100, 'stacked' => false, 'negative' => false), 'io_transactions' => array('metric' => "Disk", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(`tps`) `tp/s`, {$aggr}(`rtps`) `read tp/s`, {$aggr}(`wtps`) `write tp/s`\n FROM aloja_logs.SAR_io_rate {$where} {$group_by};", 'fields' => array('tp/s', 'read tp/s', 'write tp/s'), 'title' => "I/O Transactions/s ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'I/O Transactions/s' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'io_bytes' => array('metric' => "Disk", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(`bread/s`)/(1024) `KB_read/s`, {$aggr}(`bwrtn/s`)/(1024) `KB_wrtn/s`\n FROM aloja_logs.SAR_io_rate {$where} {$group_by};", 'fields' => array('KB_read/s', 'KB_wrtn/s'), 'title' => "KB R/W ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'KB R/W' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'block_devices_util' => array('metric' => "Disk", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(`%util_SUM`) `%util_SUM`, {$aggr}(`%util_MAX`) `%util_MAX`\n FROM (\n select\n id_exec, host, date,\n sum(`%util`) `%util_SUM`,\n max(`%util`) `%util_MAX`\n from aloja_logs.SAR_block_devices d WHERE id_exec = '{$exec}'\n GROUP BY date, host\n ) t {$where} {$group_by};", 'fields' => array('%util_SUM', '%util_MAX'), 'title' => "Disk Uitlization percentage (All DEVs, {$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'Disk Uitlization percentage' . " (All DEVs, {$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'block_devices_await' => array('metric' => "Disk", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(`await_SUM`) `await_SUM`, {$aggr}(`await_MAX`) `await_MAX`\n FROM (\n select\n id_exec, host, date,\n sum(`await`) `await_SUM`,\n max(`await`) `await_MAX`\n from aloja_logs.SAR_block_devices d WHERE id_exec = '{$exec}'\n GROUP BY date, host\n ) t {$where} {$group_by};", 'fields' => array('await_SUM', 'await_MAX'), 'title' => "Disk request wait time in ms (All DEVs, {$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'Disk request wait time in ms' . " (All DEVs, {$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'block_devices_svctm' => array('metric' => "Disk", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(`svctm_SUM`) `svctm_SUM`, {$aggr}(`svctm_MAX`) `svctm_MAX`\n FROM (\n select\n id_exec, host, date,\n sum(`svctm`) `svctm_SUM`,\n max(`svctm`) `svctm_MAX`\n from aloja_logs.SAR_block_devices d WHERE id_exec = '{$exec}'\n GROUP BY date, host\n ) t {$where} {$group_by};", 'fields' => array('svctm_SUM', 'svctm_MAX'), 'title' => "Disk service time in ms (All DEVs, {$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'Disk service time in ms' . " (All DEVs, {$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'block_devices_queues' => array('metric' => "Disk", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(`avgrq-sz`) `avg-req-size`, {$aggr}(`avgqu-sz`) `avg-queue-size`\n FROM (\n select\n id_exec, host, date,\n max(`avgrq-sz`) `avgrq-sz`,\n max(`avgqu-sz`) `avgqu-sz`\n from aloja_logs.SAR_block_devices d WHERE id_exec = '{$exec}'\n GROUP BY date, host\n ) t {$where} {$group_by};", 'fields' => array('avg-req-size', 'avg-queue-size'), 'title' => "Disk req and queue sizes ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'Disk req and queue sizes' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'vmstats_io' => array('metric' => "Disk", 'query' => "SELECT time, {$aggr}(`bi`)/(1024) `KB_IN`, {$aggr}(`bo`)/(1024) `KB_OUT`\n FROM aloja_logs.VMSTATS {$where_VMSTATS} {$group_by_vmstats};", 'fields' => array('KB_IN', 'KB_OUT'), 'title' => "VMSTATS KB I/O ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'VMSTATS KB I/O' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'vmstats_rb' => array('metric' => "CPU", 'query' => "SELECT time, {$aggr}(`r`) `runnable procs`, {$aggr}(`b`) `sleep procs` FROM aloja_logs.VMSTATS {$where_VMSTATS} {$group_by_vmstats};", 'fields' => array('runnable procs', 'sleep procs'), 'title' => "VMSTATS Processes (r-b) ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'VMSTATS Processes (r-b)' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'vmstats_memory' => array('metric' => "Memory", 'query' => "SELECT time, {$aggr}(`buff`) `buff`,\n {$aggr}(`cache`) `cache`,\n {$aggr}(`free`) `free`,\n {$aggr}(`swpd`) `swpd`\n FROM aloja_logs.VMSTATS {$where_VMSTATS} {$group_by_vmstats};", 'fields' => array('buff', 'cache', 'free', 'swpd'), 'title' => "VMSTATS Processes (r-b) ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'VMSTATS Processes (r-b)' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => true, 'negative' => false), 'net_devices_kbs' => array('metric' => "Network", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(if(IFACE != 'lo', `rxkB/s`, NULL))/1024 `rxMB/s_NET`, {$aggr}(if(IFACE != 'lo', `txkB/s`, NULL))/1024 `txMB/s_NET`\n FROM aloja_logs.SAR_net_devices {$where} AND IFACE not IN ('') {$group_by};", 'fields' => array('rxMB/s_NET', 'txMB/s_NET'), 'title' => "MB/s received and transmitted ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'MB/s received and transmitted' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'net_devices_kbs_local' => array('metric' => "Network", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(if(IFACE = 'lo', `rxkB/s`, NULL))/1024 `rxMB/s_LOCAL`, {$aggr}(if(IFACE = 'lo', `txkB/s`, NULL))/1024 `txMB/s_LOCAL`\n FROM aloja_logs.SAR_net_devices {$where} AND IFACE not IN ('') {$group_by};", 'fields' => array('rxMB/s_LOCAL', 'txMB/s_LOCAL'), 'title' => "MB/s received and transmitted LOCAL ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'MB/s received and transmitted LOCAL' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'net_devices_pcks' => array('metric' => "Network", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(if(IFACE != 'lo', `rxpck/s`, NULL))/1024 `rxpck/s_NET`, {$aggr}(if(IFACE != 'lo', `txkB/s`, NULL))/1024 `txpck/s_NET`\n FROM aloja_logs.SAR_net_devices {$where} AND IFACE not IN ('') {$group_by};", 'fields' => array('rxpck/s_NET', 'txpck/s_NET'), 'title' => "Packets/s received and transmitted ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'Packets/s received and transmitted' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'net_devices_pcks_local' => array('metric' => "Network", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(if(IFACE = 'lo', `rxkB/s`, NULL))/1024 `rxpck/s_LOCAL`, {$aggr}(if(IFACE = 'lo', `txkB/s`, NULL))/1024 `txpck/s_LOCAL`\n FROM aloja_logs.SAR_net_devices {$where} AND IFACE not IN ('') {$group_by};", 'fields' => array('rxpck/s_LOCAL', 'txpck/s_LOCAL'), 'title' => "Packets/s received and transmitted LOCAL ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'Packets/s received and transmitted LOCAL' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'net_sockets_pcks' => array('metric' => "Network", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(`totsck`) `totsck`,\n {$aggr}(`tcpsck`) `tcpsck`,\n {$aggr}(`udpsck`) `udpsck`,\n {$aggr}(`rawsck`) `rawsck`,\n {$aggr}(`ip-frag`) `ip-frag`,\n {$aggr}(`tcp-tw`) `tcp-time-wait`\n FROM aloja_logs.SAR_net_sockets {$where} {$group_by};", 'fields' => array('totsck', 'tcpsck', 'udpsck', 'rawsck', 'ip-frag', 'tcp-time-wait'), 'title' => "Packets/s received and transmitted ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'Packets/s received and transmitted' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'net_erros' => array('metric' => "Network", 'query' => "SELECT time_to_sec(timediff(date, '{$exec_details[$exec]['start_time']}')) time, {$aggr}(`rxerr/s`) `rxerr/s`,\n {$aggr}(`txerr/s`) `txerr/s`,\n {$aggr}(`coll/s`) `coll/s`,\n {$aggr}(`rxdrop/s`) `rxdrop/s`,\n {$aggr}(`txdrop/s`) `txdrop/s`,\n {$aggr}(`txcarr/s`) `txcarr/s`,\n {$aggr}(`rxfram/s`) `rxfram/s`,\n {$aggr}(`rxfifo/s`) `rxfifo/s`,\n {$aggr}(`txfifo/s`) `txfifo/s`\n FROM aloja_logs.SAR_net_errors {$where} {$group_by};", 'fields' => array('rxerr/s', 'txerr/s', 'coll/s', 'rxdrop/s', 'txdrop/s', 'txcarr/s', 'rxfram/s', 'rxfifo/s', 'txfifo/s'), 'title' => "Network errors ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'Network errors' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'bwm_in_out_total' => array('metric' => "Network", 'query' => "SELECT time_to_sec(timediff(FROM_UNIXTIME(unix_timestamp),'{$exec_details[$exec]['start_time']}')) time,\n {$aggr}(`bytes_in`)/(1024*1024) `MB_in`,\n {$aggr}(`bytes_out`)/(1024*1024) `MB_out`\n FROM aloja_logs.BWM2 {$where_BWM} AND iface_name = 'total' {$group_by_BWM};", 'fields' => array('MB_in', 'MB_out'), 'title' => "BW Monitor NG Total Bytes IN/OUT ({$aggr_text}, {$hosts}) {$exec_title}", 'group_title' => 'BW Monitor NG Total Bytes IN/OUT' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'bwm_packets_total' => array('metric' => "Network", 'query' => "SELECT time_to_sec(timediff(FROM_UNIXTIME(unix_timestamp),'{$exec_details[$exec]['start_time']}')) time,\n {$aggr}(`packets_in`) `packets_in`,\n {$aggr}(`packets_out`) `packets_out`\n FROM aloja_logs.BWM2 {$where_BWM} AND iface_name = 'total' {$group_by_BWM};", 'fields' => array('packets_in', 'packets_out'), 'title' => "BW Monitor NG Total packets IN/OUT ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'BW Monitor NG Total packets IN/OUT' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false), 'bwm_errors_total' => array('metric' => "Network", 'query' => "SELECT time_to_sec(timediff(FROM_UNIXTIME(unix_timestamp),'{$exec_details[$exec]['start_time']}')) time,\n {$aggr}(`errors_in`) `errors_in`,\n {$aggr}(`errors_out`) `errors_out`\n FROM aloja_logs.BWM2 {$where_BWM} AND iface_name = 'total' {$group_by_BWM};", 'fields' => array('errors_in', 'errors_out'), 'title' => "BW Monitor NG Total errors IN/OUT ({$aggr_text}, {$hosts}) {$exec_title} ", 'group_title' => 'BW Monitor NG Total errors IN/OUT' . " ({$aggr_text}, {$hosts})", 'percentage' => false, 'stacked' => false, 'negative' => false)); $has_records = false; //of any chart foreach ($charts[$exec] as $key_type => $chart) { if ($chart['metric'] == 'ALL' || $metric == $chart['metric']) { $charts[$exec][$key_type]['chart'] = new HighCharts(); $charts[$exec][$key_type]['chart']->setTitle($chart['title']); $charts[$exec][$key_type]['chart']->setPercentage($chart['percentage']); $charts[$exec][$key_type]['chart']->setStacked($chart['stacked']); $charts[$exec][$key_type]['chart']->setFields($chart['fields']); $charts[$exec][$key_type]['chart']->setNegativeValues($chart['negative']); list($rows, $max, $min) = Utils::minimize_exec_rows($dbUtil->get_rows($chart['query']), $chart['stacked']); if (!isset($chart_details[$key_type]['max']) || $max > $chart_details[$key_type]['max']) { $chart_details[$key_type]['max'] = $max; } if (!isset($chart_details[$key_type]['min']) || $min < $chart_details[$key_type]['min']) { $chart_details[$key_type]['min'] = $min; } //$charts[$exec][$key_type]['chart']->setMax($max); //$charts[$exec][$key_type]['chart']->setMin($min); if (count($rows) > 0) { $has_records = true; $charts[$exec][$key_type]['chart']->setRows($rows); } } } } if ($exec_details) { $max_time = null; foreach ($exec_details as $exec => $exe_time) { if (!$max_time || $exe_time['time'] > $max_time) { $max_time = $exe_time['time']; } } foreach ($exec_details as $exec => $exe_time) { #if (!$max_time) throw new Exception('Missing MAX time'); $exec_details[$exec]['size'] = round($exe_time['time'] / $max_time * 100, 2); //TODO improve $exec_details[$exec]['max_time'] = $max_time; } } if (isset($has_records)) { } else { throw new \Exception("No results for query!"); } } catch (\Exception $e) { if (empty($execs)) { $this->container->getTwig()->addGlobal('message', "No results for query!\n"); } else { $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n"); } } $chartsJS = ''; if ($charts) { reset($charts); $current_chart = current($charts); foreach ($current_chart as $chart_type => $chart) { foreach ($execs as $exec) { if (isset($charts[$exec][$chart_type]['chart'])) { //make Y axis all the same when comparing $charts[$exec][$chart_type]['chart']->setMax($chart_details[$chart_type]['max']); //the same for max X (plus 10%) $charts[$exec][$chart_type]['chart']->setMaxX($exec_details[$exec]['max_time'] * 1.007); //print the JS $chartsJS .= $charts[$exec][$chart_type]['chart']->getChartJS() . "\n\n"; } } } } if (!isset($exec)) { $exec = ''; } return $this->render('perfDetailsViews/perfcharts.html.twig', array('title' => 'Hadoop Job/s Execution details and System Performance Charts', 'chartsJS' => $chartsJS, 'charts' => $charts, 'metric' => $metric, 'execs' => $execs, 'aggr' => $aggr, 'hosts' => $hosts, 'host_rows' => $dbUtil->get_hosts($clusters), 'detail' => $detail)); }
public function mlparamEvaluationAction() { $rows = $categories = $series = ''; $must_wait = 'NO'; try { $dbml = new \PDO($this->container->get('config')['db_conn_chain_ml'], $this->container->get('config')['mysql_user'], $this->container->get('config')['mysql_pwd']); $dbml->setAttribute(\PDO::ATTR_ERRMODE, \PDO::ERRMODE_EXCEPTION); $dbml->setAttribute(\PDO::ATTR_EMULATE_PREPARES, false); $db = $this->container->getDBUtils(); $where_configs = ''; $preset = null; if (count($_GET) <= 1 || count($_GET) == 2 && array_key_exists('parameval', $_GET) || count($_GET) == 2 && array_key_exists('current_model', $_GET)) { $preset = Utils::setDefaultPreset($db, 'mlparameval'); } $selPreset = isset($_GET['presets']) ? $_GET['presets'] : "none"; $params = array(); $param_names = array('benchs', 'nets', 'disks', 'mapss', 'iosfs', 'replications', 'iofilebufs', 'comps', 'blk_sizes', 'id_clusters', 'datanodess', 'bench_types', 'vm_sizes', 'vm_coress', 'vm_RAMs', 'types'); // Order is important foreach ($param_names as $p) { $params[$p] = Utils::read_params($p, $where_configs, FALSE); sort($params[$p]); } $money = Utils::read_params('money', $where_configs); $paramEval = isset($_GET['parameval']) && $_GET['parameval'] != '' ? $_GET['parameval'] : 'maps'; $minExecs = isset($_GET['minexecs']) ? $_GET['minexecs'] : -1; $minExecsFilter = ""; // FIXME PATCH FOR PARAM LIBRARIES WITHOUT LEGACY $where_configs = str_replace("AND .", "AND ", $where_configs); $where_configs = str_replace("`id_cluster`", "e.`id_cluster`", $where_configs); if ($minExecs > 0) { $minExecsFilter = "HAVING COUNT(*) > {$minExecs}"; } $filter_execs = DBUtils::getFilterExecs(); $options = Utils::getFilterOptions($db); $paramOptions = array(); foreach ($options[$paramEval] as $option) { if ($paramEval == 'id_cluster') { $paramOptions[] = $option['name']; } else { if ($paramEval == 'comp') { $paramOptions[] = Utils::getCompressionName($option[$paramEval]); } else { if ($paramEval == 'net') { $paramOptions[] = Utils::getNetworkName($option[$paramEval]); } else { if ($paramEval == 'disk') { $paramOptions[] = Utils::getDisksName($option[$paramEval]); } else { $paramOptions[] = $option[$paramEval]; } } } } } $benchOptions = $db->get_rows("SELECT DISTINCT bench FROM execs e LEFT JOIN clusters c ON e.id_cluster = c.id_cluster WHERE 1 {$filter_execs} {$where_configs} GROUP BY {$paramEval}, bench order by {$paramEval}"); // get the result rows $query = "SELECT count(*) as count, {$paramEval}, e.id_exec, exec as conf, bench, " . "exe_time, avg(exe_time) avg_exe_time, min(exe_time) min_exe_time " . "from execs e LEFT JOIN clusters c ON e.id_cluster = c.id_cluster WHERE 1 {$filter_execs} {$where_configs}" . "GROUP BY {$paramEval}, bench {$minExecsFilter} order by bench,{$paramEval}"; $rows = $db->get_rows($query); if (!$rows) { throw new \Exception("No results for query!"); } $arrayBenchs = array(); foreach ($paramOptions as $param) { foreach ($benchOptions as $bench) { $arrayBenchs[$bench['bench']][$param] = null; $arrayBenchs[$bench['bench']][$param]['y'] = 0; $arrayBenchs[$bench['bench']][$param]['count'] = 0; } } $series = array(); $bench = ''; foreach ($rows as $row) { if ($paramEval == 'comp') { $row[$paramEval] = Utils::getCompressionName($row['comp']); } else { if ($paramEval == 'id_cluster') { $row[$paramEval] = Utils::getClusterName($row[$paramEval], $db); } else { if ($paramEval == 'net') { $row[$paramEval] = Utils::getNetworkName($row['net']); } else { if ($paramEval == 'disk') { $row[$paramEval] = Utils::getDisksName($row['disk']); } else { if ($paramEval == 'iofilebuf') { $row[$paramEval] /= 1024; } } } } } $arrayBenchs[$row['bench']][$row[$paramEval]]['y'] = round((int) $row['avg_exe_time'], 2); $arrayBenchs[$row['bench']][$row[$paramEval]]['count'] = (int) $row['count']; } // ---------------------------------------------------- // Add predictions to the series // ---------------------------------------------------- $jsonData = $jsonHeader = "[]"; $instance = ""; $arrayBenchs_pred = array(); // FIXME PATCH FOR PARAM LIBRARIES WITHOUT LEGACY $where_configs = str_replace("AND .", "AND ", $where_configs); // compose instance $instance = MLUtils::generateSimpleInstance($param_names, $params, true, $db); $model_info = MLUtils::generateModelInfo($param_names, $params, true, $db); $instances = MLUtils::generateInstances($param_names, $params, true, $db); // model for filling $possible_models = $possible_models_id = array(); MLUtils::findMatchingModels($model_info, $possible_models, $possible_models_id, $dbml); $current_model = ""; if (array_key_exists('current_model', $_GET) && in_array($_GET['current_model'], $possible_models_id)) { $current_model = $_GET['current_model']; } if (!empty($possible_models_id)) { if ($current_model == "") { $query = "SELECT AVG(ABS(exe_time - pred_time)) AS MAE, AVG(ABS(exe_time - pred_time)/exe_time) AS RAE, p.id_learner FROM predictions p, learners l WHERE l.id_learner = p.id_learner AND p.id_learner IN ('" . implode("','", $possible_models_id) . "') AND predict_code > 0 ORDER BY MAE LIMIT 1"; $result = $dbml->query($query); $row = $result->fetch(); $current_model = $row['id_learner']; } $config = $instance . '-' . $current_model . "-parameval"; $query_cache = "SELECT count(*) as total FROM trees WHERE id_learner = '" . $current_model . "' AND model = '" . $model_info . "'"; $is_cached_mysql = $dbml->query($query_cache); $tmp_result = $is_cached_mysql->fetch(); $is_cached = $tmp_result['total'] > 0; $ret_data = null; if (!$is_cached) { // Call to MLFindAttributes, to fetch data $_GET['pass'] = 2; $_GET['unseen'] = 1; $_GET['current_model'] = $current_model; $mlfa1 = new MLFindAttributesController(); $mlfa1->container = $this->container; $ret_data = $mlfa1->mlfindattributesAction(); if ($ret_data == 1) { $must_wait = "YES"; $jsonData = $jsonHeader = '[]'; } else { $is_cached_mysql = $dbml->query($query_cache); $tmp_result = $is_cached_mysql->fetch(); $is_cached = $tmp_result['total'] > 0; } } if ($is_cached) { $must_wait = 'NO'; $query = "SELECT count(*) as count, {$paramEval}, bench, exe_time, avg(pred_time) avg_pred_time, min(pred_time) min_pred_time " . "FROM predictions e WHERE e.id_learner = '" . $current_model . "' {$filter_execs} {$where_configs}" . "GROUP BY {$paramEval}, bench {$minExecsFilter} order by bench, {$paramEval}"; $result = $dbml->query($query); // Initialize array foreach ($paramOptions as $param) { foreach ($benchOptions as $bench) { $arrayBenchs_pred[$bench['bench'] . '_pred'][$param] = null; $arrayBenchs_pred[$bench['bench'] . '_pred'][$param]['y'] = 0; $arrayBenchs_pred[$bench['bench'] . '_pred'][$param]['count'] = 0; } } foreach ($result as $row) { $bench_n = $row['bench'] . '_pred'; $class = $row[$paramEval]; if ($paramEval == 'comp') { $value = Utils::getCompressionName($class); } else { if ($paramEval == 'id_cluster') { $value = Utils::getClusterName($class, $db); } else { if ($paramEval == 'net') { $value = Utils::getNetworkName($class); } else { if ($paramEval == 'disk') { $value = Utils::getDisksName($class); } else { if ($paramEval == 'iofilebuf') { $value = $class / 1024; } else { $value = $class; } } } } } if (!in_array($value, $paramOptions)) { $paramOptions[] = $value; foreach ($benchOptions as $bench) { $arrayBenchs_pred[$bench['bench'] . '_pred'][$value] = null; $arrayBenchs_pred[$bench['bench'] . '_pred'][$value]['y'] = 0; $arrayBenchs_pred[$bench['bench'] . '_pred'][$value]['count'] = 0; $arrayBenchs[$bench['bench']][$value] = null; $arrayBenchs[$bench['bench']][$value]['y'] = 0; $arrayBenchs[$bench['bench']][$value]['count'] = 0; } } $arrayBenchs_pred[$bench_n][$value]['y'] = (int) $row['avg_pred_time']; $arrayBenchs_pred[$bench_n][$value]['count'] = (int) $row['count']; } } } // ---------------------------------------------------- // END - Add predictions to the series // ---------------------------------------------------- asort($paramOptions); foreach ($arrayBenchs as $key => $arrayBench) { $caregories = ''; $data_a = null; $data_p = null; foreach ($paramOptions as $param) { if ($arrayBenchs[$key][$param]['count'] > 0 && empty($arrayBenchs_pred) || !empty($arrayBenchs_pred) && ($arrayBenchs_pred[$key . '_pred'][$param]['count'] > 0 || $arrayBenchs[$key][$param]['count'] > 0)) { $data_a[] = $arrayBenchs[$key][$param]; if (!empty($arrayBenchs_pred)) { $data_p[] = $arrayBenchs_pred[$key . '_pred'][$param]; } $categories = $categories . "'{$param} " . Utils::getParamevalUnit($paramEval) . "',"; // FIXME - Redundant n times performed... don't care now } } $series[] = array('name' => $key, 'data' => $data_a); if (!empty($arrayBenchs_pred)) { $series[] = array('name' => $key . '_pred', 'data' => $data_p); } } $series = json_encode($series); if (!empty($arrayBenchs_pred)) { $colors = "['#7cb5ec','#9cd5fc','#434348','#636368','#90ed7d','#b0fd9d','#f7a35c','#f7c37c','#8085e9','#a0a5f9','#f15c80','#f17ca0','#e4d354','#f4f374','#8085e8','#a0a5f8','#8d4653','#ad6673','#91e8e1','#b1f8f1']"; } else { $colors = "['#7cb5ec','#434348','#90ed7d','#f7a35c','#8085e9','#f15c80','#e4d354','#8085e8','#8d4653','#91e8e1']"; } } catch (\Exception $e) { $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n"); $series = $jsonHeader = $colors = '[]'; $instance = $current_model = ''; $possible_models = $possible_models_id = array(); $must_wait = 'NO'; } echo $this->container->getTwig()->render('mltemplate/mlconfigperf.html.twig', array('selected' => 'mlparameval', 'title' => 'Improvement of Hadoop Execution by SW and HW Configurations', 'categories' => $categories, 'series' => $series, 'benchs' => $params['benchs'], 'nets' => $params['nets'], 'disks' => $params['disks'], 'blk_sizes' => $params['blk_sizes'], 'comps' => $params['comps'], 'id_clusters' => $params['id_clusters'], 'mapss' => $params['mapss'], 'replications' => $params['replications'], 'iosfs' => $params['iosfs'], 'iofilebufs' => $params['iofilebufs'], 'datanodess' => $params['datanodess'], 'bench_types' => $params['bench_types'], 'vm_sizes' => $params['vm_sizes'], 'vm_coress' => $params['vm_coress'], 'vm_RAMs' => $params['vm_RAMs'], 'types' => $params['types'], 'money' => $money, 'paramEval' => $paramEval, 'instance' => $instance, 'models' => '<li>' . implode('</li><li>', $possible_models) . '</li>', 'models_id' => $possible_models_id, 'current_model' => $current_model, 'gammacolors' => $colors, 'must_wait' => $must_wait, 'preset' => $preset, 'selPreset' => $selPreset, 'options' => Utils::getFilterOptions($db))); }
public function nodesEvaluationAction() { $dbUtils = $this->container->getDBUtils(); try { $filter_execs = DBUtils::getFilterExecs(); $innerQueryWhere = str_replace("e.", "e2.", $this->whereClause); $innerQueryWhere = str_replace("c.", "c2.", $innerQueryWhere); $innerQueryWhere = str_replace("p.", "p2.", $innerQueryWhere); $execs = $dbUtils->get_rows("SELECT c.datanodes,e.exec_type,c.vm_OS,c.vm_size,(e.exe_time * (c.cost_hour/3600)) as cost,e.*,c.*\n\t\t\t\t\tFROM aloja2.execs e JOIN aloja2.clusters c USING (id_cluster)\n\t\t\t\t\tLEFT JOIN aloja_ml.predictions p USING (id_exec)\n\t\t\t\t\tINNER JOIN ( SELECT c2.datanodes,e2.exec_type,c2.vm_OS,c2.vm_size as vmsize,MIN(e2.exe_time) as minexe\n\t\t\t\t\t\t\t\tfrom execs e2 JOIN aloja2.clusters c2 USING (id_cluster)\n\t\t\t\t\t\t\t\tLEFT JOIN aloja_ml.predictions p2 USING (id_exec)\n\t\t\t\t\t\t\t\tWHERE 1 {$innerQueryWhere} GROUP BY c2.datanodes,e2.exec_type,c2.vm_OS,c2.vm_size ) t ON t.minexe = e.exe_time\n\t\t\t\t\tAND t.datanodes = c.datanodes AND t.vmsize = c.vm_size\n\t\t\t\t\tWHERE 1 {$filter_execs} GROUP BY c.datanodes,e.exec_type,c.vm_OS,c.vm_size\n\t\t\t\t\tORDER BY c.datanodes ASC,c.vm_OS,c.vm_size DESC;"); $vmSizes = array(); $categories = array(); $dataNodes = array(); $vmOS = array(); $execTypes = array(); foreach ($execs as &$exec) { if (!isset($dataNodes[$exec['datanodes']])) { $dataNodes[$exec['datanodes']] = 1; $categories[] = $exec['datanodes']; } if (!isset($vmOS[$exec['vm_OS']])) { $vmOS[$exec['vm_OS']] = 1; } if (!isset($execTypes[$exec['exec_type']])) { $execTypes[$exec['exec_type']] = 1; } $vmSizes[$exec['vm_size']][$exec['exec_type']][$exec['vm_OS']][$exec['datanodes']] = array(round($exec['exe_time'], 2), round($exec['cost'], 2)); } $i = 0; $seriesColors = array('#7cb5ec', '#434348', '#90ed7d', '#f7a35c', '#8085e9', '#f15c80', '#e4d354', '#2b908f', '#f45b5b', '#91e8e1'); $series = array(); foreach ($vmSizes as $vmSize => $value) { foreach ($execTypes as $execType => $typevalue) { foreach ($vmOS as $OS => $osvalue) { if (isset($vmSizes[$vmSize][$execType][$OS])) { if ($i == sizeof($seriesColors)) { $i = 0; } $costSeries = array('name' => "{$vmSize} {$execType} {$OS} Run cost", 'type' => 'spline', 'dashStyle' => 'longdash', 'yAxis' => 0, 'data' => array(), 'tooltip' => array('valueSuffix' => ' US$'), 'color' => $seriesColors[$i]); $timeSeries = array('name' => "{$vmSize} {$execType} {$OS} Run execution time", 'type' => 'spline', 'yAxis' => 1, 'data' => array(), 'tooltip' => array('valueSuffix' => ' s'), 'color' => $seriesColors[$i++]); foreach ($dataNodes as $datanodes => $dvalue) { if (!isset($value[$execType][$OS][$datanodes])) { $costSeries['data'][] = "null"; $timeSeries['data'][] = "null"; } else { $costSeries['data'][] = $value[$execType][$OS][$datanodes][1]; $timeSeries['data'][] = $value[$execType][$OS][$datanodes][0]; } } $series[] = $timeSeries; $series[] = $costSeries; } } } } } catch (\Exception $e) { $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n"); } return $this->render('costPerfEvaluationViews/nodes_evaluation.html.twig', array('highcharts_js' => HighCharts::getHeader(), 'categories' => json_encode($categories), 'seriesData' => str_replace('"null"', 'null', json_encode($series)), 'datanodess' => $datanodes)); }
public function mlparamEvaluationAction() { $rows = $categories = $series = $instance = $model_info = $config = $current_model = $slice_info = ''; $arrayBenchs_pred = $possible_models = $possible_models_id = $other_models = array(); $jsonData = $jsonHeader = "[]"; $must_wait = 'NO'; try { $dbml = new \PDO($this->container->get('config')['db_conn_chain'], $this->container->get('config')['mysql_user'], $this->container->get('config')['mysql_pwd']); $dbml->setAttribute(\PDO::ATTR_ERRMODE, \PDO::ERRMODE_EXCEPTION); $dbml->setAttribute(\PDO::ATTR_EMULATE_PREPARES, false); $db = $this->container->getDBUtils(); if (array_key_exists('parameval', $_GET)) { $paramEval = isset($_GET['parameval']) && Utils::get_GET_string('parameval') != '' ? Utils::get_GET_string('parameval') : 'maps'; unset($_GET["parameval"]); } $this->buildFilters(array('current_model' => array('type' => 'selectOne', 'default' => null, 'label' => 'Model to use: ', 'generateChoices' => function () { return array(); }, 'parseFunction' => function () { $choice = isset($_GET['current_model']) ? $_GET['current_model'] : array(""); return array('whereClause' => '', 'currentChoice' => $choice); }, 'filterGroup' => 'MLearning'), 'minExecs' => array('default' => 0, 'type' => 'inputNumber', 'label' => 'Minimum executions:', 'parseFunction' => function () { return 0; }, 'filterGroup' => 'basic'), 'minexetime' => array('default' => 0), 'valid' => array('default' => 0), 'filter' => array('default' => 0), 'prepares' => array('default' => 0))); $this->buildFilterGroups(array('MLearning' => array('label' => 'Machine Learning', 'tabOpenDefault' => true, 'filters' => array('current_model')))); $where_configs = $this->filters->getWhereClause(); $params = array(); $param_names = array('bench', 'net', 'disk', 'maps', 'iosf', 'replication', 'iofilebuf', 'comp', 'blk_size', 'id_cluster', 'datanodes', 'vm_OS', 'vm_cores', 'vm_RAM', 'provider', 'vm_size', 'type', 'bench_type', 'hadoop_version'); // Order is important $params = $this->filters->getFiltersSelectedChoices($param_names); foreach ($param_names as $p) { if (!is_null($params[$p]) && is_array($params[$p])) { sort($params[$p]); } } $params_additional = array(); $param_names_additional = array('datefrom', 'dateto', 'minexetime', 'maxexetime', 'valid', 'filter'); // Order is important $params_additional = $this->filters->getFiltersSelectedChoices($param_names_additional); $param_variables = $this->filters->getFiltersSelectedChoices(array('current_model', 'minExecs')); $param_current_model = $param_variables['current_model']; $minExecs = $param_variables['minExecs']; $where_configs = str_replace("AND .", "AND ", $where_configs); $where_configs = str_replace("id_cluster", "e.id_cluster", $where_configs); $minExecsFilter = ""; if ($minExecs > 0) { $minExecsFilter = "HAVING COUNT(*) > {$minExecs}"; } $filter_execs = DBUtils::getFilterExecs(); $options = $this->filters->getFilterChoices(); $paramOptions = array(); foreach ($options[$paramEval] as $option) { if ($paramEval == 'comp') { $paramOptions[] = Utils::getCompressionName($option); } else { if ($paramEval == 'net') { $paramOptions[] = Utils::getNetworkName($option); } else { if ($paramEval == 'disk') { $paramOptions[] = Utils::getDisksName($option); } else { $paramOptions[] = $option; } } } } $param_eval_query = $paramEval == 'id_cluster' ? 'e.id_cluster' : $paramEval; $benchOptions = $db->get_rows("SELECT DISTINCT bench FROM aloja2.execs e LEFT JOIN aloja2.clusters c ON e.id_cluster = c.id_cluster WHERE 1 {$filter_execs} {$where_configs} GROUP BY {$param_eval_query}, bench order by {$param_eval_query}"); // get the result rows $query = "SELECT count(*) as count, {$param_eval_query}, e.id_exec, exec as conf, bench, " . "exe_time, avg(exe_time) avg_exe_time, min(exe_time) min_exe_time " . "from aloja2.execs e LEFT JOIN aloja2.clusters c ON e.id_cluster = c.id_cluster WHERE 1 {$filter_execs} {$where_configs}" . "GROUP BY {$param_eval_query},bench {$minExecsFilter} order by bench,{$param_eval_query}"; $rows = $db->get_rows($query); if (!$rows) { throw new \Exception("No results for query!"); } $arrayBenchs = array(); foreach ($paramOptions as $param) { foreach ($benchOptions as $bench) { $arrayBenchs[$bench['bench']][$param] = null; $arrayBenchs[$bench['bench']][$param]['y'] = 0; $arrayBenchs[$bench['bench']][$param]['count'] = 0; } } $series = array(); $bench = ''; foreach ($rows as $row) { if ($paramEval == 'comp') { $row[$paramEval] = Utils::getCompressionName($row['comp']); } else { if ($paramEval == 'net') { $row[$paramEval] = Utils::getNetworkName($row['net']); } else { if ($paramEval == 'disk') { $row[$paramEval] = Utils::getDisksName($row['disk']); } else { if ($paramEval == 'iofilebuf') { $row[$paramEval] /= 1024; } } } } $arrayBenchs[$row['bench']][$row[$paramEval]]['y'] = round((int) $row['avg_exe_time'], 2); $arrayBenchs[$row['bench']][$row[$paramEval]]['count'] = (int) $row['count']; } // ---------------------------------------------------- // Add predictions to the series // ---------------------------------------------------- $param_variables = $this->filters->getFiltersSelectedChoices(array('current_model')); $param_current_model = $param_variables['current_model']; $where_configs = str_replace("AND .", "AND ", $where_configs); // compose instance $instance = MLUtils::generateSimpleInstance($this->filters, $param_names, $params, true); $model_info = MLUtils::generateModelInfo($this->filters, $param_names, $params, true); $slice_info = MLUtils::generateDatasliceInfo($this->filters, $param_names_additional, $params_additional); // model for filling MLUtils::findMatchingModels($model_info, $possible_models, $possible_models_id, $dbml); $current_model = ''; if (!is_null($possible_models_id) && in_array($param_current_model, $possible_models_id)) { $current_model = $param_current_model; } // Other models for filling $where_models = ''; if (!empty($possible_models_id)) { $where_models = " WHERE id_learner NOT IN ('" . implode("','", $possible_models_id) . "')"; } $result = $dbml->query("SELECT id_learner FROM aloja_ml.learners" . $where_models); foreach ($result as $row) { $other_models[] = $row['id_learner']; } if (!empty($possible_models_id)) { if ($current_model == "") { $query = "SELECT AVG(ABS(exe_time - pred_time)) AS MAE, AVG(ABS(exe_time - pred_time)/exe_time) AS RAE, p.id_learner FROM aloja_ml.predictions p, aloja_ml.learners l WHERE l.id_learner = p.id_learner AND p.id_learner IN ('" . implode("','", $possible_models_id) . "') AND predict_code > 0 ORDER BY MAE LIMIT 1"; $result = $dbml->query($query); $row = $result->fetch(); $current_model = $row['id_learner']; } $config = $instance . '-' . $current_model . ' ' . $slice_info . "-parameval"; $query_cache = "SELECT count(*) as total FROM aloja_ml.trees WHERE id_learner = '" . $current_model . "' AND model = '" . $model_info . "'"; $is_cached_mysql = $dbml->query($query_cache); $tmp_result = $is_cached_mysql->fetch(); $is_cached = $tmp_result['total'] > 0; $ret_data = null; if (!$is_cached) { // Call to MLFindAttributes, to fetch data $_GET['pass'] = 2; $_GET['unseen'] = 1; $_GET['current_model'] = $current_model; $mlfa1 = new MLFindAttributesController(); $mlfa1->container = $this->container; $ret_data = $mlfa1->mlfindattributesAction(); if ($ret_data == 1) { $must_wait = "YES"; $jsonData = $jsonHeader = '[]'; } else { $is_cached_mysql = $dbml->query($query_cache); $tmp_result = $is_cached_mysql->fetch(); $is_cached = $tmp_result['total'] > 0; } } if ($is_cached) { $must_wait = 'NO'; $query = "SELECT count(*) as count, {$param_eval_query}, bench, exe_time, avg(pred_time) avg_pred_time, min(pred_time) min_pred_time " . "FROM aloja_ml.predictions e WHERE e.id_learner = '" . $current_model . "' {$filter_execs} {$where_configs}" . "GROUP BY {$param_eval_query}, bench {$minExecsFilter} order by bench,{$param_eval_query}"; $result = $dbml->query($query); // Initialize array foreach ($paramOptions as $param) { foreach ($benchOptions as $bench) { $arrayBenchs_pred[$bench['bench'] . '_pred'][$param] = null; $arrayBenchs_pred[$bench['bench'] . '_pred'][$param]['y'] = 0; $arrayBenchs_pred[$bench['bench'] . '_pred'][$param]['count'] = 0; } } foreach ($result as $row) { $bench_n = $row['bench'] . '_pred'; $class = $row[$paramEval]; if ($paramEval == 'comp') { $value = Utils::getCompressionName($class); } else { if ($paramEval == 'id_cluster') { $value = Utils::getClusterName($class, $db); } else { if ($paramEval == 'net') { $value = Utils::getNetworkName($class); } else { if ($paramEval == 'disk') { $value = Utils::getDisksName($class); } else { if ($paramEval == 'iofilebuf') { $value = $class / 1024; } else { $value = $class; } } } } } if (!in_array($value, $paramOptions)) { $paramOptions[] = $value; foreach ($benchOptions as $bench) { $arrayBenchs_pred[$bench['bench'] . '_pred'][$value] = null; $arrayBenchs_pred[$bench['bench'] . '_pred'][$value]['y'] = 0; $arrayBenchs_pred[$bench['bench'] . '_pred'][$value]['count'] = 0; $arrayBenchs[$bench['bench']][$value] = null; $arrayBenchs[$bench['bench']][$value]['y'] = 0; $arrayBenchs[$bench['bench']][$value]['count'] = 0; } } $arrayBenchs_pred[$bench_n][$value]['y'] = (int) $row['avg_pred_time']; $arrayBenchs_pred[$bench_n][$value]['count'] = (int) $row['count']; } } } // ---------------------------------------------------- // END - Add predictions to the series // ---------------------------------------------------- asort($paramOptions); foreach ($arrayBenchs as $key => $arrayBench) { $caregories = ''; $data_a = null; $data_p = null; foreach ($paramOptions as $param) { if ($arrayBenchs[$key][$param]['count'] > 0 && empty($arrayBenchs_pred) || !empty($arrayBenchs_pred) && ($arrayBenchs_pred[$key . '_pred'][$param]['count'] > 0 || $arrayBenchs[$key][$param]['count'] > 0)) { $data_a[] = $arrayBenchs[$key][$param]; if (!empty($arrayBenchs_pred)) { $data_p[] = $arrayBenchs_pred[$key . '_pred'][$param]; } $categories = $categories . "'{$param} " . Utils::getParamevalUnit($paramEval) . "',"; // FIXME - Redundant n times performed... don't care now } } $series[] = array('name' => $key, 'data' => $data_a); if (!empty($arrayBenchs_pred)) { $series[] = array('name' => $key . '_pred', 'data' => $data_p); } } $series = json_encode($series); if (!empty($arrayBenchs_pred)) { $colors = "['#7cb5ec','#9cd5fc','#434348','#636368','#90ed7d','#b0fd9d','#f7a35c','#f7c37c','#8085e9','#a0a5f9','#f15c80','#f17ca0','#e4d354','#f4f374','#8085e8','#a0a5f8','#8d4653','#ad6673','#91e8e1','#b1f8f1']"; } else { $colors = "['#7cb5ec','#434348','#90ed7d','#f7a35c','#8085e9','#f15c80','#e4d354','#8085e8','#8d4653','#91e8e1']"; } } catch (\Exception $e) { $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n"); $series = $jsonHeader = $colors = '[]'; $must_wait = 'NO'; } $return_params = array('title' => 'Improvement of Hadoop Execution by SW and HW Configurations', 'categories' => $categories, 'series' => $series, 'paramEval' => $paramEval, 'instance' => $instance, 'models' => '<li>' . implode('</li><li>', $possible_models) . '</li>', 'models_id' => $possible_models_id, 'current_model' => $current_model, 'gammacolors' => $colors, 'model_info' => $model_info, 'slice_info' => $slice_info, 'must_wait' => $must_wait); $this->filters->setCurrentChoices('current_model', array_merge($possible_models_id, array('---Other models---'), $other_models)); return $this->render('mltemplate/mlparameval.html.twig', $return_params); }
public function paramEvaluationAction() { $db = $this->container->getDBUtils(); $this->buildFilters(array('minexecs' => array('default' => null, 'type' => 'inputNumber', 'label' => 'Minimum executions:', 'parseFunction' => function () { return 0; }, 'filterGroup' => 'basic'))); $whereClause = $this->filters->getWhereClause(); $model_html = ''; $model_info = $db->get_rows("SELECT id_learner, model, algorithm, dataslice FROM aloja_ml.learners"); foreach ($model_info as $row) { $model_html = $model_html . "<li><b>" . $row['id_learner'] . "</b> => " . $row['algorithm'] . " : " . $row['model'] . " : " . $row['dataslice'] . "</li>"; } $categories = ''; $series = ''; try { $paramEval = isset($_GET['parameval']) && Utils::get_GET_string('parameval') != '' ? Utils::get_GET_string('parameval') : 'maps'; $minExecs = isset($_GET['minexecs']) ? Utils::get_GET_int('minexecs') : -1; $this->filters->changeCurrentChoice('minexecs', $minExecs == -1 ? null : $minExecs); $shortAliasParamEval = array('maps' => 'e', 'comp' => 'e', 'id_cluster' => 'c', 'net' => 'e', 'disk' => 'e', 'replication' => 'e', 'iofilebuf' => 'e', 'blk_size' => 'e', 'iosf' => 'e', 'vm_size' => 'c', 'vm_cores' => 'c', 'vm_ram' => 'c', 'datanodes' => 'c', 'hadoop_version' => 'e', 'type' => 'c'); $minExecsFilter = ""; if ($minExecs > 0) { $minExecsFilter = "HAVING COUNT(*) > {$minExecs}"; } $filter_execs = DBUtils::getFilterExecs(); $options = $this->filters->getFiltersArray()[$paramEval]['choices']; $benchOptions = "SELECT DISTINCT e.bench FROM aloja2.execs e JOIN aloja2.clusters c USING (id_cluster) LEFT JOIN aloja_ml.predictions p USING (id_exec) WHERE 1 {$filter_execs} {$whereClause} GROUP BY {$shortAliasParamEval[$paramEval]}.{$paramEval}, e.bench order by {$shortAliasParamEval[$paramEval]}.{$paramEval}"; $params = $this->filters->getFiltersSelectedChoices(array('prediction_model', 'upred', 'uobsr')); $whereClauseML = str_replace("exe_time", "pred_time", $whereClause); $whereClauseML = str_replace("start_time", "creation_time", $whereClauseML); $query = "SELECT COUNT(*) AS count, {$shortAliasParamEval[$paramEval]}.{$paramEval}, e.bench, avg(e.exe_time) avg_exe_time, min(e.exe_time) min_exe_time\n\t\t\t\t\t FROM aloja2.execs AS e JOIN aloja2.clusters AS c USING (id_cluster)\n\t\t\t\t\t LEFT JOIN aloja_ml.predictions AS p USING (id_exec)\n\t\t\t\t\t WHERE 1 {$filter_execs} {$whereClause}\n\t\t\t\t\t GROUP BY {$shortAliasParamEval[$paramEval]}.{$paramEval}, e.bench {$minExecsFilter} ORDER BY e.bench, {$shortAliasParamEval[$paramEval]}.{$paramEval}"; $queryPredictions = "\n\t\t\t\t\tSELECT COUNT(*) AS count, {$shortAliasParamEval[$paramEval]}.{$paramEval}, CONCAT('pred_',e.bench) as bench,\n\t\t\t\t\t\tavg(e.pred_time) as avg_exe_time, min(e.pred_time) as min_exe_time\n\t\t\t\t\t\tFROM aloja_ml.predictions AS e\n\t\t\t\t\t\tJOIN clusters c USING (id_cluster)\n\t\t\t\t\t\tWHERE 1 {$filter_execs} " . str_replace("p.", "e.", $whereClauseML) . " AND e.id_learner = '" . $params['prediction_model'] . "'\n\t\t\t\t\t\tGROUP BY {$shortAliasParamEval[$paramEval]}.{$paramEval}, e.bench {$minExecsFilter} ORDER BY e.bench, {$shortAliasParamEval[$paramEval]}.{$paramEval}"; // get the result rows if ($params['uobsr'] == 1 && $params['upred'] == 1) { $query = "({$query}) UNION ({$queryPredictions})"; $benchOptions = "SELECT DISTINCT e.bench FROM aloja2.execs e JOIN aloja2.clusters c USING (id_cluster) LEFT JOIN aloja_ml.predictions p USING (id_exec) WHERE 1 {$filter_execs} {$whereClause} GROUP BY {$shortAliasParamEval[$paramEval]}.{$paramEval}, e.bench\n\t\t\t\t\t\t\t\t UNION\n\t\t\t\t\t\t\t\t (SELECT DISTINCT CONCAT('pred_', e.bench) as bench FROM aloja_ml.predictions AS e\n\t\t\t\t\t\t\t\t JOIN clusters c USING (id_cluster)\n\t\t\t\t\t\t\t\t WHERE 1 {$filter_execs} " . str_replace("p.", "e.", $whereClauseML) . " AND e.id_learner = '" . $params['prediction_model'] . "'\n\t\t\t\t\t\t\t\t GROUP BY {$shortAliasParamEval[$paramEval]}.{$paramEval}, e.bench {$minExecsFilter})\n\t\t\t\t\t\t\t\t ORDER BY bench"; $optionsPredictions = "SELECT DISTINCT {$shortAliasParamEval[$paramEval]}.{$paramEval} FROM aloja_ml.predictions AS e JOIN clusters c USING (id_cluster) WHERE 1 {$filter_execs} " . str_replace("p.", "e.", $whereClauseML) . " AND e.id_learner = '" . $params['prediction_model'] . "' ORDER BY {$shortAliasParamEval[$paramEval]}.{$paramEval}"; $optionsPredictions = $db->get_rows($optionsPredictions); foreach ($optionsPredictions as $predOption) { $options[] = $predOption[$paramEval]; } } else { if ($params['uobsr'] == 0 && $params['upred'] == 1) { $query = $queryPredictions; $benchOptions = "SELECT DISTINCT CONCAT('pred_', e.bench) as bench FROM aloja_ml.predictions AS e\n \t\t\t\t\t\t\t\t JOIN clusters c USING (id_cluster)\n\t\t\t\t\t\t\t\t WHERE 1 {$filter_execs} " . str_replace("p.", "e.", $whereClauseML) . " AND e.id_learner = '" . $params['prediction_model'] . "'\n\t\t\t\t\t\t\t\t GROUP BY {$shortAliasParamEval[$paramEval]}.{$paramEval}, e.bench {$minExecsFilter} ORDER BY e.bench, {$shortAliasParamEval[$paramEval]}.{$paramEval}"; $options = array(); $optionsPredictions = "SELECT DISTINCT {$shortAliasParamEval[$paramEval]}.{$paramEval} FROM aloja_ml.predictions AS e JOIN clusters c USING (id_cluster) WHERE 1 {$filter_execs} " . str_replace("p.", "e.", $whereClauseML) . " AND e.id_learner = '" . $params['prediction_model'] . "' ORDER BY {$shortAliasParamEval[$paramEval]}.{$paramEval}"; $optionsPredictions = $db->get_rows($optionsPredictions); foreach ($optionsPredictions as $predOption) { $options[] = $predOption[$paramEval]; } } else { if ($params['uobsr'] == 0 && $params['upred'] == 0) { $this->container->getTwig()->addGlobal('message', "Warning: No data selected (Predictions|Observations) from the ML Filters. Adding the Observed executions to the figure by default.\n"); } } } $rows = $db->get_rows($query); $benchOptions = $db->get_rows($benchOptions); if (!$rows) { throw new \Exception("No results for query!"); } $paramOptions = array(); foreach ($options as $option) { if ($paramEval == 'id_cluster') { $paramOptions[] = Utils::getClusterName($option, $db); } else { if ($paramEval == 'comp') { $paramOptions[] = Utils::getCompressionName($option); } else { if ($paramEval == 'net') { $paramOptions[] = Utils::getNetworkName($option); } else { if ($paramEval == 'disk') { $paramOptions[] = Utils::getDisksName($option); } else { if ($paramEval == 'vm_ram') { $paramOptions[] = Utils::getBeautyRam($option); } else { $paramOptions[] = $option; } } } } } } $categories = ''; $arrayBenchs = array(); foreach ($paramOptions as $param) { $categories .= "'{$param}" . Utils::getParamevalUnit($paramEval) . "',"; foreach ($benchOptions as $bench) { $arrayBenchs[$bench['bench']][$param] = null; } } $series = array(); foreach ($rows as $row) { if ($paramEval == 'comp') { $row[$paramEval] = Utils::getCompressionName($row['comp']); } else { if ($paramEval == 'id_cluster') { $row[$paramEval] = Utils::getClusterName($row[$paramEval], $db); } else { if ($paramEval == 'net') { $row[$paramEval] = Utils::getNetworkName($row['net']); } else { if ($paramEval == 'disk') { $row[$paramEval] = Utils::getDisksName($row['disk']); } else { if ($paramEval == 'vm_ram') { $row[$paramEval] = Utils::getBeautyRam($row['vm_ram']); } } } } } $arrayBenchs[strtolower($row['bench'])][$row[$paramEval]]['y'] = round((int) $row['avg_exe_time'], 2); $arrayBenchs[strtolower($row['bench'])][$row[$paramEval]]['count'] = (int) $row['count']; } foreach ($arrayBenchs as $key => $arrayBench) { $series[] = array('name' => $key, 'data' => array_values($arrayBench)); } $series = json_encode($series); } catch (\Exception $e) { $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n"); } return $this->render('configEvaluationViews/parameval.html.twig', array('title' => 'Improvement of Hadoop Execution by SW and HW Configurations', 'minexecs' => $minExecs, 'categories' => $categories, 'series' => $series, 'paramEval' => $paramEval, 'models' => $model_html)); }
public function bestConfigDataAction() { $db = $this->container->getDBUtils(); $rows_config = ''; try { $configurations = array(); $where_configs = ''; $concat_config = ""; $benchs = Utils::read_params('benchs', $where_configs); $nets = Utils::read_params('nets', $where_configs); $disks = Utils::read_params('disks', $where_configs); $blk_sizes = Utils::read_params('blk_sizes', $where_configs); $comps = Utils::read_params('comps', $where_configs); $id_clusters = Utils::read_params('id_clusters', $where_configs); $mapss = Utils::read_params('mapss', $where_configs); $replications = Utils::read_params('replications', $where_configs); $iosfs = Utils::read_params('iosfs', $where_configs); $iofilebufs = Utils::read_params('iofilebufs', $where_configs); //$concat_config = join(',\'_\',', $configurations); //$concat_config = substr($concat_config, 1); //make sure there are some defaults if (!$concat_config) { $concat_config = 'disk'; $disks = array('HDD'); } $filter_execs = DBUtils::getFilterExecs(); $order_conf = 'LENGTH(conf), conf'; //get best config $query = "SELECT e2.* from aloja2.execs e2 WHERE e.id_exec IN " . "(SELECT MIN(e2.exe_time) FROM aloja2.execs e WHERE 1 {$filter_execs} {$where_configs} LIMIT 1);"; $rows = $db->get_rows($query); if (!$rows) { throw new \Exception("No results for query!"); } } catch (\Exception $e) { echo 'No data available'; /* $noData = array(); for($i = 0; $i<=sizeof($show_in_result); ++$i) $noData[] = 'error'; echo json_encode(array('aaData' => array($noData)));*/ } }
private function nodesEvalCore($scalabilityType, $dbUtils) { $categories = array(); $series = array(); $datanodes = array(); try { $filter_execs = DBUtils::getFilterExecs(); $innerQueryWhere = str_replace("e.", "e2.", $this->whereClause); $innerQueryWhere = str_replace("c.", "c2.", $innerQueryWhere); $innerQueryWhere = str_replace("p.", "p2.", $innerQueryWhere); $execs = "SELECT c.datanodes as 'category',e.exec_type,c.vm_OS,c.vm_size,(e.exe_time * (c.cost_hour/3600)) as cost,e.exe_time,c.*\n\t\t\t\t\tFROM aloja2.execs e JOIN aloja2.clusters c USING (id_cluster)\n\t\t\t\t\tLEFT JOIN aloja_ml.predictions p USING (id_exec)\n\t\t\t\t\tINNER JOIN ( SELECT c2.datanodes as 'category',e2.exec_type,c2.vm_OS,c2.vm_size as vmsize,MIN(e2.exe_time) as minexe\n\t\t\t\t\t\t\t\tfrom execs e2 JOIN aloja2.clusters c2 USING (id_cluster)\n\t\t\t\t\t\t\t\tLEFT JOIN aloja_ml.predictions p2 USING (id_exec)\n\t\t\t\t\t\t\t\tWHERE 1 {$innerQueryWhere} GROUP BY c2.datanodes,e2.exec_type,c2.vm_OS,c2.vm_size ) t ON t.minexe = e.exe_time\n\t\t\t\t\tAND t.category = category AND t.vmsize = c.vm_size\n\t\t\t\t\tWHERE 1 {$filter_execs} GROUP BY c.datanodes,e.exec_type,c.vm_OS,c.vm_size\n\t\t\t\t\tORDER BY c.datanodes ASC,c.vm_OS,c.vm_size DESC"; $predExecs = "SELECT c.datanodes as 'category','predicted' as 'exec_type',c.vm_OS,c.vm_size,(e.exe_time * (c.cost_hour/3600)) as cost,e.exe_time,c.*\n\t\t\t\t\tFROM aloja_ml.predictions e JOIN aloja2.clusters c USING (id_cluster)\n\t\t\t\t\tINNER JOIN ( SELECT c2.datanodes as 'category','default' as 'exec_type',c2.vm_OS,c2.vm_size as vmsize,MIN(p2.exe_time) as minexe\n\t\t\t\t\t\t\t\tfrom aloja_ml.predictions p2 JOIN aloja2.clusters c2 USING (id_cluster)\n\t\t\t\t\t\t\t\tWHERE 1 " . str_replace("e2.", "p2.", $innerQueryWhere) . " GROUP BY c2.datanodes,exec_type,c2.vm_OS,c2.vm_size ) t ON t.minexe = e.exe_time\n\t\t\t\t\tAND t.category = category AND t.vmsize = c.vm_size\n\t\t\t\t\tWHERE 1 {$filter_execs} GROUP BY c.datanodes,exec_type,c.vm_OS,c.vm_size\n\t\t\t\t\tORDER BY c.datanodes ASC,c.vm_OS,c.vm_size DESC"; if ($scalabilityType == 'Datasize') { $execs = str_replace("c2.datanodes", "e2.datasize", $execs); $execs = str_replace("c.datanodes", "e.datasize", $execs); $predExecs = str_replace("c2.datanodes", "p2.datasize", $predExecs); $predExecs = str_replace("c.datanodes", "e.datasize", $predExecs); } $params = $this->filters->getFiltersSelectedChoices(array('upred', 'uobsr')); if ($params['uobsr'] == 1 && $params['upred'] == 1) { $execs = "({$execs}) UNION ({$predExecs})"; } else { if ($params['uobsr'] == 0 && $params['upred'] == 1) { $execs = $predExecs; } } $execs = $dbUtils->get_rows($execs); $vmSizes = array(); $dataNodes = array(); $vmOS = array(); $execTypes = array(); foreach ($execs as &$exec) { if (!isset($dataNodes[$exec['category']])) { $dataNodes[$exec['category']] = 1; $categories[] = $exec['category']; } if (!isset($vmOS[$exec['vm_OS']])) { $vmOS[$exec['vm_OS']] = 1; } if (!isset($execTypes[$exec['exec_type']])) { $execTypes[$exec['exec_type']] = 1; } $vmSizes[$exec['vm_size']][$exec['exec_type']][$exec['vm_OS']][$exec['category']] = array(round($exec['exe_time'], 2), round($exec['cost'], 2)); } $i = 0; $seriesColors = array('#7cb5ec', '#434348', '#90ed7d', '#f7a35c', '#8085e9', '#f15c80', '#e4d354', '#2b908f', '#f45b5b', '#91e8e1'); foreach ($vmSizes as $vmSize => $value) { foreach ($execTypes as $execType => $typevalue) { foreach ($vmOS as $OS => $osvalue) { if (isset($vmSizes[$vmSize][$execType][$OS])) { if ($i == sizeof($seriesColors)) { $i = 0; } $costSeries = array('name' => "{$vmSize} {$execType} {$OS} Run cost", 'type' => 'spline', 'dashStyle' => 'longdash', 'yAxis' => 0, 'data' => array(), 'tooltip' => array('valueSuffix' => ' US$'), 'color' => $seriesColors[$i]); $timeSeries = array('name' => "{$vmSize} {$execType} {$OS} Run execution time", 'type' => 'spline', 'yAxis' => 1, 'data' => array(), 'tooltip' => array('valueSuffix' => ' s'), 'color' => $seriesColors[$i++]); foreach ($dataNodes as $datanode => $dvalue) { $datanodes[] = $datanode; if (!isset($value[$execType][$OS][$datanode])) { $costSeries['data'][] = "null"; $timeSeries['data'][] = "null"; } else { $costSeries['data'][] = $value[$execType][$OS][$datanode][1]; $timeSeries['data'][] = $value[$execType][$OS][$datanode][0]; } } $series[] = $timeSeries; $series[] = $costSeries; } } } } } catch (\Exception $e) { $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n"); } if ($scalabilityType == 'Datasize') { foreach ($categories as &$category) { $category = Utils::beautifyDatasize($category); } } return array('categories' => json_encode($categories), 'seriesData' => str_replace('"null"', 'null', json_encode($series)), 'datanodess' => $datanodes); }
public function nodesEvaluationAction() { $dbUtils = $this->container->getDBUtils(); $preset = null; if (sizeof($_GET) <= 1) { $preset = Utils::setDefaultPreset($dbUtils, 'Number of Nodes Evaluation'); } $selPreset = isset($_GET['presets']) ? $_GET['presets'] : "none"; try { $filter_execs = DBUtils::getFilterExecs(); $where_configs = ''; $datefrom = Utils::read_params('datefrom', $where_configs); $dateto = Utils::read_params('dateto', $where_configs); $benchs = Utils::read_params('benchs', $where_configs, true); $nets = Utils::read_params('nets', $where_configs, true); $disks = Utils::read_params('disks', $where_configs, true); $blk_sizes = Utils::read_params('blk_sizes', $where_configs, true); $comps = Utils::read_params('comps', $where_configs, true); $id_clusters = Utils::read_params('id_clusters', $where_configs, true); $mapss = Utils::read_params('mapss', $where_configs, true); $replications = Utils::read_params('replications', $where_configs, true); $iosfs = Utils::read_params('iosfs', $where_configs, true); $iofilebufs = Utils::read_params('iofilebufs', $where_configs, true); $money = Utils::read_params('money', $where_configs, true); $datanodes = Utils::read_params('datanodess', $where_configs, true); $benchtype = Utils::read_params('bench_types', $where_configs, true); $vm_sizes = Utils::read_params('vm_sizes', $where_configs, true); $vm_coress = Utils::read_params('vm_coress', $where_configs, true); $vm_RAMs = Utils::read_params('vm_RAMs', $where_configs, true); $hadoop_versions = Utils::read_params('hadoop_versions', $where_configs, true); $types = Utils::read_params('types', $where_configs, true); $filters = Utils::read_params('filters', $where_configs, true); $allunchecked = isset($_GET['allunchecked']) ? $_GET['allunchecked'] : ''; $minexetime = Utils::read_params('minexetime', $where_configs, true); $maxexetime = Utils::read_params('maxexetime', $where_configs, true); $provider = Utils::read_params('providers', $where_configs, false); $vm_OS = Utils::read_params('vm_OSs', $where_configs, false); if (!$benchs) { $where_configs .= 'AND bench IN (\'terasort\')'; } /* $selectedGroups = array(); if(isset($_GET['selected-groups']) && $_GET['selected-groups'] != "") { $selectedGroups = explode(",",$_GET['selected-groups']); } else { $selectedGroups[] = 'exec_type'; $selectedGroups[] = 'vm_OS'; } $selectedString = implode(',',Utils::getStandardGroupBy($selectedGroups)); $query = "SELECT ".str_replace("execTable","e",str_replace("clusterTable","c",$selectedString)).",c.vm_size,(e.exe_time * (c.cost_hour / 3600)) as cost, e.*, c.*". " FROM execs e JOIN clusters c USING (id_cluster) INNER JOIN ( SELECT ".str_replace("execTable","e2",str_replace("clusterTable","c2",$selectedString)).",c2.vm_size as vmsize,MIN(e2.exe_time) as minexe from execs e2 JOIN clusters c2 USING (id_cluster) WHERE 1 $where_configs GROUP BY ".str_replace("execTable","e2",str_replace("clusterTable","c2",$selectedString)).",c2.vm_size ) t ON"; $it = 0; foreach(explode(',',$selectedString) as $group) { if ($it != 0) $query .= " AND"; $tableName = str_replace("execTable", "e", $group); $tableName = str_replace("clusterTable", "c", $tableName); $withoutPrefixes = str_replace(array("execTable.", "clusterTable."), '', $group); $query .= " t.$withoutPrefixes = $tableName"; $it++; } $query .= " AND t.vmsize = c.vm_size WHERE 1 GROUP BY ".str_replace("execTable","e",str_replace("clusterTable","c",$selectedString)).",c.vm_size ORDER BY ".str_replace("execTable","e",str_replace("clusterTable","c",$selectedString)).",c.vm_size DESC;"; $execs = $dbUtils->get_rows($query); */ $execs = $dbUtils->get_rows("SELECT c.datanodes,e.exec_type,c.vm_OS,c.vm_size,(e.exe_time * (c.cost_hour/3600)) as cost,e.*,c.* FROM execs e JOIN clusters c USING (id_cluster) INNER JOIN ( SELECT c2.datanodes,e2.exec_type,c2.vm_OS,c2.vm_size as vmsize,MIN(e2.exe_time) as minexe from execs e2 JOIN clusters c2 USING (id_cluster) WHERE 1 {$where_configs} GROUP BY c2.datanodes,e2.exec_type,c2.vm_OS,c2.vm_size ) t ON t.minexe = e.exe_time AND t.datanodes = c.datanodes AND t.vmsize = c.vm_size WHERE 1 {$filter_execs} GROUP BY c.datanodes,e.exec_type,c.vm_OS,c.vm_size ORDER BY c.datanodes ASC,c.vm_OS,c.vm_size DESC;"); $vmSizes = array(); $categories = array(); $dataNodes = array(); $vmOS = array(); $execTypes = array(); foreach ($execs as &$exec) { if (!isset($dataNodes[$exec['datanodes']])) { $dataNodes[$exec['datanodes']] = 1; $categories[] = $exec['datanodes']; } if (!isset($vmOS[$exec['vm_OS']])) { $vmOS[$exec['vm_OS']] = 1; } if (!isset($execTypes[$exec['exec_type']])) { $execTypes[$exec['exec_type']] = 1; } $vmSizes[$exec['vm_size']][$exec['exec_type']][$exec['vm_OS']][$exec['datanodes']] = array(round($exec['exe_time'], 2), round($exec['cost'], 2)); } $i = 0; $seriesColors = array('#7cb5ec', '#434348', '#90ed7d', '#f7a35c', '#8085e9', '#f15c80', '#e4d354', '#2b908f', '#f45b5b', '#91e8e1'); $series = array(); foreach ($vmSizes as $vmSize => $value) { foreach ($execTypes as $execType => $typevalue) { foreach ($vmOS as $OS => $osvalue) { if (isset($vmSizes[$vmSize][$execType][$OS])) { if ($i == sizeof($seriesColors)) { $i = 0; } $costSeries = array('name' => "{$vmSize} {$execType} {$OS} Run cost", 'type' => 'spline', 'dashStyle' => 'longdash', 'yAxis' => 0, 'data' => array(), 'tooltip' => array('valueSuffix' => ' US$'), 'color' => $seriesColors[$i]); $timeSeries = array('name' => "{$vmSize} {$execType} {$OS} Run execution time", 'type' => 'spline', 'yAxis' => 1, 'data' => array(), 'tooltip' => array('valueSuffix' => ' s'), 'color' => $seriesColors[$i++]); foreach ($dataNodes as $datanodes => $dvalue) { if (!isset($value[$execType][$OS][$datanodes])) { $costSeries['data'][] = "null"; $timeSeries['data'][] = "null"; } else { $costSeries['data'][] = $value[$execType][$OS][$datanodes][1]; $timeSeries['data'][] = $value[$execType][$OS][$datanodes][0]; } } $series[] = $timeSeries; $series[] = $costSeries; } } } } } catch (\Exception $e) { $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n"); } echo $this->container->getTwig()->render('nodeseval/nodes_evaluation.html.twig', array('selected' => 'Number of Nodes Evaluation', 'highcharts_js' => HighCharts::getHeader(), 'categories' => json_encode($categories), 'seriesData' => str_replace('"null"', 'null', json_encode($series)), 'options' => Utils::getFilterOptions($dbUtils), 'datefrom' => $datefrom, 'dateto' => $dateto, 'benchs' => $benchs, 'nets' => $nets, 'disks' => $disks, 'blk_sizes' => $blk_sizes, 'comps' => $comps, 'id_clusters' => $id_clusters, 'mapss' => $mapss, 'replications' => $replications, 'iosfs' => $iosfs, 'iofilebufs' => $iofilebufs, 'money' => $money, 'datanodess' => $datanodes, 'bench_types' => $benchtype, 'vm_sizes' => $vm_sizes, 'vm_coress' => $vm_coress, 'vm_RAMs' => $vm_RAMs, 'vm_OS' => $vm_OS, 'hadoop_versions' => $hadoop_versions, 'types' => $types, 'providers' => $provider, 'filters' => $filters, 'allunchecked' => $allunchecked, 'select_multiple_benchs' => false, 'minexetime' => $minexetime, 'maxexetime' => $maxexetime, 'preset' => $preset, 'selPreset' => $selPreset, 'select_multiple_benchs' => false)); }