public function clustersAction() { $clusterNameSelected = null; if (isset($_GET['cluster_name'])) { $clusterNameSelected = Utils::get_GET_string('cluster_name'); } $filter_execs = DBUtils::getFilterExecs(); $db = $this->container->getDBUtils(); $clusters = $db->get_rows("SELECT * FROM aloja2.clusters c WHERE id_cluster IN (SELECT distinct(id_cluster) FROM aloja2.execs e WHERE 1 {$filter_execs});"); echo $this->container->getTwig()->render('defaultViews/clusters.html.twig', array('selected' => 'Clusters', 'clusters' => $clusters, 'clusterNameSelected' => $clusterNameSelected, 'title' => 'ALOJA Clusters')); }
public function histogramHDIAction() { $idExec = ''; $dbConn = $this->getContainer()->getDBUtils(); try { $idExec = Utils::get_GET_string('id_exec'); if (!$idExec) { $idExec = @$dbConn->get_rows("SELECT id_exec FROM aloja2.execs WHERE perf_details = 1 AND valid = 1 AND filter = 0 AND hadoop_version = 2 LIMIT 5")[rand(0, 5)]['id_exec']; } } catch (\Exception $e) { $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n"); } if (!$idExec) { $this->container->getTwig()->addGlobal('message', 'No executions of Hadoop 2 with performance details available'); $exec = null; } else { $exec = $dbConn->get_rows("SELECT * FROM aloja2.execs JOIN aloja2.clusters USING (id_cluster) WHERE id_exec = {$idExec}")[0]; } return $this->render('perfDetailsViews/histogramhdi.html.twig', array('idExec' => $idExec, 'exec' => $exec)); }
public function hdp2CountersAction() { try { $db = $this->container->getDBUtils(); $this->buildFilters(array('bench' => array('default' => null))); $whereClause = $this->filters->getWhereClause(); $benchOptions = $db->get_rows("SELECT DISTINCT bench FROM aloja2.execs e JOIN aloja2.HDI_JOB_details USING (id_exec) WHERE valid = 1"); $discreteOptions = array(); $discreteOptions['bench'][] = 'All'; foreach ($benchOptions as $option) { $discreteOptions['bench'][] = array_shift($option); } $dbUtil = $this->container->getDBUtils(); $message = null; //check the URL $execs = Utils::get_GET_intArray('execs'); if (Utils::get_GET_string('pageTab')) { $type = Utils::get_GET_string('pageTab'); } else { $type = 'SUMMARY'; } $join = "JOIN aloja2.execs e using (id_exec) JOIN aloja2.clusters USING (id_cluster) LEFT JOIN aloja_ml.predictions p USING (id_exec) WHERE job_name NOT IN\n ('TeraGen', 'random-text-writer', 'mahout-examples-0.7-job.jar', 'Create pagerank nodes', 'Create pagerank links') {$whereClause}" . ($execs ? ' AND id_exec IN (' . join(',', $execs) . ') ' : ''); if (isset($_GET['jobid'])) { $join .= " AND JOB_ID = '" . Utils::get_GET_string('jobid') . "' "; } $join .= " LIMIT 10000"; $query = ""; if ($type == 'SUMMARY') { $query = "SELECT e.bench, e.exe_time, c.id_exec, c.JOB_ID, c.job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME, c.TOTAL_MAPS, c.FAILED_MAPS, c.FINISHED_MAPS, c.TOTAL_REDUCES, c.FAILED_REDUCES, c.job_name as CHARTS\n \t\t\tFROM aloja2.HDI_JOB_details c {$join}"; } else { if ($type == "MAP") { $query = "SELECT e.bench, e.exe_time, c.id_exec, JOB_ID, job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME, c.TOTAL_MAPS, c.FAILED_MAPS, c.FINISHED_MAPS, `TOTAL_LAUNCHED_MAPS`,\n \t\t\t`RACK_LOCAL_MAPS`,\n \t\t\t`SPILLED_RECORDS`,\n \t\t\t`MAP_INPUT_RECORDS`,\n \t\t\t`MAP_OUTPUT_RECORDS`,\n \t\t\t`MAP_OUTPUT_BYTES`,\n \t\t\t`MAP_OUTPUT_MATERIALIZED_BYTES`\n \t\t\tFROM aloja2.HDI_JOB_details c {$join}"; } else { if ($type == 'REDUCE') { $query = "SELECT e.bench, e.exe_time, c.id_exec, c.JOB_ID, c.job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME, c.TOTAL_REDUCES, c.FAILED_REDUCES,\n \t\t\t`TOTAL_LAUNCHED_REDUCES`,\n \t\t\t`REDUCE_INPUT_GROUPS`,\n \t\t\t`REDUCE_INPUT_RECORDS`,\n \t\t\t`REDUCE_OUTPUT_RECORDS`,\n \t\t\t`REDUCE_SHUFFLE_BYTES`,\n \t\t\t`COMBINE_INPUT_RECORDS`,\n \t\t\t`COMBINE_OUTPUT_RECORDS`\n \t\t\tFROM aloja2.HDI_JOB_details c {$join}"; } else { if ($type == 'FILE-IO') { $query = "SELECT e.bench, e.exe_time, c.id_exec, c.JOB_ID, c.job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME,\n \t\t\t`SLOTS_MILLIS_MAPS`,\n \t\t\t`SLOTS_MILLIS_REDUCES`,\n \t\t\t`SPLIT_RAW_BYTES`,\n \t\t\t`FILE_BYTES_WRITTEN`,\n \t\t\t`FILE_BYTES_READ`,\n \t\t\t`WASB_BYTES_WRITTEN`,\n \t\t\t`WASB_BYTES_READ`,\n \t\t\t`BYTES_READ`,\n \t\t\t`BYTES_WRITTEN`\n \t\t\tFROM aloja2.HDI_JOB_details c {$join}"; } else { if ($type == 'DETAIL') { $query = "SELECT e.bench, e.exe_time, c.* FROM aloja_logs.aloja2 c {$join}"; } else { if ($type == "TASKS") { $query = "SELECT e.bench, e.exe_time, j.job_name, c.* FROM aloja_logs.HDI_JOB_tasks c\n \t\t\tJOIN aloja2.HDI_JOB_details j USING(id_exec,JOB_ID) {$join} "; // $taskStatusOptions = $db->get_rows("SELECT DISTINCT TASK_STATUS FROM HDI_JOB_tasks JOIN aloja2.execs USING (id_exec) WHERE valid = 1"); // $typeOptions = $db->get_rows("SELECT DISTINCT TASK_TYPE FROM HDI_JOB_tasks JOIN aloja2.execs USING (id_exec) WHERE valid = 1"); // // $discreteOptions['TASK_STATUS'][] = 'All'; // $discreteOptions['TASK_TYPE'][] = 'All'; // foreach($taskStatusOptions as $option) { // $discreteOptions['TASK_STATUS'][] = array_shift($option); // } // foreach($typeOptions as $option) { // $discreteOptions['TASK_TYPE'][] = array_shift($option); // } $discreteOptions['TASK_STATUS'] = array('All', 'SUCCEEDED'); $discreteOptions['TASK_TYPE'] = array('All', 'MAP', 'REDUCE'); } else { throw new \Exception('Unknown type!'); } } } } } } $exec_rows = $dbUtil->get_rows($query); if (count($exec_rows) > 0) { $show_in_result_counters = array('id_exec' => 'ID', 'JOB_ID' => 'JOBID', 'bench' => 'Bench', 'job_name' => 'JOBNAME'); $show_in_result_counters = Utils::generate_show($show_in_result_counters, $exec_rows, 4); } } catch (\Exception $e) { $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n"); } return $this->render('repositoryViews/hdp2counters.html.twig', array('theaders' => isset($show_in_result_counters) ? $show_in_result_counters : array(), 'message' => $message, 'title' => 'Hadoop Jobs and Tasks Execution Counters', 'type' => $type, 'execs' => $execs, 'execsParam' => isset($_GET['execs']) ? Utils::get_GET_intArray('execs') : '', 'discreteOptions' => $discreteOptions, 'hdp2' => true)); }
public function mlparamEvaluationAction() { $rows = $categories = $series = $instance = $model_info = $config = $current_model = $slice_info = ''; $arrayBenchs_pred = $possible_models = $possible_models_id = $other_models = array(); $jsonData = $jsonHeader = "[]"; $must_wait = 'NO'; try { $dbml = new \PDO($this->container->get('config')['db_conn_chain'], $this->container->get('config')['mysql_user'], $this->container->get('config')['mysql_pwd']); $dbml->setAttribute(\PDO::ATTR_ERRMODE, \PDO::ERRMODE_EXCEPTION); $dbml->setAttribute(\PDO::ATTR_EMULATE_PREPARES, false); $db = $this->container->getDBUtils(); if (array_key_exists('parameval', $_GET)) { $paramEval = isset($_GET['parameval']) && Utils::get_GET_string('parameval') != '' ? Utils::get_GET_string('parameval') : 'maps'; unset($_GET["parameval"]); } $this->buildFilters(array('current_model' => array('type' => 'selectOne', 'default' => null, 'label' => 'Model to use: ', 'generateChoices' => function () { return array(); }, 'parseFunction' => function () { $choice = isset($_GET['current_model']) ? $_GET['current_model'] : array(""); return array('whereClause' => '', 'currentChoice' => $choice); }, 'filterGroup' => 'MLearning'), 'minExecs' => array('default' => 0, 'type' => 'inputNumber', 'label' => 'Minimum executions:', 'parseFunction' => function () { return 0; }, 'filterGroup' => 'basic'), 'minexetime' => array('default' => 0), 'valid' => array('default' => 0), 'filter' => array('default' => 0), 'prepares' => array('default' => 0))); $this->buildFilterGroups(array('MLearning' => array('label' => 'Machine Learning', 'tabOpenDefault' => true, 'filters' => array('current_model')))); $where_configs = $this->filters->getWhereClause(); $params = array(); $param_names = array('bench', 'net', 'disk', 'maps', 'iosf', 'replication', 'iofilebuf', 'comp', 'blk_size', 'id_cluster', 'datanodes', 'vm_OS', 'vm_cores', 'vm_RAM', 'provider', 'vm_size', 'type', 'bench_type', 'hadoop_version'); // Order is important $params = $this->filters->getFiltersSelectedChoices($param_names); foreach ($param_names as $p) { if (!is_null($params[$p]) && is_array($params[$p])) { sort($params[$p]); } } $params_additional = array(); $param_names_additional = array('datefrom', 'dateto', 'minexetime', 'maxexetime', 'valid', 'filter'); // Order is important $params_additional = $this->filters->getFiltersSelectedChoices($param_names_additional); $param_variables = $this->filters->getFiltersSelectedChoices(array('current_model', 'minExecs')); $param_current_model = $param_variables['current_model']; $minExecs = $param_variables['minExecs']; $where_configs = str_replace("AND .", "AND ", $where_configs); $where_configs = str_replace("id_cluster", "e.id_cluster", $where_configs); $minExecsFilter = ""; if ($minExecs > 0) { $minExecsFilter = "HAVING COUNT(*) > {$minExecs}"; } $filter_execs = DBUtils::getFilterExecs(); $options = $this->filters->getFilterChoices(); $paramOptions = array(); foreach ($options[$paramEval] as $option) { if ($paramEval == 'comp') { $paramOptions[] = Utils::getCompressionName($option); } else { if ($paramEval == 'net') { $paramOptions[] = Utils::getNetworkName($option); } else { if ($paramEval == 'disk') { $paramOptions[] = Utils::getDisksName($option); } else { $paramOptions[] = $option; } } } } $param_eval_query = $paramEval == 'id_cluster' ? 'e.id_cluster' : $paramEval; $benchOptions = $db->get_rows("SELECT DISTINCT bench FROM aloja2.execs e LEFT JOIN aloja2.clusters c ON e.id_cluster = c.id_cluster WHERE 1 {$filter_execs} {$where_configs} GROUP BY {$param_eval_query}, bench order by {$param_eval_query}"); // get the result rows $query = "SELECT count(*) as count, {$param_eval_query}, e.id_exec, exec as conf, bench, " . "exe_time, avg(exe_time) avg_exe_time, min(exe_time) min_exe_time " . "from aloja2.execs e LEFT JOIN aloja2.clusters c ON e.id_cluster = c.id_cluster WHERE 1 {$filter_execs} {$where_configs}" . "GROUP BY {$param_eval_query},bench {$minExecsFilter} order by bench,{$param_eval_query}"; $rows = $db->get_rows($query); if (!$rows) { throw new \Exception("No results for query!"); } $arrayBenchs = array(); foreach ($paramOptions as $param) { foreach ($benchOptions as $bench) { $arrayBenchs[$bench['bench']][$param] = null; $arrayBenchs[$bench['bench']][$param]['y'] = 0; $arrayBenchs[$bench['bench']][$param]['count'] = 0; } } $series = array(); $bench = ''; foreach ($rows as $row) { if ($paramEval == 'comp') { $row[$paramEval] = Utils::getCompressionName($row['comp']); } else { if ($paramEval == 'net') { $row[$paramEval] = Utils::getNetworkName($row['net']); } else { if ($paramEval == 'disk') { $row[$paramEval] = Utils::getDisksName($row['disk']); } else { if ($paramEval == 'iofilebuf') { $row[$paramEval] /= 1024; } } } } $arrayBenchs[$row['bench']][$row[$paramEval]]['y'] = round((int) $row['avg_exe_time'], 2); $arrayBenchs[$row['bench']][$row[$paramEval]]['count'] = (int) $row['count']; } // ---------------------------------------------------- // Add predictions to the series // ---------------------------------------------------- $param_variables = $this->filters->getFiltersSelectedChoices(array('current_model')); $param_current_model = $param_variables['current_model']; $where_configs = str_replace("AND .", "AND ", $where_configs); // compose instance $instance = MLUtils::generateSimpleInstance($this->filters, $param_names, $params, true); $model_info = MLUtils::generateModelInfo($this->filters, $param_names, $params, true); $slice_info = MLUtils::generateDatasliceInfo($this->filters, $param_names_additional, $params_additional); // model for filling MLUtils::findMatchingModels($model_info, $possible_models, $possible_models_id, $dbml); $current_model = ''; if (!is_null($possible_models_id) && in_array($param_current_model, $possible_models_id)) { $current_model = $param_current_model; } // Other models for filling $where_models = ''; if (!empty($possible_models_id)) { $where_models = " WHERE id_learner NOT IN ('" . implode("','", $possible_models_id) . "')"; } $result = $dbml->query("SELECT id_learner FROM aloja_ml.learners" . $where_models); foreach ($result as $row) { $other_models[] = $row['id_learner']; } if (!empty($possible_models_id)) { if ($current_model == "") { $query = "SELECT AVG(ABS(exe_time - pred_time)) AS MAE, AVG(ABS(exe_time - pred_time)/exe_time) AS RAE, p.id_learner FROM aloja_ml.predictions p, aloja_ml.learners l WHERE l.id_learner = p.id_learner AND p.id_learner IN ('" . implode("','", $possible_models_id) . "') AND predict_code > 0 ORDER BY MAE LIMIT 1"; $result = $dbml->query($query); $row = $result->fetch(); $current_model = $row['id_learner']; } $config = $instance . '-' . $current_model . ' ' . $slice_info . "-parameval"; $query_cache = "SELECT count(*) as total FROM aloja_ml.trees WHERE id_learner = '" . $current_model . "' AND model = '" . $model_info . "'"; $is_cached_mysql = $dbml->query($query_cache); $tmp_result = $is_cached_mysql->fetch(); $is_cached = $tmp_result['total'] > 0; $ret_data = null; if (!$is_cached) { // Call to MLFindAttributes, to fetch data $_GET['pass'] = 2; $_GET['unseen'] = 1; $_GET['current_model'] = $current_model; $mlfa1 = new MLFindAttributesController(); $mlfa1->container = $this->container; $ret_data = $mlfa1->mlfindattributesAction(); if ($ret_data == 1) { $must_wait = "YES"; $jsonData = $jsonHeader = '[]'; } else { $is_cached_mysql = $dbml->query($query_cache); $tmp_result = $is_cached_mysql->fetch(); $is_cached = $tmp_result['total'] > 0; } } if ($is_cached) { $must_wait = 'NO'; $query = "SELECT count(*) as count, {$param_eval_query}, bench, exe_time, avg(pred_time) avg_pred_time, min(pred_time) min_pred_time " . "FROM aloja_ml.predictions e WHERE e.id_learner = '" . $current_model . "' {$filter_execs} {$where_configs}" . "GROUP BY {$param_eval_query}, bench {$minExecsFilter} order by bench,{$param_eval_query}"; $result = $dbml->query($query); // Initialize array foreach ($paramOptions as $param) { foreach ($benchOptions as $bench) { $arrayBenchs_pred[$bench['bench'] . '_pred'][$param] = null; $arrayBenchs_pred[$bench['bench'] . '_pred'][$param]['y'] = 0; $arrayBenchs_pred[$bench['bench'] . '_pred'][$param]['count'] = 0; } } foreach ($result as $row) { $bench_n = $row['bench'] . '_pred'; $class = $row[$paramEval]; if ($paramEval == 'comp') { $value = Utils::getCompressionName($class); } else { if ($paramEval == 'id_cluster') { $value = Utils::getClusterName($class, $db); } else { if ($paramEval == 'net') { $value = Utils::getNetworkName($class); } else { if ($paramEval == 'disk') { $value = Utils::getDisksName($class); } else { if ($paramEval == 'iofilebuf') { $value = $class / 1024; } else { $value = $class; } } } } } if (!in_array($value, $paramOptions)) { $paramOptions[] = $value; foreach ($benchOptions as $bench) { $arrayBenchs_pred[$bench['bench'] . '_pred'][$value] = null; $arrayBenchs_pred[$bench['bench'] . '_pred'][$value]['y'] = 0; $arrayBenchs_pred[$bench['bench'] . '_pred'][$value]['count'] = 0; $arrayBenchs[$bench['bench']][$value] = null; $arrayBenchs[$bench['bench']][$value]['y'] = 0; $arrayBenchs[$bench['bench']][$value]['count'] = 0; } } $arrayBenchs_pred[$bench_n][$value]['y'] = (int) $row['avg_pred_time']; $arrayBenchs_pred[$bench_n][$value]['count'] = (int) $row['count']; } } } // ---------------------------------------------------- // END - Add predictions to the series // ---------------------------------------------------- asort($paramOptions); foreach ($arrayBenchs as $key => $arrayBench) { $caregories = ''; $data_a = null; $data_p = null; foreach ($paramOptions as $param) { if ($arrayBenchs[$key][$param]['count'] > 0 && empty($arrayBenchs_pred) || !empty($arrayBenchs_pred) && ($arrayBenchs_pred[$key . '_pred'][$param]['count'] > 0 || $arrayBenchs[$key][$param]['count'] > 0)) { $data_a[] = $arrayBenchs[$key][$param]; if (!empty($arrayBenchs_pred)) { $data_p[] = $arrayBenchs_pred[$key . '_pred'][$param]; } $categories = $categories . "'{$param} " . Utils::getParamevalUnit($paramEval) . "',"; // FIXME - Redundant n times performed... don't care now } } $series[] = array('name' => $key, 'data' => $data_a); if (!empty($arrayBenchs_pred)) { $series[] = array('name' => $key . '_pred', 'data' => $data_p); } } $series = json_encode($series); if (!empty($arrayBenchs_pred)) { $colors = "['#7cb5ec','#9cd5fc','#434348','#636368','#90ed7d','#b0fd9d','#f7a35c','#f7c37c','#8085e9','#a0a5f9','#f15c80','#f17ca0','#e4d354','#f4f374','#8085e8','#a0a5f8','#8d4653','#ad6673','#91e8e1','#b1f8f1']"; } else { $colors = "['#7cb5ec','#434348','#90ed7d','#f7a35c','#8085e9','#f15c80','#e4d354','#8085e8','#8d4653','#91e8e1']"; } } catch (\Exception $e) { $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n"); $series = $jsonHeader = $colors = '[]'; $must_wait = 'NO'; } $return_params = array('title' => 'Improvement of Hadoop Execution by SW and HW Configurations', 'categories' => $categories, 'series' => $series, 'paramEval' => $paramEval, 'instance' => $instance, 'models' => '<li>' . implode('</li><li>', $possible_models) . '</li>', 'models_id' => $possible_models_id, 'current_model' => $current_model, 'gammacolors' => $colors, 'model_info' => $model_info, 'slice_info' => $slice_info, 'must_wait' => $must_wait); $this->filters->setCurrentChoices('current_model', array_merge($possible_models_id, array('---Other models---'), $other_models)); return $this->render('mltemplate/mlparameval.html.twig', $return_params); }
public function paramEvaluationAction() { $db = $this->container->getDBUtils(); $this->buildFilters(array('minexecs' => array('default' => null, 'type' => 'inputNumber', 'label' => 'Minimum executions:', 'parseFunction' => function () { return 0; }, 'filterGroup' => 'basic'))); $whereClause = $this->filters->getWhereClause(); $model_html = ''; $model_info = $db->get_rows("SELECT id_learner, model, algorithm, dataslice FROM aloja_ml.learners"); foreach ($model_info as $row) { $model_html = $model_html . "<li><b>" . $row['id_learner'] . "</b> => " . $row['algorithm'] . " : " . $row['model'] . " : " . $row['dataslice'] . "</li>"; } $categories = ''; $series = ''; try { $paramEval = isset($_GET['parameval']) && Utils::get_GET_string('parameval') != '' ? Utils::get_GET_string('parameval') : 'maps'; $minExecs = isset($_GET['minexecs']) ? Utils::get_GET_int('minexecs') : -1; $this->filters->changeCurrentChoice('minexecs', $minExecs == -1 ? null : $minExecs); $shortAliasParamEval = array('maps' => 'e', 'comp' => 'e', 'id_cluster' => 'c', 'net' => 'e', 'disk' => 'e', 'replication' => 'e', 'iofilebuf' => 'e', 'blk_size' => 'e', 'iosf' => 'e', 'vm_size' => 'c', 'vm_cores' => 'c', 'vm_ram' => 'c', 'datanodes' => 'c', 'hadoop_version' => 'e', 'type' => 'c'); $minExecsFilter = ""; if ($minExecs > 0) { $minExecsFilter = "HAVING COUNT(*) > {$minExecs}"; } $filter_execs = DBUtils::getFilterExecs(); $options = $this->filters->getFiltersArray()[$paramEval]['choices']; $benchOptions = "SELECT DISTINCT e.bench FROM aloja2.execs e JOIN aloja2.clusters c USING (id_cluster) LEFT JOIN aloja_ml.predictions p USING (id_exec) WHERE 1 {$filter_execs} {$whereClause} GROUP BY {$shortAliasParamEval[$paramEval]}.{$paramEval}, e.bench order by {$shortAliasParamEval[$paramEval]}.{$paramEval}"; $params = $this->filters->getFiltersSelectedChoices(array('prediction_model', 'upred', 'uobsr')); $whereClauseML = str_replace("exe_time", "pred_time", $whereClause); $whereClauseML = str_replace("start_time", "creation_time", $whereClauseML); $query = "SELECT COUNT(*) AS count, {$shortAliasParamEval[$paramEval]}.{$paramEval}, e.bench, avg(e.exe_time) avg_exe_time, min(e.exe_time) min_exe_time\n\t\t\t\t\t FROM aloja2.execs AS e JOIN aloja2.clusters AS c USING (id_cluster)\n\t\t\t\t\t LEFT JOIN aloja_ml.predictions AS p USING (id_exec)\n\t\t\t\t\t WHERE 1 {$filter_execs} {$whereClause}\n\t\t\t\t\t GROUP BY {$shortAliasParamEval[$paramEval]}.{$paramEval}, e.bench {$minExecsFilter} ORDER BY e.bench, {$shortAliasParamEval[$paramEval]}.{$paramEval}"; $queryPredictions = "\n\t\t\t\t\tSELECT COUNT(*) AS count, {$shortAliasParamEval[$paramEval]}.{$paramEval}, CONCAT('pred_',e.bench) as bench,\n\t\t\t\t\t\tavg(e.pred_time) as avg_exe_time, min(e.pred_time) as min_exe_time\n\t\t\t\t\t\tFROM aloja_ml.predictions AS e\n\t\t\t\t\t\tJOIN clusters c USING (id_cluster)\n\t\t\t\t\t\tWHERE 1 {$filter_execs} " . str_replace("p.", "e.", $whereClauseML) . " AND e.id_learner = '" . $params['prediction_model'] . "'\n\t\t\t\t\t\tGROUP BY {$shortAliasParamEval[$paramEval]}.{$paramEval}, e.bench {$minExecsFilter} ORDER BY e.bench, {$shortAliasParamEval[$paramEval]}.{$paramEval}"; // get the result rows if ($params['uobsr'] == 1 && $params['upred'] == 1) { $query = "({$query}) UNION ({$queryPredictions})"; $benchOptions = "SELECT DISTINCT e.bench FROM aloja2.execs e JOIN aloja2.clusters c USING (id_cluster) LEFT JOIN aloja_ml.predictions p USING (id_exec) WHERE 1 {$filter_execs} {$whereClause} GROUP BY {$shortAliasParamEval[$paramEval]}.{$paramEval}, e.bench\n\t\t\t\t\t\t\t\t UNION\n\t\t\t\t\t\t\t\t (SELECT DISTINCT CONCAT('pred_', e.bench) as bench FROM aloja_ml.predictions AS e\n\t\t\t\t\t\t\t\t JOIN clusters c USING (id_cluster)\n\t\t\t\t\t\t\t\t WHERE 1 {$filter_execs} " . str_replace("p.", "e.", $whereClauseML) . " AND e.id_learner = '" . $params['prediction_model'] . "'\n\t\t\t\t\t\t\t\t GROUP BY {$shortAliasParamEval[$paramEval]}.{$paramEval}, e.bench {$minExecsFilter})\n\t\t\t\t\t\t\t\t ORDER BY bench"; $optionsPredictions = "SELECT DISTINCT {$shortAliasParamEval[$paramEval]}.{$paramEval} FROM aloja_ml.predictions AS e JOIN clusters c USING (id_cluster) WHERE 1 {$filter_execs} " . str_replace("p.", "e.", $whereClauseML) . " AND e.id_learner = '" . $params['prediction_model'] . "' ORDER BY {$shortAliasParamEval[$paramEval]}.{$paramEval}"; $optionsPredictions = $db->get_rows($optionsPredictions); foreach ($optionsPredictions as $predOption) { $options[] = $predOption[$paramEval]; } } else { if ($params['uobsr'] == 0 && $params['upred'] == 1) { $query = $queryPredictions; $benchOptions = "SELECT DISTINCT CONCAT('pred_', e.bench) as bench FROM aloja_ml.predictions AS e\n \t\t\t\t\t\t\t\t JOIN clusters c USING (id_cluster)\n\t\t\t\t\t\t\t\t WHERE 1 {$filter_execs} " . str_replace("p.", "e.", $whereClauseML) . " AND e.id_learner = '" . $params['prediction_model'] . "'\n\t\t\t\t\t\t\t\t GROUP BY {$shortAliasParamEval[$paramEval]}.{$paramEval}, e.bench {$minExecsFilter} ORDER BY e.bench, {$shortAliasParamEval[$paramEval]}.{$paramEval}"; $options = array(); $optionsPredictions = "SELECT DISTINCT {$shortAliasParamEval[$paramEval]}.{$paramEval} FROM aloja_ml.predictions AS e JOIN clusters c USING (id_cluster) WHERE 1 {$filter_execs} " . str_replace("p.", "e.", $whereClauseML) . " AND e.id_learner = '" . $params['prediction_model'] . "' ORDER BY {$shortAliasParamEval[$paramEval]}.{$paramEval}"; $optionsPredictions = $db->get_rows($optionsPredictions); foreach ($optionsPredictions as $predOption) { $options[] = $predOption[$paramEval]; } } else { if ($params['uobsr'] == 0 && $params['upred'] == 0) { $this->container->getTwig()->addGlobal('message', "Warning: No data selected (Predictions|Observations) from the ML Filters. Adding the Observed executions to the figure by default.\n"); } } } $rows = $db->get_rows($query); $benchOptions = $db->get_rows($benchOptions); if (!$rows) { throw new \Exception("No results for query!"); } $paramOptions = array(); foreach ($options as $option) { if ($paramEval == 'id_cluster') { $paramOptions[] = Utils::getClusterName($option, $db); } else { if ($paramEval == 'comp') { $paramOptions[] = Utils::getCompressionName($option); } else { if ($paramEval == 'net') { $paramOptions[] = Utils::getNetworkName($option); } else { if ($paramEval == 'disk') { $paramOptions[] = Utils::getDisksName($option); } else { if ($paramEval == 'vm_ram') { $paramOptions[] = Utils::getBeautyRam($option); } else { $paramOptions[] = $option; } } } } } } $categories = ''; $arrayBenchs = array(); foreach ($paramOptions as $param) { $categories .= "'{$param}" . Utils::getParamevalUnit($paramEval) . "',"; foreach ($benchOptions as $bench) { $arrayBenchs[$bench['bench']][$param] = null; } } $series = array(); foreach ($rows as $row) { if ($paramEval == 'comp') { $row[$paramEval] = Utils::getCompressionName($row['comp']); } else { if ($paramEval == 'id_cluster') { $row[$paramEval] = Utils::getClusterName($row[$paramEval], $db); } else { if ($paramEval == 'net') { $row[$paramEval] = Utils::getNetworkName($row['net']); } else { if ($paramEval == 'disk') { $row[$paramEval] = Utils::getDisksName($row['disk']); } else { if ($paramEval == 'vm_ram') { $row[$paramEval] = Utils::getBeautyRam($row['vm_ram']); } } } } } $arrayBenchs[strtolower($row['bench'])][$row[$paramEval]]['y'] = round((int) $row['avg_exe_time'], 2); $arrayBenchs[strtolower($row['bench'])][$row[$paramEval]]['count'] = (int) $row['count']; } foreach ($arrayBenchs as $key => $arrayBench) { $series[] = array('name' => $key, 'data' => array_values($arrayBench)); } $series = json_encode($series); } catch (\Exception $e) { $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n"); } return $this->render('configEvaluationViews/parameval.html.twig', array('title' => 'Improvement of Hadoop Execution by SW and HW Configurations', 'minexecs' => $minExecs, 'categories' => $categories, 'series' => $series, 'paramEval' => $paramEval, 'models' => $model_html)); }
public function hdp2CountersDataAction() { $db = $this->container->getDBUtils(); $this->buildFilters(array('bench' => array('table' => 'execs', 'default' => null, 'type' => 'selectMultiple'))); $whereClause = $this->filters->getWhereClause(); try { //check the URL $execs = Utils::get_GET_intArray('execs'); if (!($type = Utils::get_GET_string('pageTab'))) { $type = 'SUMMARY'; } $join = "JOIN aloja2.execs e using (id_exec) LEFT JOIN aloja_ml.predictions p USING (id_exec) WHERE e.valid = 1 AND job_name NOT IN\n ('TeraGen', 'random-text-writer', 'mahout-examples-0.7-job.jar', 'Create pagerank nodes', 'Create pagerank links') {$whereClause}" . ($execs ? ' AND id_exec IN (' . join(',', $execs) . ') ' : '') . " LIMIT 10000"; if ($type == 'SUMMARY') { $query = "SELECT e.bench, e.exe_time, c.id_exec, c.JOB_ID, c.job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME, c.TOTAL_MAPS, c.FAILED_MAPS, c.FINISHED_MAPS, c.TOTAL_REDUCES, c.FAILED_REDUCES, c.job_name as CHARTS,\n \t\t\te.perf_details\n \t\t\tFROM aloja2.HDI_JOB_details c {$join}"; } elseif ($type == 'MAP') { $query = "SELECT e.bench, e.exe_time, c.id_exec, JOB_ID, job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME, c.TOTAL_MAPS, c.FAILED_MAPS, c.FINISHED_MAPS, `TOTAL_LAUNCHED_MAPS`,\n \t\t\t`RACK_LOCAL_MAPS`,\n \t\t\t`SPILLED_RECORDS`,\n \t\t\t`MAP_INPUT_RECORDS`,\n \t\t\t`MAP_OUTPUT_RECORDS`,\n \t\t\t`MAP_OUTPUT_BYTES`,\n \t\t\t`MAP_OUTPUT_MATERIALIZED_BYTES`,\n \t\t\te.perf_details\n \t\t\tFROM aloja2.HDI_JOB_details c {$join}"; } elseif ($type == 'REDUCE') { $query = "SELECT e.bench, e.exe_time, c.id_exec, c.JOB_ID, c.job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME, c.TOTAL_REDUCES, c.FAILED_REDUCES,\n \t\t\t`TOTAL_LAUNCHED_REDUCES`,\n \t\t\t`REDUCE_INPUT_GROUPS`,\n \t\t\t`REDUCE_INPUT_RECORDS`,\n \t\t\t`REDUCE_OUTPUT_RECORDS`,\n \t\t\t`REDUCE_SHUFFLE_BYTES`,\n \t\t\t`COMBINE_INPUT_RECORDS`,\n \t\t\t`COMBINE_OUTPUT_RECORDS`,\n \t\t\te.perf_details\n \t\t\tFROM aloja2.HDI_JOB_details c {$join}"; } elseif ($type == 'FILE-IO') { $query = "SELECT e.bench, e.exe_time, c.id_exec, c.JOB_ID, c.job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME,\n \t\t\t`SLOTS_MILLIS_MAPS`,\n \t\t\t`SLOTS_MILLIS_REDUCES`,\n \t\t\t`SPLIT_RAW_BYTES`,\n \t\t\t`FILE_BYTES_WRITTEN`,\n \t\t\t`FILE_BYTES_READ`,\n \t\t\t`WASB_BYTES_WRITTEN`,\n \t\t\t`WASB_BYTES_READ`,\n \t\t\t`BYTES_READ`,\n \t\t\t`BYTES_WRITTEN`,\n \t\t\te.perf_details\n \t\t\tFROM aloja2.HDI_JOB_details c {$join}"; } elseif ($type == 'DETAIL') { $query = "SELECT e.bench, e.exe_time, c.*,e.perf_details FROM aloja2.HDI_JOB_details c {$join}"; } elseif ($type == 'TASKS') { $query = "SELECT e.bench, e.exe_time, j.job_name, c.*,e.perf_details FROM aloja_logs.HDI_JOB_tasks c\n \t\t\tJOIN aloja2.HDI_JOB_details j USING(id_exec,JOB_ID) {$join} "; } else { throw new \Exception('Unknown type!'); } $exec_rows = $db->get_rows($query); if (count($exec_rows) > 0) { $show_in_result_counters = array('id_exec' => 'ID', 'JOB_ID' => 'JOBID', 'bench' => 'Bench', 'job_name' => 'JOBNAME'); $show_in_result_counters = Utils::generate_show($show_in_result_counters, $exec_rows, 4); $jsonData = Utils::generateJSONTable($exec_rows, $show_in_result_counters, 0, 'COUNTER'); header('Content-Type: application/json'); echo json_encode(array('aaData' => $jsonData)); // if (count($exec_rows) > 10000) { // $message .= 'WARNING, large resulset, please limit the query! Rows: '.count($exec_rows); // } } else { echo 'No data available'; } } catch (\Exception $e) { exit($e->getMessage()); echo 'No data available'; /*$noData = array(); for($i = 0; $i<=sizeof($show_in_result); ++$i) $noData[] = 'error'; echo json_encode(array('aaData' => array($noData)));*/ } }