Пример #1
0
 public function dbscanexecsAction()
 {
     $dbUtils = $this->container->getDBUtils();
     $this->buildFilters(array('perf_details' => array('default' => 1)));
     $whereClause = $this->filters->getWhereClause(array('execs' => 'e', 'clusters' => 'c'));
     $jobid = Utils::get_GET_string("jobid");
     // if no job requested, show a random one
     if (strlen($jobid) == 0 || $jobid === "random") {
         $_GET['NO_CACHE'] = 1;
         // Disable cache, otherwise random will not work
         $db = $this->container->getDBUtils();
         $query = "\n                SELECT DISTINCT(t.JOBID)\n                FROM aloja_logs.JOB_tasks t JOIN aloja2.execs e USING (id_exec)\n                JOIN aloja2.clusters c USING (id_cluster)\n                LEFT JOIN aloja_ml.predictions p USING (id_exec)\n                WHERE 1=1 {$whereClause}\n                LIMIT 100\n            ;";
         $jobid = $db->get_rows($query)[rand(0, 9)]['JOBID'];
     }
     list($bench, $job_offset, $id_exec) = $this->container->getDBUtils()->get_jobid_info($jobid);
     echo $this->render('perfDetailsViews/dbscanexecs.html.twig', array('highcharts_js' => HighCharts::getHeader(), 'jobid' => $jobid, 'bench' => $bench, 'job_offset' => $job_offset, 'METRICS' => DBUtils::$TASK_METRICS));
 }
Пример #2
0
 public function nodesEvaluationAction()
 {
     $dbUtils = $this->container->getDBUtils();
     try {
         $filter_execs = DBUtils::getFilterExecs();
         $innerQueryWhere = str_replace("e.", "e2.", $this->whereClause);
         $innerQueryWhere = str_replace("c.", "c2.", $innerQueryWhere);
         $innerQueryWhere = str_replace("p.", "p2.", $innerQueryWhere);
         $execs = $dbUtils->get_rows("SELECT c.datanodes,e.exec_type,c.vm_OS,c.vm_size,(e.exe_time * (c.cost_hour/3600)) as cost,e.*,c.*\n\t\t\t\t\tFROM aloja2.execs e JOIN aloja2.clusters c USING (id_cluster)\n\t\t\t\t\tLEFT JOIN aloja_ml.predictions p USING (id_exec)\n\t\t\t\t\tINNER JOIN ( SELECT c2.datanodes,e2.exec_type,c2.vm_OS,c2.vm_size as vmsize,MIN(e2.exe_time) as minexe\n\t\t\t\t\t\t\t\tfrom execs e2 JOIN aloja2.clusters c2 USING (id_cluster)\n\t\t\t\t\t\t\t\tLEFT JOIN aloja_ml.predictions p2 USING (id_exec)\n\t\t\t\t\t\t\t\tWHERE 1 {$innerQueryWhere} GROUP BY c2.datanodes,e2.exec_type,c2.vm_OS,c2.vm_size ) t ON t.minexe = e.exe_time\n\t\t\t\t\tAND t.datanodes = c.datanodes AND t.vmsize = c.vm_size\n\t\t\t\t\tWHERE 1 {$filter_execs}  GROUP BY c.datanodes,e.exec_type,c.vm_OS,c.vm_size\n\t\t\t\t\tORDER BY c.datanodes ASC,c.vm_OS,c.vm_size DESC;");
         $vmSizes = array();
         $categories = array();
         $dataNodes = array();
         $vmOS = array();
         $execTypes = array();
         foreach ($execs as &$exec) {
             if (!isset($dataNodes[$exec['datanodes']])) {
                 $dataNodes[$exec['datanodes']] = 1;
                 $categories[] = $exec['datanodes'];
             }
             if (!isset($vmOS[$exec['vm_OS']])) {
                 $vmOS[$exec['vm_OS']] = 1;
             }
             if (!isset($execTypes[$exec['exec_type']])) {
                 $execTypes[$exec['exec_type']] = 1;
             }
             $vmSizes[$exec['vm_size']][$exec['exec_type']][$exec['vm_OS']][$exec['datanodes']] = array(round($exec['exe_time'], 2), round($exec['cost'], 2));
         }
         $i = 0;
         $seriesColors = array('#7cb5ec', '#434348', '#90ed7d', '#f7a35c', '#8085e9', '#f15c80', '#e4d354', '#2b908f', '#f45b5b', '#91e8e1');
         $series = array();
         foreach ($vmSizes as $vmSize => $value) {
             foreach ($execTypes as $execType => $typevalue) {
                 foreach ($vmOS as $OS => $osvalue) {
                     if (isset($vmSizes[$vmSize][$execType][$OS])) {
                         if ($i == sizeof($seriesColors)) {
                             $i = 0;
                         }
                         $costSeries = array('name' => "{$vmSize} {$execType} {$OS} Run cost", 'type' => 'spline', 'dashStyle' => 'longdash', 'yAxis' => 0, 'data' => array(), 'tooltip' => array('valueSuffix' => ' US$'), 'color' => $seriesColors[$i]);
                         $timeSeries = array('name' => "{$vmSize} {$execType} {$OS} Run execution time", 'type' => 'spline', 'yAxis' => 1, 'data' => array(), 'tooltip' => array('valueSuffix' => ' s'), 'color' => $seriesColors[$i++]);
                         foreach ($dataNodes as $datanodes => $dvalue) {
                             if (!isset($value[$execType][$OS][$datanodes])) {
                                 $costSeries['data'][] = "null";
                                 $timeSeries['data'][] = "null";
                             } else {
                                 $costSeries['data'][] = $value[$execType][$OS][$datanodes][1];
                                 $timeSeries['data'][] = $value[$execType][$OS][$datanodes][0];
                             }
                         }
                         $series[] = $timeSeries;
                         $series[] = $costSeries;
                     }
                 }
             }
         }
     } catch (\Exception $e) {
         $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n");
     }
     return $this->render('costPerfEvaluationViews/nodes_evaluation.html.twig', array('highcharts_js' => HighCharts::getHeader(), 'categories' => json_encode($categories), 'seriesData' => str_replace('"null"', 'null', json_encode($series)), 'datanodess' => $datanodes));
 }
Пример #3
0
 public function datasizesEvaluationAction()
 {
     $dbUtils = $this->container->getDBUtils();
     return $this->render('costPerfEvaluationViews/nodes_evaluation.html.twig', array_merge(array('highcharts_js' => HighCharts::getHeader()), $this->nodesEvalCore("Datasize", $dbUtils)));
 }
Пример #4
0
 public function configImprovementAction()
 {
     $db = $this->container->getDBUtils();
     $this->buildFilters();
     $this->buildGroupFilters();
     $whereClause = $this->filters->getWhereClause();
     $model_html = '';
     $model_info = $db->get_rows("SELECT id_learner, model, algorithm, dataslice FROM aloja_ml.learners");
     foreach ($model_info as $row) {
         $model_html = $model_html . "<li><b>" . $row['id_learner'] . "</b> => " . $row['algorithm'] . " : " . $row['model'] . " : " . $row['dataslice'] . "</li>";
     }
     $rows_config = '';
     try {
         $concat_config = Utils::getConfig($this->filters->getGroupFilters());
         $filter_execs = DBUtils::getFilterExecs();
         $order_conf = 'LENGTH(conf), conf';
         $params = $this->filters->getFiltersSelectedChoices(array('prediction_model', 'upred', 'uobsr'));
         $whereClauseML = str_replace("exe_time", "pred_time", $whereClause);
         $whereClauseML = str_replace("start_time", "creation_time", $whereClauseML);
         $query = "SELECT COUNT(*) AS num, CONCAT({$concat_config}) conf\n\t\t\t\t\tFROM aloja2.execs AS e JOIN aloja2.clusters AS c USING (id_cluster)\n\t\t\t\t\tLEFT JOIN aloja_ml.predictions p USING (id_exec)\n\t\t\t\t\tWHERE 1 {$filter_execs} {$whereClause}\n\t\t\t\t\tGROUP BY conf ORDER BY {$order_conf}";
         $queryPredicted = "SELECT COUNT(*) AS num, CONCAT({$concat_config}) conf\n\t\t\t\t\tFROM aloja_ml.predictions AS e\n\t\t\t\t\tJOIN clusters c USING (id_cluster)\n\t\t\t\t\tWHERE 1 {$filter_execs} " . str_replace("p.", "e.", $whereClauseML) . " AND e.id_learner = '" . $params['prediction_model'] . "'\n\t\t\t\t\tGROUP BY conf ORDER BY {$order_conf}";
         //get configs first (categories)
         if ($params['uobsr'] == 1 && $params['upred'] == 1) {
             $query = "\n\t\t\t\tSELECT SUM(u1.num) AS num, u1.conf as conf\n\t\t\t\tFROM (\n\t\t\t\t\t({$query})\n\t\t\t\t\tUNION\n\t\t\t\t\t({$queryPredicted})\n\t\t\t\t) AS u1\n\t\t\t\tGROUP BY conf ORDER BY {$order_conf}\n\t\t\t";
         } else {
             if ($params['uobsr'] == 0 && $params['upred'] == 1) {
                 $query = $queryPredicted;
             }
         }
         $rows_config = $db->get_rows($query);
         $height = 600;
         if (count($rows_config) > 4) {
             $num_configs = count($rows_config);
             $height = round($height + 10 * ($num_configs - 4));
         }
         $query = "SELECT e.id_exec,\n\t\t\t concat({$concat_config}) conf, e.bench as bench,\n\t\t\t avg(e.exe_time) AVG_exe_time,\n\t\t\t min(e.exe_time) MIN_exe_time,\n\t\t\t (select AVG(exe_time) FROM aloja2.execs WHERE bench = e.bench {$whereClause}) AVG_ALL_exe_time,\n\t\t\t 'none'\n\t\t\t from aloja2.execs e JOIN aloja2.clusters c USING (id_cluster)\n\t\t\t LEFT JOIN aloja_ml.predictions AS p USING (id_exec)\n\t\t\t WHERE 1 {$filter_execs} {$whereClause}\n\t\t\t GROUP BY conf, e.bench order by e.bench, {$order_conf}";
         $queryPredicted = "\n\t\t\t\tSELECT e.id_exec, CONCAT({$concat_config}) conf, CONCAT('pred_',e.bench) as bench, AVG(e.pred_time) AVG_exe_time, min(e.pred_time) MIN_exe_time,\n\t\t\t\t(\n\t\t\t\t\tSELECT AVG(p.pred_time)\n\t\t\t\t\tFROM aloja_ml.predictions p\n\t\t\t\t\tWHERE p.bench = e.bench " . str_replace("e.", "p.", $whereClauseML) . " AND p.id_learner = '" . $params['prediction_model'] . "'\n\t\t\t\t) AVG_ALL_exe_time, 'none'\n\t\t\t\tFROM aloja_ml.predictions AS e\n\t\t\t\tJOIN clusters c USING (id_cluster)\n\t\t\t\tWHERE 1 {$filter_execs} " . str_replace("p.", "e.", $whereClauseML) . " AND e.id_learner = '" . $params['prediction_model'] . "'\n\t\t\t\tGROUP BY conf, e.bench ORDER BY e.bench, {$order_conf}\n\t\t\t";
         //get the result rows
         if ($params['uobsr'] == 1 && $params['upred'] == 1) {
             $query = "SELECT j.id_exec, j.conf, j.bench, j.AVG_exe_time, j.MIN_exe_time, j.AVG_ALL_exe_time, 'none' FROM (\n\t\t\t\t\t({$query}) UNION ({$queryPredicted})) AS j\n\t\t\t\tGROUP BY j.conf, j.bench ORDER BY j.bench, {$order_conf}\n\t\t\t";
         } else {
             if ($params['uobsr'] == 0 && $params['upred'] == 1) {
                 $query = $queryPredicted;
             } else {
                 if ($params['uobsr'] == 0 && $params['upred'] == 0) {
                     $this->container->getTwig()->addGlobal('message', "Warning: No data selected (Predictions|Observations) from the ML Filters. Adding the Observed executions to the figure by default.\n");
                 }
             }
         }
         $rows = $db->get_rows($query);
         if (!$rows) {
             throw new \Exception("No results for query!");
         }
     } catch (\Exception $e) {
         $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n");
     }
     $categories = '';
     $count = 0;
     $confOrders = array();
     foreach ($rows_config as $row_config) {
         $categories .= "'{$row_config['conf']} #{$row_config['num']}',";
         $count += $row_config['num'];
         $confOrders[] = $row_config['conf'];
     }
     $series = '';
     $bench = '';
     if ($rows) {
         $seriesIndex = 0;
         foreach ($rows as $row) {
             //close previous serie if not first one
             if ($bench && $bench != strtolower($row['bench'])) {
                 $series .= "]\n                        }, ";
             }
             //starts a new series
             if ($bench != strtolower($row['bench'])) {
                 $seriesIndex = 0;
                 $bench = strtolower($row['bench']);
                 $series .= "\n                        {\n                            name: '" . strtolower($row['bench']) . "',\n                                data: [";
             }
             while ($row['conf'] != $confOrders[$seriesIndex]) {
                 $series .= "[null],";
                 $seriesIndex++;
             }
             $series .= "['{$row['conf']}'," . round($row['AVG_ALL_exe_time'] / $row['AVG_exe_time'], 3) . "],";
             $seriesIndex++;
         }
         //close the last series
         $series .= "]\n                    }, ";
     }
     return $this->render('configEvaluationViews/config_improvement.html.twig', array('title' => 'Improvement of Hadoop Execution by SW and HW Configurations', 'highcharts_js' => HighCharts::getHeader(), 'categories' => $categories, 'series' => $series, 'models' => $model_html));
 }
Пример #5
0
 public function nodesEvaluationAction()
 {
     $dbUtils = $this->container->getDBUtils();
     $preset = null;
     if (sizeof($_GET) <= 1) {
         $preset = Utils::setDefaultPreset($dbUtils, 'Number of Nodes Evaluation');
     }
     $selPreset = isset($_GET['presets']) ? $_GET['presets'] : "none";
     try {
         $filter_execs = DBUtils::getFilterExecs();
         $where_configs = '';
         $datefrom = Utils::read_params('datefrom', $where_configs);
         $dateto = Utils::read_params('dateto', $where_configs);
         $benchs = Utils::read_params('benchs', $where_configs, true);
         $nets = Utils::read_params('nets', $where_configs, true);
         $disks = Utils::read_params('disks', $where_configs, true);
         $blk_sizes = Utils::read_params('blk_sizes', $where_configs, true);
         $comps = Utils::read_params('comps', $where_configs, true);
         $id_clusters = Utils::read_params('id_clusters', $where_configs, true);
         $mapss = Utils::read_params('mapss', $where_configs, true);
         $replications = Utils::read_params('replications', $where_configs, true);
         $iosfs = Utils::read_params('iosfs', $where_configs, true);
         $iofilebufs = Utils::read_params('iofilebufs', $where_configs, true);
         $money = Utils::read_params('money', $where_configs, true);
         $datanodes = Utils::read_params('datanodess', $where_configs, true);
         $benchtype = Utils::read_params('bench_types', $where_configs, true);
         $vm_sizes = Utils::read_params('vm_sizes', $where_configs, true);
         $vm_coress = Utils::read_params('vm_coress', $where_configs, true);
         $vm_RAMs = Utils::read_params('vm_RAMs', $where_configs, true);
         $hadoop_versions = Utils::read_params('hadoop_versions', $where_configs, true);
         $types = Utils::read_params('types', $where_configs, true);
         $filters = Utils::read_params('filters', $where_configs, true);
         $allunchecked = isset($_GET['allunchecked']) ? $_GET['allunchecked'] : '';
         $minexetime = Utils::read_params('minexetime', $where_configs, true);
         $maxexetime = Utils::read_params('maxexetime', $where_configs, true);
         $provider = Utils::read_params('providers', $where_configs, false);
         $vm_OS = Utils::read_params('vm_OSs', $where_configs, false);
         if (!$benchs) {
             $where_configs .= 'AND bench IN (\'terasort\')';
         }
         /*
                     $selectedGroups = array();
                     if(isset($_GET['selected-groups']) && $_GET['selected-groups'] != "") {
                         $selectedGroups = explode(",",$_GET['selected-groups']);
                     }
                     else {
                         $selectedGroups[] = 'exec_type';
                         $selectedGroups[] = 'vm_OS';
                     }
                     $selectedString = implode(',',Utils::getStandardGroupBy($selectedGroups));
         
                     $query = "SELECT ".str_replace("execTable","e",str_replace("clusterTable","c",$selectedString)).",c.vm_size,(e.exe_time * (c.cost_hour / 3600)) as cost, e.*, c.*".
                         " FROM execs e JOIN clusters c USING (id_cluster) INNER JOIN (
                             SELECT ".str_replace("execTable","e2",str_replace("clusterTable","c2",$selectedString)).",c2.vm_size as vmsize,MIN(e2.exe_time) as minexe from execs e2 JOIN clusters c2 USING (id_cluster) WHERE 1 $where_configs GROUP BY ".str_replace("execTable","e2",str_replace("clusterTable","c2",$selectedString)).",c2.vm_size
                         ) t ON";
         
                     $it = 0;
                     foreach(explode(',',$selectedString) as $group) {
                         if ($it != 0)
                             $query .= " AND";
         
                         $tableName = str_replace("execTable", "e", $group);
                         $tableName = str_replace("clusterTable", "c", $tableName);
                         $withoutPrefixes = str_replace(array("execTable.", "clusterTable."), '', $group);
                         $query .= " t.$withoutPrefixes = $tableName";
                         $it++;
                     }
                     $query .= " AND t.vmsize = c.vm_size WHERE 1 GROUP BY ".str_replace("execTable","e",str_replace("clusterTable","c",$selectedString)).",c.vm_size ORDER BY ".str_replace("execTable","e",str_replace("clusterTable","c",$selectedString)).",c.vm_size DESC;";
                     $execs = $dbUtils->get_rows($query);
         */
         $execs = $dbUtils->get_rows("SELECT c.datanodes,e.exec_type,c.vm_OS,c.vm_size,(e.exe_time * (c.cost_hour/3600)) as cost,e.*,c.* FROM execs e JOIN clusters c USING (id_cluster) INNER JOIN ( SELECT c2.datanodes,e2.exec_type,c2.vm_OS,c2.vm_size as vmsize,MIN(e2.exe_time) as minexe from execs e2 JOIN clusters c2 USING (id_cluster) WHERE 1 {$where_configs} GROUP BY c2.datanodes,e2.exec_type,c2.vm_OS,c2.vm_size ) t ON t.minexe = e.exe_time AND t.datanodes = c.datanodes AND t.vmsize = c.vm_size WHERE 1 {$filter_execs}  GROUP BY c.datanodes,e.exec_type,c.vm_OS,c.vm_size ORDER BY c.datanodes ASC,c.vm_OS,c.vm_size DESC;");
         $vmSizes = array();
         $categories = array();
         $dataNodes = array();
         $vmOS = array();
         $execTypes = array();
         foreach ($execs as &$exec) {
             if (!isset($dataNodes[$exec['datanodes']])) {
                 $dataNodes[$exec['datanodes']] = 1;
                 $categories[] = $exec['datanodes'];
             }
             if (!isset($vmOS[$exec['vm_OS']])) {
                 $vmOS[$exec['vm_OS']] = 1;
             }
             if (!isset($execTypes[$exec['exec_type']])) {
                 $execTypes[$exec['exec_type']] = 1;
             }
             $vmSizes[$exec['vm_size']][$exec['exec_type']][$exec['vm_OS']][$exec['datanodes']] = array(round($exec['exe_time'], 2), round($exec['cost'], 2));
         }
         $i = 0;
         $seriesColors = array('#7cb5ec', '#434348', '#90ed7d', '#f7a35c', '#8085e9', '#f15c80', '#e4d354', '#2b908f', '#f45b5b', '#91e8e1');
         $series = array();
         foreach ($vmSizes as $vmSize => $value) {
             foreach ($execTypes as $execType => $typevalue) {
                 foreach ($vmOS as $OS => $osvalue) {
                     if (isset($vmSizes[$vmSize][$execType][$OS])) {
                         if ($i == sizeof($seriesColors)) {
                             $i = 0;
                         }
                         $costSeries = array('name' => "{$vmSize} {$execType} {$OS} Run cost", 'type' => 'spline', 'dashStyle' => 'longdash', 'yAxis' => 0, 'data' => array(), 'tooltip' => array('valueSuffix' => ' US$'), 'color' => $seriesColors[$i]);
                         $timeSeries = array('name' => "{$vmSize} {$execType} {$OS} Run execution time", 'type' => 'spline', 'yAxis' => 1, 'data' => array(), 'tooltip' => array('valueSuffix' => ' s'), 'color' => $seriesColors[$i++]);
                         foreach ($dataNodes as $datanodes => $dvalue) {
                             if (!isset($value[$execType][$OS][$datanodes])) {
                                 $costSeries['data'][] = "null";
                                 $timeSeries['data'][] = "null";
                             } else {
                                 $costSeries['data'][] = $value[$execType][$OS][$datanodes][1];
                                 $timeSeries['data'][] = $value[$execType][$OS][$datanodes][0];
                             }
                         }
                         $series[] = $timeSeries;
                         $series[] = $costSeries;
                     }
                 }
             }
         }
     } catch (\Exception $e) {
         $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n");
     }
     echo $this->container->getTwig()->render('nodeseval/nodes_evaluation.html.twig', array('selected' => 'Number of Nodes Evaluation', 'highcharts_js' => HighCharts::getHeader(), 'categories' => json_encode($categories), 'seriesData' => str_replace('"null"', 'null', json_encode($series)), 'options' => Utils::getFilterOptions($dbUtils), 'datefrom' => $datefrom, 'dateto' => $dateto, 'benchs' => $benchs, 'nets' => $nets, 'disks' => $disks, 'blk_sizes' => $blk_sizes, 'comps' => $comps, 'id_clusters' => $id_clusters, 'mapss' => $mapss, 'replications' => $replications, 'iosfs' => $iosfs, 'iofilebufs' => $iofilebufs, 'money' => $money, 'datanodess' => $datanodes, 'bench_types' => $benchtype, 'vm_sizes' => $vm_sizes, 'vm_coress' => $vm_coress, 'vm_RAMs' => $vm_RAMs, 'vm_OS' => $vm_OS, 'hadoop_versions' => $hadoop_versions, 'types' => $types, 'providers' => $provider, 'filters' => $filters, 'allunchecked' => $allunchecked, 'select_multiple_benchs' => false, 'minexetime' => $minexetime, 'maxexetime' => $maxexetime, 'preset' => $preset, 'selPreset' => $selPreset, 'select_multiple_benchs' => false));
 }