public function dbscanexecsAction()
{
$dbUtils = $this->container->getDBUtils();
$this->buildFilters(array('perf_details' => array('default' => 1)));
$whereClause = $this->filters->getWhereClause(array('execs' => 'e', 'clusters' => 'c'));
$jobid = Utils::get_GET_string("jobid");
// if no job requested, show a random one
if (strlen($jobid) == 0 || $jobid === "random") {
$_GET['NO_CACHE'] = 1;
// Disable cache, otherwise random will not work
$db = $this->container->getDBUtils();
$query = "\n SELECT DISTINCT(t.JOBID)\n FROM aloja_logs.JOB_tasks t JOIN aloja2.execs e USING (id_exec)\n JOIN aloja2.clusters c USING (id_cluster)\n LEFT JOIN aloja_ml.predictions p USING (id_exec)\n WHERE 1=1 {$whereClause}\n LIMIT 100\n ;";
$jobid = $db->get_rows($query)[rand(0, 9)]['JOBID'];
}
list($bench, $job_offset, $id_exec) = $this->container->getDBUtils()->get_jobid_info($jobid);
echo $this->render('perfDetailsViews/dbscanexecs.html.twig', array('highcharts_js' => HighCharts::getHeader(), 'jobid' => $jobid, 'bench' => $bench, 'job_offset' => $job_offset, 'METRICS' => DBUtils::$TASK_METRICS));
}
public function nodesEvaluationAction()
{
$dbUtils = $this->container->getDBUtils();
try {
$filter_execs = DBUtils::getFilterExecs();
$innerQueryWhere = str_replace("e.", "e2.", $this->whereClause);
$innerQueryWhere = str_replace("c.", "c2.", $innerQueryWhere);
$innerQueryWhere = str_replace("p.", "p2.", $innerQueryWhere);
$execs = $dbUtils->get_rows("SELECT c.datanodes,e.exec_type,c.vm_OS,c.vm_size,(e.exe_time * (c.cost_hour/3600)) as cost,e.*,c.*\n\t\t\t\t\tFROM aloja2.execs e JOIN aloja2.clusters c USING (id_cluster)\n\t\t\t\t\tLEFT JOIN aloja_ml.predictions p USING (id_exec)\n\t\t\t\t\tINNER JOIN ( SELECT c2.datanodes,e2.exec_type,c2.vm_OS,c2.vm_size as vmsize,MIN(e2.exe_time) as minexe\n\t\t\t\t\t\t\t\tfrom execs e2 JOIN aloja2.clusters c2 USING (id_cluster)\n\t\t\t\t\t\t\t\tLEFT JOIN aloja_ml.predictions p2 USING (id_exec)\n\t\t\t\t\t\t\t\tWHERE 1 {$innerQueryWhere} GROUP BY c2.datanodes,e2.exec_type,c2.vm_OS,c2.vm_size ) t ON t.minexe = e.exe_time\n\t\t\t\t\tAND t.datanodes = c.datanodes AND t.vmsize = c.vm_size\n\t\t\t\t\tWHERE 1 {$filter_execs} GROUP BY c.datanodes,e.exec_type,c.vm_OS,c.vm_size\n\t\t\t\t\tORDER BY c.datanodes ASC,c.vm_OS,c.vm_size DESC;");
$vmSizes = array();
$categories = array();
$dataNodes = array();
$vmOS = array();
$execTypes = array();
foreach ($execs as &$exec) {
if (!isset($dataNodes[$exec['datanodes']])) {
$dataNodes[$exec['datanodes']] = 1;
$categories[] = $exec['datanodes'];
}
if (!isset($vmOS[$exec['vm_OS']])) {
$vmOS[$exec['vm_OS']] = 1;
}
if (!isset($execTypes[$exec['exec_type']])) {
$execTypes[$exec['exec_type']] = 1;
}
$vmSizes[$exec['vm_size']][$exec['exec_type']][$exec['vm_OS']][$exec['datanodes']] = array(round($exec['exe_time'], 2), round($exec['cost'], 2));
}
$i = 0;
$seriesColors = array('#7cb5ec', '#434348', '#90ed7d', '#f7a35c', '#8085e9', '#f15c80', '#e4d354', '#2b908f', '#f45b5b', '#91e8e1');
$series = array();
foreach ($vmSizes as $vmSize => $value) {
foreach ($execTypes as $execType => $typevalue) {
foreach ($vmOS as $OS => $osvalue) {
if (isset($vmSizes[$vmSize][$execType][$OS])) {
if ($i == sizeof($seriesColors)) {
$i = 0;
}
$costSeries = array('name' => "{$vmSize} {$execType} {$OS} Run cost", 'type' => 'spline', 'dashStyle' => 'longdash', 'yAxis' => 0, 'data' => array(), 'tooltip' => array('valueSuffix' => ' US$'), 'color' => $seriesColors[$i]);
$timeSeries = array('name' => "{$vmSize} {$execType} {$OS} Run execution time", 'type' => 'spline', 'yAxis' => 1, 'data' => array(), 'tooltip' => array('valueSuffix' => ' s'), 'color' => $seriesColors[$i++]);
foreach ($dataNodes as $datanodes => $dvalue) {
if (!isset($value[$execType][$OS][$datanodes])) {
$costSeries['data'][] = "null";
$timeSeries['data'][] = "null";
} else {
$costSeries['data'][] = $value[$execType][$OS][$datanodes][1];
$timeSeries['data'][] = $value[$execType][$OS][$datanodes][0];
}
}
$series[] = $timeSeries;
$series[] = $costSeries;
}
}
}
}
} catch (\Exception $e) {
$this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n");
}
return $this->render('costPerfEvaluationViews/nodes_evaluation.html.twig', array('highcharts_js' => HighCharts::getHeader(), 'categories' => json_encode($categories), 'seriesData' => str_replace('"null"', 'null', json_encode($series)), 'datanodess' => $datanodes));
}
public function datasizesEvaluationAction()
{
$dbUtils = $this->container->getDBUtils();
return $this->render('costPerfEvaluationViews/nodes_evaluation.html.twig', array_merge(array('highcharts_js' => HighCharts::getHeader()), $this->nodesEvalCore("Datasize", $dbUtils)));
}
public function configImprovementAction()
{
$db = $this->container->getDBUtils();
$this->buildFilters();
$this->buildGroupFilters();
$whereClause = $this->filters->getWhereClause();
$model_html = '';
$model_info = $db->get_rows("SELECT id_learner, model, algorithm, dataslice FROM aloja_ml.learners");
foreach ($model_info as $row) {
$model_html = $model_html . "<li><b>" . $row['id_learner'] . "</b> => " . $row['algorithm'] . " : " . $row['model'] . " : " . $row['dataslice'] . "</li>";
}
$rows_config = '';
try {
$concat_config = Utils::getConfig($this->filters->getGroupFilters());
$filter_execs = DBUtils::getFilterExecs();
$order_conf = 'LENGTH(conf), conf';
$params = $this->filters->getFiltersSelectedChoices(array('prediction_model', 'upred', 'uobsr'));
$whereClauseML = str_replace("exe_time", "pred_time", $whereClause);
$whereClauseML = str_replace("start_time", "creation_time", $whereClauseML);
$query = "SELECT COUNT(*) AS num, CONCAT({$concat_config}) conf\n\t\t\t\t\tFROM aloja2.execs AS e JOIN aloja2.clusters AS c USING (id_cluster)\n\t\t\t\t\tLEFT JOIN aloja_ml.predictions p USING (id_exec)\n\t\t\t\t\tWHERE 1 {$filter_execs} {$whereClause}\n\t\t\t\t\tGROUP BY conf ORDER BY {$order_conf}";
$queryPredicted = "SELECT COUNT(*) AS num, CONCAT({$concat_config}) conf\n\t\t\t\t\tFROM aloja_ml.predictions AS e\n\t\t\t\t\tJOIN clusters c USING (id_cluster)\n\t\t\t\t\tWHERE 1 {$filter_execs} " . str_replace("p.", "e.", $whereClauseML) . " AND e.id_learner = '" . $params['prediction_model'] . "'\n\t\t\t\t\tGROUP BY conf ORDER BY {$order_conf}";
//get configs first (categories)
if ($params['uobsr'] == 1 && $params['upred'] == 1) {
$query = "\n\t\t\t\tSELECT SUM(u1.num) AS num, u1.conf as conf\n\t\t\t\tFROM (\n\t\t\t\t\t({$query})\n\t\t\t\t\tUNION\n\t\t\t\t\t({$queryPredicted})\n\t\t\t\t) AS u1\n\t\t\t\tGROUP BY conf ORDER BY {$order_conf}\n\t\t\t";
} else {
if ($params['uobsr'] == 0 && $params['upred'] == 1) {
$query = $queryPredicted;
}
}
$rows_config = $db->get_rows($query);
$height = 600;
if (count($rows_config) > 4) {
$num_configs = count($rows_config);
$height = round($height + 10 * ($num_configs - 4));
}
$query = "SELECT e.id_exec,\n\t\t\t concat({$concat_config}) conf, e.bench as bench,\n\t\t\t avg(e.exe_time) AVG_exe_time,\n\t\t\t min(e.exe_time) MIN_exe_time,\n\t\t\t (select AVG(exe_time) FROM aloja2.execs WHERE bench = e.bench {$whereClause}) AVG_ALL_exe_time,\n\t\t\t 'none'\n\t\t\t from aloja2.execs e JOIN aloja2.clusters c USING (id_cluster)\n\t\t\t LEFT JOIN aloja_ml.predictions AS p USING (id_exec)\n\t\t\t WHERE 1 {$filter_execs} {$whereClause}\n\t\t\t GROUP BY conf, e.bench order by e.bench, {$order_conf}";
$queryPredicted = "\n\t\t\t\tSELECT e.id_exec, CONCAT({$concat_config}) conf, CONCAT('pred_',e.bench) as bench, AVG(e.pred_time) AVG_exe_time, min(e.pred_time) MIN_exe_time,\n\t\t\t\t(\n\t\t\t\t\tSELECT AVG(p.pred_time)\n\t\t\t\t\tFROM aloja_ml.predictions p\n\t\t\t\t\tWHERE p.bench = e.bench " . str_replace("e.", "p.", $whereClauseML) . " AND p.id_learner = '" . $params['prediction_model'] . "'\n\t\t\t\t) AVG_ALL_exe_time, 'none'\n\t\t\t\tFROM aloja_ml.predictions AS e\n\t\t\t\tJOIN clusters c USING (id_cluster)\n\t\t\t\tWHERE 1 {$filter_execs} " . str_replace("p.", "e.", $whereClauseML) . " AND e.id_learner = '" . $params['prediction_model'] . "'\n\t\t\t\tGROUP BY conf, e.bench ORDER BY e.bench, {$order_conf}\n\t\t\t";
//get the result rows
if ($params['uobsr'] == 1 && $params['upred'] == 1) {
$query = "SELECT j.id_exec, j.conf, j.bench, j.AVG_exe_time, j.MIN_exe_time, j.AVG_ALL_exe_time, 'none' FROM (\n\t\t\t\t\t({$query}) UNION ({$queryPredicted})) AS j\n\t\t\t\tGROUP BY j.conf, j.bench ORDER BY j.bench, {$order_conf}\n\t\t\t";
} else {
if ($params['uobsr'] == 0 && $params['upred'] == 1) {
$query = $queryPredicted;
} else {
if ($params['uobsr'] == 0 && $params['upred'] == 0) {
$this->container->getTwig()->addGlobal('message', "Warning: No data selected (Predictions|Observations) from the ML Filters. Adding the Observed executions to the figure by default.\n");
}
}
}
$rows = $db->get_rows($query);
if (!$rows) {
throw new \Exception("No results for query!");
}
} catch (\Exception $e) {
$this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n");
}
$categories = '';
$count = 0;
$confOrders = array();
foreach ($rows_config as $row_config) {
$categories .= "'{$row_config['conf']} #{$row_config['num']}',";
$count += $row_config['num'];
$confOrders[] = $row_config['conf'];
}
$series = '';
$bench = '';
if ($rows) {
$seriesIndex = 0;
foreach ($rows as $row) {
//close previous serie if not first one
if ($bench && $bench != strtolower($row['bench'])) {
$series .= "]\n }, ";
}
//starts a new series
if ($bench != strtolower($row['bench'])) {
$seriesIndex = 0;
$bench = strtolower($row['bench']);
$series .= "\n {\n name: '" . strtolower($row['bench']) . "',\n data: [";
}
while ($row['conf'] != $confOrders[$seriesIndex]) {
$series .= "[null],";
$seriesIndex++;
}
$series .= "['{$row['conf']}'," . round($row['AVG_ALL_exe_time'] / $row['AVG_exe_time'], 3) . "],";
$seriesIndex++;
}
//close the last series
$series .= "]\n }, ";
}
return $this->render('configEvaluationViews/config_improvement.html.twig', array('title' => 'Improvement of Hadoop Execution by SW and HW Configurations', 'highcharts_js' => HighCharts::getHeader(), 'categories' => $categories, 'series' => $series, 'models' => $model_html));
}
public function nodesEvaluationAction()
{
$dbUtils = $this->container->getDBUtils();
$preset = null;
if (sizeof($_GET) <= 1) {
$preset = Utils::setDefaultPreset($dbUtils, 'Number of Nodes Evaluation');
}
$selPreset = isset($_GET['presets']) ? $_GET['presets'] : "none";
try {
$filter_execs = DBUtils::getFilterExecs();
$where_configs = '';
$datefrom = Utils::read_params('datefrom', $where_configs);
$dateto = Utils::read_params('dateto', $where_configs);
$benchs = Utils::read_params('benchs', $where_configs, true);
$nets = Utils::read_params('nets', $where_configs, true);
$disks = Utils::read_params('disks', $where_configs, true);
$blk_sizes = Utils::read_params('blk_sizes', $where_configs, true);
$comps = Utils::read_params('comps', $where_configs, true);
$id_clusters = Utils::read_params('id_clusters', $where_configs, true);
$mapss = Utils::read_params('mapss', $where_configs, true);
$replications = Utils::read_params('replications', $where_configs, true);
$iosfs = Utils::read_params('iosfs', $where_configs, true);
$iofilebufs = Utils::read_params('iofilebufs', $where_configs, true);
$money = Utils::read_params('money', $where_configs, true);
$datanodes = Utils::read_params('datanodess', $where_configs, true);
$benchtype = Utils::read_params('bench_types', $where_configs, true);
$vm_sizes = Utils::read_params('vm_sizes', $where_configs, true);
$vm_coress = Utils::read_params('vm_coress', $where_configs, true);
$vm_RAMs = Utils::read_params('vm_RAMs', $where_configs, true);
$hadoop_versions = Utils::read_params('hadoop_versions', $where_configs, true);
$types = Utils::read_params('types', $where_configs, true);
$filters = Utils::read_params('filters', $where_configs, true);
$allunchecked = isset($_GET['allunchecked']) ? $_GET['allunchecked'] : '';
$minexetime = Utils::read_params('minexetime', $where_configs, true);
$maxexetime = Utils::read_params('maxexetime', $where_configs, true);
$provider = Utils::read_params('providers', $where_configs, false);
$vm_OS = Utils::read_params('vm_OSs', $where_configs, false);
if (!$benchs) {
$where_configs .= 'AND bench IN (\'terasort\')';
}
/*
$selectedGroups = array();
if(isset($_GET['selected-groups']) && $_GET['selected-groups'] != "") {
$selectedGroups = explode(",",$_GET['selected-groups']);
}
else {
$selectedGroups[] = 'exec_type';
$selectedGroups[] = 'vm_OS';
}
$selectedString = implode(',',Utils::getStandardGroupBy($selectedGroups));
$query = "SELECT ".str_replace("execTable","e",str_replace("clusterTable","c",$selectedString)).",c.vm_size,(e.exe_time * (c.cost_hour / 3600)) as cost, e.*, c.*".
" FROM execs e JOIN clusters c USING (id_cluster) INNER JOIN (
SELECT ".str_replace("execTable","e2",str_replace("clusterTable","c2",$selectedString)).",c2.vm_size as vmsize,MIN(e2.exe_time) as minexe from execs e2 JOIN clusters c2 USING (id_cluster) WHERE 1 $where_configs GROUP BY ".str_replace("execTable","e2",str_replace("clusterTable","c2",$selectedString)).",c2.vm_size
) t ON";
$it = 0;
foreach(explode(',',$selectedString) as $group) {
if ($it != 0)
$query .= " AND";
$tableName = str_replace("execTable", "e", $group);
$tableName = str_replace("clusterTable", "c", $tableName);
$withoutPrefixes = str_replace(array("execTable.", "clusterTable."), '', $group);
$query .= " t.$withoutPrefixes = $tableName";
$it++;
}
$query .= " AND t.vmsize = c.vm_size WHERE 1 GROUP BY ".str_replace("execTable","e",str_replace("clusterTable","c",$selectedString)).",c.vm_size ORDER BY ".str_replace("execTable","e",str_replace("clusterTable","c",$selectedString)).",c.vm_size DESC;";
$execs = $dbUtils->get_rows($query);
*/
$execs = $dbUtils->get_rows("SELECT c.datanodes,e.exec_type,c.vm_OS,c.vm_size,(e.exe_time * (c.cost_hour/3600)) as cost,e.*,c.* FROM execs e JOIN clusters c USING (id_cluster) INNER JOIN ( SELECT c2.datanodes,e2.exec_type,c2.vm_OS,c2.vm_size as vmsize,MIN(e2.exe_time) as minexe from execs e2 JOIN clusters c2 USING (id_cluster) WHERE 1 {$where_configs} GROUP BY c2.datanodes,e2.exec_type,c2.vm_OS,c2.vm_size ) t ON t.minexe = e.exe_time AND t.datanodes = c.datanodes AND t.vmsize = c.vm_size WHERE 1 {$filter_execs} GROUP BY c.datanodes,e.exec_type,c.vm_OS,c.vm_size ORDER BY c.datanodes ASC,c.vm_OS,c.vm_size DESC;");
$vmSizes = array();
$categories = array();
$dataNodes = array();
$vmOS = array();
$execTypes = array();
foreach ($execs as &$exec) {
if (!isset($dataNodes[$exec['datanodes']])) {
$dataNodes[$exec['datanodes']] = 1;
$categories[] = $exec['datanodes'];
}
if (!isset($vmOS[$exec['vm_OS']])) {
$vmOS[$exec['vm_OS']] = 1;
}
if (!isset($execTypes[$exec['exec_type']])) {
$execTypes[$exec['exec_type']] = 1;
}
$vmSizes[$exec['vm_size']][$exec['exec_type']][$exec['vm_OS']][$exec['datanodes']] = array(round($exec['exe_time'], 2), round($exec['cost'], 2));
}
$i = 0;
$seriesColors = array('#7cb5ec', '#434348', '#90ed7d', '#f7a35c', '#8085e9', '#f15c80', '#e4d354', '#2b908f', '#f45b5b', '#91e8e1');
$series = array();
foreach ($vmSizes as $vmSize => $value) {
foreach ($execTypes as $execType => $typevalue) {
foreach ($vmOS as $OS => $osvalue) {
if (isset($vmSizes[$vmSize][$execType][$OS])) {
if ($i == sizeof($seriesColors)) {
$i = 0;
}
$costSeries = array('name' => "{$vmSize} {$execType} {$OS} Run cost", 'type' => 'spline', 'dashStyle' => 'longdash', 'yAxis' => 0, 'data' => array(), 'tooltip' => array('valueSuffix' => ' US$'), 'color' => $seriesColors[$i]);
$timeSeries = array('name' => "{$vmSize} {$execType} {$OS} Run execution time", 'type' => 'spline', 'yAxis' => 1, 'data' => array(), 'tooltip' => array('valueSuffix' => ' s'), 'color' => $seriesColors[$i++]);
foreach ($dataNodes as $datanodes => $dvalue) {
if (!isset($value[$execType][$OS][$datanodes])) {
$costSeries['data'][] = "null";
$timeSeries['data'][] = "null";
} else {
$costSeries['data'][] = $value[$execType][$OS][$datanodes][1];
$timeSeries['data'][] = $value[$execType][$OS][$datanodes][0];
}
}
$series[] = $timeSeries;
$series[] = $costSeries;
}
}
}
}
} catch (\Exception $e) {
$this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n");
}
echo $this->container->getTwig()->render('nodeseval/nodes_evaluation.html.twig', array('selected' => 'Number of Nodes Evaluation', 'highcharts_js' => HighCharts::getHeader(), 'categories' => json_encode($categories), 'seriesData' => str_replace('"null"', 'null', json_encode($series)), 'options' => Utils::getFilterOptions($dbUtils), 'datefrom' => $datefrom, 'dateto' => $dateto, 'benchs' => $benchs, 'nets' => $nets, 'disks' => $disks, 'blk_sizes' => $blk_sizes, 'comps' => $comps, 'id_clusters' => $id_clusters, 'mapss' => $mapss, 'replications' => $replications, 'iosfs' => $iosfs, 'iofilebufs' => $iofilebufs, 'money' => $money, 'datanodess' => $datanodes, 'bench_types' => $benchtype, 'vm_sizes' => $vm_sizes, 'vm_coress' => $vm_coress, 'vm_RAMs' => $vm_RAMs, 'vm_OS' => $vm_OS, 'hadoop_versions' => $hadoop_versions, 'types' => $types, 'providers' => $provider, 'filters' => $filters, 'allunchecked' => $allunchecked, 'select_multiple_benchs' => false, 'minexetime' => $minexetime, 'maxexetime' => $maxexetime, 'preset' => $preset, 'selPreset' => $selPreset, 'select_multiple_benchs' => false));
}