public function __construct(\alojaweb\inc\DBUtils $dbConnection) { $this->dbConnection = $dbConnection; /* In this array there are the filter names with its default options * that will be overwritten by the given custom defaults and options if given * Array with filter => filter specific settings * * Specific settings is an array with * types: inputText, inputNumber[{le,ge}], inputDate[{le,ge}], selectOne, selectMultiple, checkbox[Negated] * default: null (any), array(values) * table: associated DB table name * parseFunction: function to parse special filter, for filters that need a lot of customization * * Very custom filters such as advanced filters not in this array * */ $this->filters = array('bench' => array('table' => 'execs', 'default' => array('terasort', 'wordcount'), 'type' => 'selectMultiple', 'label' => 'Benchmarks:'), 'datasize' => array('database' => 'aloja2', 'table' => 'execs', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Datasize: ', 'beautifier' => function ($value) { return Utils::beautifyDatasize($value); }, 'parseFunction' => 'parseDatasize'), 'scale_factor' => array('table' => 'execs', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Scale factor: '), 'bench_type' => array('table' => 'execs', 'default' => array('HiBench'), 'type' => 'selectOne', 'label' => 'Bench suite:'), 'net' => array('table' => 'execs', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Network:', 'beautifier' => function ($value) { return Utils::getNetworkName($value); }), 'disk' => array('table' => 'execs', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Disk:'), 'blk_size' => array('table' => 'execs', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Block size (b):', 'beautifier' => function ($value) { return $value . ' MB'; }), 'comp' => array('table' => 'execs', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Compression (c):', 'beautifier' => function ($value) { return Utils::getCompressionName($value); }), 'id_cluster' => array('table' => 'execs', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Clusters (CL):', 'beautifier' => function ($value) { return $this->filters['id_cluster']['namesClusters'][$value]; }, 'generateChoices' => function () { $choices = $this->dbConnection->get_rows("select distinct id_cluster,CONCAT_WS('/',LPAD(id_cluster,2,0),c.vm_size,CONCAT(c.datanodes,'Dn')) as name from aloja2.execs e join aloja2.clusters c using (id_cluster) WHERE 1 " . DBUtils::getFilterExecs(' ') . " ORDER BY c.name ASC"); $returnChoices = array(); foreach ($choices as $choice) { $returnChoices[] = $choice['id_cluster']; //Not nice, but saves multiple queries to DB in the beautifier $this->filters['id_cluster']['namesClusters'][$choice['id_cluster']] = $choice['name']; } return $returnChoices; }), 'maps' => array('table' => 'execs', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Maps:', 'beautifier' => function ($value) { if ($value == 0) { return 'N/A'; } else { return $value; } }), 'replication' => array('table' => 'execs', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Replication (r):'), 'iosf' => array('table' => 'execs', 'default' => null, 'type' => 'selectMultiple', 'label' => 'I/O sort factor (I):'), 'iofilebuf' => array('table' => 'execs', 'default' => null, 'type' => 'selectMultiple', 'label' => 'I/O file buffer:', 'beautifier' => function ($value) { $suffix = ' KB'; if ($value >= 1024) { $value /= 1024; $suffix = ' MB'; } return $value . $suffix; }), 'provider' => array('table' => 'clusters', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Provider:'), 'vm_OS' => array('table' => 'clusters', 'default' => null, 'type' => 'selectMultiple', 'label' => 'VM OS:'), 'datanodes' => array('table' => 'clusters', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Cluster datanodes:'), 'vm_size' => array('table' => 'clusters', 'default' => null, 'type' => 'selectMultiple', 'label' => 'VM Size:'), 'vm_cores' => array('table' => 'clusters', 'default' => null, 'type' => 'selectMultiple', 'label' => 'VM cores:'), 'vm_RAM' => array('table' => 'clusters', 'default' => null, 'type' => 'selectMultiple', 'label' => 'VM RAM:', 'beautifier' => function ($value) { if ($value * 10 % 10 != 0) { return number_format($value, 1) . ' GB'; } else { return number_format($value, 0) . ' GB'; } }), 'type' => array('table' => 'clusters', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Cluster type:'), 'hadoop_version' => array('table' => 'execs', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Hadoop version:'), 'minexetime' => array('table' => 'execs', 'field' => 'exe_time', 'default' => Utils::in_dev() ? 1 : 50, 'type' => 'inputNumberge', 'label' => 'Min exec time:'), 'maxexetime' => array('table' => 'execs', 'field' => 'exe_time', 'default' => null, 'type' => 'inputNumberle', 'label' => 'Max exec time:'), 'datefrom' => array('table' => 'execs', 'field' => 'start_time', 'default' => null, 'type' => 'inputDatege', 'label' => 'Date from:'), 'dateto' => array('table' => 'execs', 'field' => 'end_time', 'default' => null, 'type' => 'inputDatele', 'label' => 'Date to:'), 'money' => array('table' => 'mixed', 'field' => '(clustersAlias.cost_hour/3600)*execsAlias.exe_time', 'default' => null, 'type' => 'inputNumberle', 'label' => 'Max cost (US$):'), 'valid' => array('table' => 'execs', 'field' => 'valid', 'type' => 'checkbox', 'default' => 1, 'label' => 'Only valid execs'), 'filter' => array('table' => 'execs', 'field' => 'filter', 'type' => 'checkbox', 'default' => 1, 'label' => 'Filter', 'parseFunction' => function () { $whereClause = ""; if (isset($_GET['filter'])) { $values = 1; } else { if (!$this->formisSubmitted()) { $values = $this->filters['filter']['default']; } else { $values = 0; } } if ($values) { $whereClause = " AND execsAlias.filter = 0 "; } return array('currentChoice' => $values, 'whereClause' => $whereClause); }), 'prepares' => array('table' => 'execs', 'type' => 'checkbox', 'default' => Utils::in_dev() ? 1 : 0, 'label' => 'Include prepares', 'parseFunction' => function () { $whereClause = ""; $values = 0; if (isset($_GET['prepares'])) { $values = 1; } else { $values = $this->filters['prepares']['default']; if (!$values) { $whereClause = " AND execsAlias.bench NOT LIKE 'prep_%' "; } } return array('currentChoice' => $values, 'whereClause' => $whereClause); }), 'perf_details' => array('table' => 'execs', 'type' => 'checkbox', 'default' => 0, 'label' => 'Only execs with perf details'), 'prediction_model' => array('type' => 'selectOne', 'default' => null, 'label' => 'Reference Model: ', 'generateChoices' => function () { $query = "SELECT DISTINCT id_learner FROM aloja_ml.predictions"; $retval = $this->dbConnection->get_rows($query); return array_column($retval, "id_learner"); }, 'parseFunction' => function () { $choice = isset($_GET['prediction_model']) ? Utils::get_GET_stringArray('prediction_model') : array(""); if ($choice = array("")) { $query = "SELECT DISTINCT id_learner FROM aloja_ml.predictions LIMIT 1"; $choice = $this->dbConnection->get_rows($query)[0]['id_learner']; } return array('whereClause' => '', 'currentChoice' => $choice); }, 'filterGroup' => 'MLearning'), 'upred' => array('type' => 'checkbox', 'default' => 0, 'label' => 'Use predictions', 'parseFunction' => function () { $choice = !isset($_GET['upred']) ? 0 : 1; return array('whereClause' => '', 'currentChoice' => $choice); }, 'filterGroup' => 'MLearning'), 'uobsr' => array('type' => 'checkbox', 'default' => 1, 'label' => 'Use observations', 'parseFunction' => function () { $choice = !isset($_GET['uobsr']) && $this->formIssubmitted() ? 0 : 1; return array('whereClause' => '', 'currentChoice' => $choice); }, 'filterGroup' => 'MLearning'), 'warning' => array('field' => 'outlier', 'table' => 'ml_predictions', 'type' => 'checkbox', 'default' => 0, 'label' => 'Show warnings', 'parseFunction' => function () { $learner = $this->filters['prediction_model']['currentChoice']; $whereClause = ""; $values = isset($_GET['warning']) ? 1 : 0; if ($values && !empty($learner)) { $whereClause = " AND (ml_predictionsAlias.outlier <= {$values} OR ml_predictionsAlias.outlier IS NULL) " . "AND (ml_predictionsAlias.id_learner = '{$learner[0]}' OR ml_predictionsAlias.id_learner IS NULL)"; } return array('currentChoice' => $values, 'whereClause' => $whereClause); }, 'filterGroup' => 'MLearning'), 'outlier' => array('table' => 'ml_predictions', 'type' => 'checkbox', 'default' => 0, 'label' => 'Show outliers', 'parseFunction' => function () { $learner = $this->filters['prediction_model']['currentChoice']; $whereClause = ""; $values = isset($_GET['outlier']) ? 2 : 0; if ($values && !empty($learner)) { $whereClause = " AND (ml_predictionsAlias.outlier <= 2 OR ml_predictionsAlias.outlier IS NULL) " . "AND (ml_predictionsAlias.id_learner = '{$learner}' OR ml_predictionsAlias.id_learner IS NULL)"; $values = 1; } else { if (!empty($learner) && !isset($_GET['warning'])) { $whereClause = " AND (ml_predictionsAlias.outlier = 0 OR ml_predictionsAlias.outlier IS NULL) " . "AND (ml_predictionsAlias.id_learner = '{$learner}' OR ml_predictionsAlias.id_learner IS NULL)"; } } return array('currentChoice' => $values, 'whereClause' => $whereClause); }, 'filterGroup' => 'MLearning')); $this->aliasesTables = array('execs' => 'e', 'clusters' => 'c', 'ml_predictions' => 'p'); //To render groups on template. Rows are of 2 columns each. emptySpace puts an empty element on the rendered row $this->filterGroups = array('basic' => array('label' => 'Basic filters', 'filters' => array('money', 'bench', 'bench_type', 'datasize', 'scale_factor', 'id_cluster', 'net', 'disk'), 'tabOpenDefault' => true), 'hardware' => array('label' => 'Hardware', 'filters' => array('datanodes', 'vm_size', 'vm_cores', 'vm_RAM', 'type', 'provider', 'vm_OS'), 'tabOpenDefault' => false), 'hadoop' => array('label' => 'Hadoop', 'filters' => array('maps', 'comp', 'replication', 'blk_size', 'iosf', 'iofilebuf', 'hadoop_version'), 'tabOpenDefault' => false), 'advanced' => array('label' => 'Advanced filters', 'filters' => array('valid', 'filter', 'prepares', 'perf_details', 'datefrom', 'dateto', 'minexetime', 'maxexetime'), 'tabOpenDefault' => false), 'MLearning' => array('label' => 'Machine Learning', 'filters' => array('prediction_model', 'upred', 'uobsr', 'warning', 'outlier'), 'tabOpenDefault' => true)); }
public function mlparamEvaluationAction() { $rows = $categories = $series = ''; $must_wait = 'NO'; try { $dbml = new \PDO($this->container->get('config')['db_conn_chain_ml'], $this->container->get('config')['mysql_user'], $this->container->get('config')['mysql_pwd']); $dbml->setAttribute(\PDO::ATTR_ERRMODE, \PDO::ERRMODE_EXCEPTION); $dbml->setAttribute(\PDO::ATTR_EMULATE_PREPARES, false); $db = $this->container->getDBUtils(); $where_configs = ''; $preset = null; if (count($_GET) <= 1 || count($_GET) == 2 && array_key_exists('parameval', $_GET) || count($_GET) == 2 && array_key_exists('current_model', $_GET)) { $preset = Utils::setDefaultPreset($db, 'mlparameval'); } $selPreset = isset($_GET['presets']) ? $_GET['presets'] : "none"; $params = array(); $param_names = array('benchs', 'nets', 'disks', 'mapss', 'iosfs', 'replications', 'iofilebufs', 'comps', 'blk_sizes', 'id_clusters', 'datanodess', 'bench_types', 'vm_sizes', 'vm_coress', 'vm_RAMs', 'types'); // Order is important foreach ($param_names as $p) { $params[$p] = Utils::read_params($p, $where_configs, FALSE); sort($params[$p]); } $money = Utils::read_params('money', $where_configs); $paramEval = isset($_GET['parameval']) && $_GET['parameval'] != '' ? $_GET['parameval'] : 'maps'; $minExecs = isset($_GET['minexecs']) ? $_GET['minexecs'] : -1; $minExecsFilter = ""; // FIXME PATCH FOR PARAM LIBRARIES WITHOUT LEGACY $where_configs = str_replace("AND .", "AND ", $where_configs); $where_configs = str_replace("`id_cluster`", "e.`id_cluster`", $where_configs); if ($minExecs > 0) { $minExecsFilter = "HAVING COUNT(*) > {$minExecs}"; } $filter_execs = DBUtils::getFilterExecs(); $options = Utils::getFilterOptions($db); $paramOptions = array(); foreach ($options[$paramEval] as $option) { if ($paramEval == 'id_cluster') { $paramOptions[] = $option['name']; } else { if ($paramEval == 'comp') { $paramOptions[] = Utils::getCompressionName($option[$paramEval]); } else { if ($paramEval == 'net') { $paramOptions[] = Utils::getNetworkName($option[$paramEval]); } else { if ($paramEval == 'disk') { $paramOptions[] = Utils::getDisksName($option[$paramEval]); } else { $paramOptions[] = $option[$paramEval]; } } } } } $benchOptions = $db->get_rows("SELECT DISTINCT bench FROM execs e LEFT JOIN clusters c ON e.id_cluster = c.id_cluster WHERE 1 {$filter_execs} {$where_configs} GROUP BY {$paramEval}, bench order by {$paramEval}"); // get the result rows $query = "SELECT count(*) as count, {$paramEval}, e.id_exec, exec as conf, bench, " . "exe_time, avg(exe_time) avg_exe_time, min(exe_time) min_exe_time " . "from execs e LEFT JOIN clusters c ON e.id_cluster = c.id_cluster WHERE 1 {$filter_execs} {$where_configs}" . "GROUP BY {$paramEval}, bench {$minExecsFilter} order by bench,{$paramEval}"; $rows = $db->get_rows($query); if (!$rows) { throw new \Exception("No results for query!"); } $arrayBenchs = array(); foreach ($paramOptions as $param) { foreach ($benchOptions as $bench) { $arrayBenchs[$bench['bench']][$param] = null; $arrayBenchs[$bench['bench']][$param]['y'] = 0; $arrayBenchs[$bench['bench']][$param]['count'] = 0; } } $series = array(); $bench = ''; foreach ($rows as $row) { if ($paramEval == 'comp') { $row[$paramEval] = Utils::getCompressionName($row['comp']); } else { if ($paramEval == 'id_cluster') { $row[$paramEval] = Utils::getClusterName($row[$paramEval], $db); } else { if ($paramEval == 'net') { $row[$paramEval] = Utils::getNetworkName($row['net']); } else { if ($paramEval == 'disk') { $row[$paramEval] = Utils::getDisksName($row['disk']); } else { if ($paramEval == 'iofilebuf') { $row[$paramEval] /= 1024; } } } } } $arrayBenchs[$row['bench']][$row[$paramEval]]['y'] = round((int) $row['avg_exe_time'], 2); $arrayBenchs[$row['bench']][$row[$paramEval]]['count'] = (int) $row['count']; } // ---------------------------------------------------- // Add predictions to the series // ---------------------------------------------------- $jsonData = $jsonHeader = "[]"; $instance = ""; $arrayBenchs_pred = array(); // FIXME PATCH FOR PARAM LIBRARIES WITHOUT LEGACY $where_configs = str_replace("AND .", "AND ", $where_configs); // compose instance $instance = MLUtils::generateSimpleInstance($param_names, $params, true, $db); $model_info = MLUtils::generateModelInfo($param_names, $params, true, $db); $instances = MLUtils::generateInstances($param_names, $params, true, $db); // model for filling $possible_models = $possible_models_id = array(); MLUtils::findMatchingModels($model_info, $possible_models, $possible_models_id, $dbml); $current_model = ""; if (array_key_exists('current_model', $_GET) && in_array($_GET['current_model'], $possible_models_id)) { $current_model = $_GET['current_model']; } if (!empty($possible_models_id)) { if ($current_model == "") { $query = "SELECT AVG(ABS(exe_time - pred_time)) AS MAE, AVG(ABS(exe_time - pred_time)/exe_time) AS RAE, p.id_learner FROM predictions p, learners l WHERE l.id_learner = p.id_learner AND p.id_learner IN ('" . implode("','", $possible_models_id) . "') AND predict_code > 0 ORDER BY MAE LIMIT 1"; $result = $dbml->query($query); $row = $result->fetch(); $current_model = $row['id_learner']; } $config = $instance . '-' . $current_model . "-parameval"; $query_cache = "SELECT count(*) as total FROM trees WHERE id_learner = '" . $current_model . "' AND model = '" . $model_info . "'"; $is_cached_mysql = $dbml->query($query_cache); $tmp_result = $is_cached_mysql->fetch(); $is_cached = $tmp_result['total'] > 0; $ret_data = null; if (!$is_cached) { // Call to MLFindAttributes, to fetch data $_GET['pass'] = 2; $_GET['unseen'] = 1; $_GET['current_model'] = $current_model; $mlfa1 = new MLFindAttributesController(); $mlfa1->container = $this->container; $ret_data = $mlfa1->mlfindattributesAction(); if ($ret_data == 1) { $must_wait = "YES"; $jsonData = $jsonHeader = '[]'; } else { $is_cached_mysql = $dbml->query($query_cache); $tmp_result = $is_cached_mysql->fetch(); $is_cached = $tmp_result['total'] > 0; } } if ($is_cached) { $must_wait = 'NO'; $query = "SELECT count(*) as count, {$paramEval}, bench, exe_time, avg(pred_time) avg_pred_time, min(pred_time) min_pred_time " . "FROM predictions e WHERE e.id_learner = '" . $current_model . "' {$filter_execs} {$where_configs}" . "GROUP BY {$paramEval}, bench {$minExecsFilter} order by bench, {$paramEval}"; $result = $dbml->query($query); // Initialize array foreach ($paramOptions as $param) { foreach ($benchOptions as $bench) { $arrayBenchs_pred[$bench['bench'] . '_pred'][$param] = null; $arrayBenchs_pred[$bench['bench'] . '_pred'][$param]['y'] = 0; $arrayBenchs_pred[$bench['bench'] . '_pred'][$param]['count'] = 0; } } foreach ($result as $row) { $bench_n = $row['bench'] . '_pred'; $class = $row[$paramEval]; if ($paramEval == 'comp') { $value = Utils::getCompressionName($class); } else { if ($paramEval == 'id_cluster') { $value = Utils::getClusterName($class, $db); } else { if ($paramEval == 'net') { $value = Utils::getNetworkName($class); } else { if ($paramEval == 'disk') { $value = Utils::getDisksName($class); } else { if ($paramEval == 'iofilebuf') { $value = $class / 1024; } else { $value = $class; } } } } } if (!in_array($value, $paramOptions)) { $paramOptions[] = $value; foreach ($benchOptions as $bench) { $arrayBenchs_pred[$bench['bench'] . '_pred'][$value] = null; $arrayBenchs_pred[$bench['bench'] . '_pred'][$value]['y'] = 0; $arrayBenchs_pred[$bench['bench'] . '_pred'][$value]['count'] = 0; $arrayBenchs[$bench['bench']][$value] = null; $arrayBenchs[$bench['bench']][$value]['y'] = 0; $arrayBenchs[$bench['bench']][$value]['count'] = 0; } } $arrayBenchs_pred[$bench_n][$value]['y'] = (int) $row['avg_pred_time']; $arrayBenchs_pred[$bench_n][$value]['count'] = (int) $row['count']; } } } // ---------------------------------------------------- // END - Add predictions to the series // ---------------------------------------------------- asort($paramOptions); foreach ($arrayBenchs as $key => $arrayBench) { $caregories = ''; $data_a = null; $data_p = null; foreach ($paramOptions as $param) { if ($arrayBenchs[$key][$param]['count'] > 0 && empty($arrayBenchs_pred) || !empty($arrayBenchs_pred) && ($arrayBenchs_pred[$key . '_pred'][$param]['count'] > 0 || $arrayBenchs[$key][$param]['count'] > 0)) { $data_a[] = $arrayBenchs[$key][$param]; if (!empty($arrayBenchs_pred)) { $data_p[] = $arrayBenchs_pred[$key . '_pred'][$param]; } $categories = $categories . "'{$param} " . Utils::getParamevalUnit($paramEval) . "',"; // FIXME - Redundant n times performed... don't care now } } $series[] = array('name' => $key, 'data' => $data_a); if (!empty($arrayBenchs_pred)) { $series[] = array('name' => $key . '_pred', 'data' => $data_p); } } $series = json_encode($series); if (!empty($arrayBenchs_pred)) { $colors = "['#7cb5ec','#9cd5fc','#434348','#636368','#90ed7d','#b0fd9d','#f7a35c','#f7c37c','#8085e9','#a0a5f9','#f15c80','#f17ca0','#e4d354','#f4f374','#8085e8','#a0a5f8','#8d4653','#ad6673','#91e8e1','#b1f8f1']"; } else { $colors = "['#7cb5ec','#434348','#90ed7d','#f7a35c','#8085e9','#f15c80','#e4d354','#8085e8','#8d4653','#91e8e1']"; } } catch (\Exception $e) { $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n"); $series = $jsonHeader = $colors = '[]'; $instance = $current_model = ''; $possible_models = $possible_models_id = array(); $must_wait = 'NO'; } echo $this->container->getTwig()->render('mltemplate/mlconfigperf.html.twig', array('selected' => 'mlparameval', 'title' => 'Improvement of Hadoop Execution by SW and HW Configurations', 'categories' => $categories, 'series' => $series, 'benchs' => $params['benchs'], 'nets' => $params['nets'], 'disks' => $params['disks'], 'blk_sizes' => $params['blk_sizes'], 'comps' => $params['comps'], 'id_clusters' => $params['id_clusters'], 'mapss' => $params['mapss'], 'replications' => $params['replications'], 'iosfs' => $params['iosfs'], 'iofilebufs' => $params['iofilebufs'], 'datanodess' => $params['datanodess'], 'bench_types' => $params['bench_types'], 'vm_sizes' => $params['vm_sizes'], 'vm_coress' => $params['vm_coress'], 'vm_RAMs' => $params['vm_RAMs'], 'types' => $params['types'], 'money' => $money, 'paramEval' => $paramEval, 'instance' => $instance, 'models' => '<li>' . implode('</li><li>', $possible_models) . '</li>', 'models_id' => $possible_models_id, 'current_model' => $current_model, 'gammacolors' => $colors, 'must_wait' => $must_wait, 'preset' => $preset, 'selPreset' => $selPreset, 'options' => Utils::getFilterOptions($db))); }
public function mlfindattributesAction() { $instance = $instances = $message = $tree_descriptor = $model_html = $config = ''; $possible_models = $possible_models_id = $other_models = array(); $must_wait = 'NO'; try { $dbml = new \PDO($this->container->get('config')['db_conn_chain'], $this->container->get('config')['mysql_user'], $this->container->get('config')['mysql_pwd']); $dbml->setAttribute(\PDO::ATTR_ERRMODE, \PDO::ERRMODE_EXCEPTION); $dbml->setAttribute(\PDO::ATTR_EMULATE_PREPARES, false); $db = $this->container->getDBUtils(); if (array_key_exists('dump', $_GET)) { $dump = $_GET["dump"]; unset($_GET["dump"]); } if (array_key_exists('pass', $_GET)) { $pass = $_GET["pass"]; unset($_GET["pass"]); } $this->buildFilters(array('current_model' => array('type' => 'selectOne', 'default' => null, 'label' => 'Model to use: ', 'generateChoices' => function () { return array(); }, 'parseFunction' => function () { $choice = isset($_GET['current_model']) ? $_GET['current_model'] : array(""); return array('whereClause' => '', 'currentChoice' => $choice); }, 'filterGroup' => 'MLearning'), 'unseen' => array('type' => 'checkbox', 'default' => 1, 'label' => 'Predict with unseen atributes ⚠', 'parseFunction' => function () { $choice = isset($_GET['unseen']) && !isset($_GET['unseen']) ? 0 : 1; return array('whereClause' => '', 'currentChoice' => $choice); }, 'filterGroup' => 'MLearning'), 'minexetime' => array('default' => 0), 'valid' => array('default' => 0), 'filter' => array('default' => 0), 'prepares' => array('default' => 1))); $this->buildFilterGroups(array('MLearning' => array('label' => 'Machine Learning', 'tabOpenDefault' => true, 'filters' => array('current_model', 'unseen')))); $where_configs = $this->filters->getWhereClause(); $param_names = array('bench', 'net', 'disk', 'maps', 'iosf', 'replication', 'iofilebuf', 'comp', 'blk_size', 'id_cluster', 'datanodes', 'vm_OS', 'vm_cores', 'vm_RAM', 'provider', 'vm_size', 'type', 'bench_type', 'hadoop_version'); // Order is important $params = $this->filters->getFiltersSelectedChoices($param_names); foreach ($param_names as $p) { if (!is_null($params[$p]) && is_array($params[$p])) { sort($params[$p]); } } $learnParams = $this->filters->getFiltersSelectedChoices(array('current_model', 'unseen')); $param_current_model = $learnParams['current_model']; $unseen = $learnParams['unseen'] ? true : false; // FIXME PATCH FOR PARAM LIBRARIES WITHOUT LEGACY $where_configs = str_replace("AND .", "AND ", $where_configs); $jsonData = $jsonHeader = "[]"; $mae = $rae = 0; // compose instance $model_info = MLUtils::generateModelInfo($this->filters, $param_names, $params, $unseen); $instance = MLUtils::generateSimpleInstance($this->filters, $param_names, $params, $unseen); $instances = MLUtils::generateInstances($this->filters, $param_names, $params, $unseen, $db); // Model for filling MLUtils::findMatchingModels($model_info, $possible_models, $possible_models_id, $dbml); $current_model = ''; if (!is_null($possible_models_id) && in_array($param_current_model, $possible_models_id)) { $current_model = $param_current_model; } // Other models for filling $where_models = ''; if (!empty($possible_models_id)) { $where_models = " WHERE id_learner NOT IN ('" . implode("','", $possible_models_id) . "')"; } $result = $dbml->query("SELECT id_learner FROM aloja_ml.learners" . $where_models); foreach ($result as $row) { $other_models[] = $row['id_learner']; } if (!empty($possible_models_id) || $current_model != "") { $result = $dbml->query("SELECT id_learner, model, algorithm, CASE WHEN `id_learner` IN ('" . implode("','", $possible_models_id) . "') THEN 'COMPATIBLE' ELSE 'NOT MATCHED' END AS compatible FROM aloja_ml.learners"); foreach ($result as $row) { $model_html = $model_html . "<li>" . $row['id_learner'] . " => " . $row['algorithm'] . " : " . $row['compatible'] . " : " . $row['model'] . "</li>"; } if ($current_model == "") { $query = "SELECT AVG(ABS(exe_time - pred_time)) AS MAE, AVG(ABS(exe_time - pred_time)/exe_time) AS RAE, p.id_learner FROM aloja_ml.predictions p, aloja_ml.learners l WHERE l.id_learner = p.id_learner AND p.id_learner IN ('" . implode("','", $possible_models_id) . "') AND predict_code > 0 ORDER BY MAE LIMIT 1"; $result = $dbml->query($query); $row = $result->fetch(); $current_model = $row['id_learner']; } $config = $instance . '-' . $current_model . '-' . ($unseen ? 'U' : 'R'); $is_cached_mysql = $dbml->query("SELECT count(*) as total FROM aloja_ml.trees WHERE id_findattrs = '" . md5($config) . "'"); $tmp_result = $is_cached_mysql->fetch(); $is_cached = $tmp_result['total'] > 0; $tmp_file = md5($config) . '.tmp'; $in_process = file_exists(getcwd() . '/cache/query/' . md5($config) . '.lock'); $finished_process = $in_process && (int) shell_exec('ls ' . getcwd() . '/cache/query/' . md5($config) . '-*.lock | wc -w ') == count($instances); if (!$in_process && !$finished_process && !$is_cached) { // Retrieve file model from DB $query = "SELECT file FROM aloja_ml.model_storage WHERE id_hash='" . $current_model . "' AND type='learner';"; $result = $dbml->query($query); $row = $result->fetch(); $content = $row['file']; $filemodel = getcwd() . '/cache/query/' . $current_model . '-object.rds'; $fp = fopen($filemodel, 'w'); fwrite($fp, $content); fclose($fp); // Run the predictor exec('cd ' . getcwd() . '/cache/query ; touch ' . md5($config) . '.lock ; rm -f ' . $tmp_file); $count = 1; foreach ($instances as $inst) { exec(getcwd() . '/resources/queue -d -c "cd ' . getcwd() . '/cache/query ; ../../resources/aloja_cli.r -m aloja_predict_instance -l ' . $current_model . ' -p inst_predict=\'' . $inst . '\' -v | grep -v \'Prediction\' >>' . $tmp_file . ' 2>/dev/null; touch ' . md5($config) . '-' . $count++ . '.lock" >/dev/null 2>&1 &'); } } $finished_process = (int) shell_exec('ls ' . getcwd() . '/cache/query/' . md5($config) . '-*.lock | wc -w ') == count($instances); if ($finished_process && !$is_cached) { // Read results and dump to DB $i = 0; $token = 0; $token_i = 0; $query = "INSERT IGNORE INTO aloja_ml.predictions (id_exec,exe_time,bench,net,disk,maps,iosf,replication,iofilebuf,comp,blk_size,id_cluster,datanodes,vm_OS,vm_cores,vm_RAM,provider,vm_size,type,bench_type,hadoop_version,pred_time,id_learner,instance,predict_code) VALUES "; if (($handle = fopen(getcwd() . '/cache/query/' . $tmp_file, "r")) !== FALSE) { while (($line = fgets($handle, 1000)) !== FALSE && $i < 1000) { if ($line == '') { break; } // Fetch Real Value $inst_aux = preg_split("/\\s+/", $line); $query_var = "SELECT AVG(exe_time) as AVG, id_exec, outlier FROM aloja_ml.predictions WHERE instance = '" . $inst_aux[1] . "' AND predict_code > 0"; $result = $dbml->query($query_var); $row = $result->fetch(); $realexecval = is_null($row['AVG']) || $row['outlier'] == 2 ? 0 : $row['AVG']; $realid_exec = is_null($row['id_exec']) || $row['outlier'] == 2 ? 0 : $row['id_exec']; $query_var = "SELECT count(*) as num FROM aloja_ml.predictions WHERE instance = '" . $inst_aux[1] . "' AND id_learner = '" . $current_model . "'"; $result = $dbml->query($query_var); $row = $result->fetch(); // Insert instance values if ($row['num'] == 0) { $token_i = 1; $selected_instance = preg_replace('/,Cmp(\\d+),/', ',${1},', $inst_aux[1]); $selected_instance = preg_replace('/,Cl(\\d+),/', ',${1},', $selected_instance); if ($token > 0) { $query = $query . ","; } $token = 1; $query = $query . "('" . $realid_exec . "','" . $realexecval . "','" . str_replace(",", "','", $selected_instance) . "','" . $inst_aux[2] . "','" . $current_model . "','" . $inst_aux[1] . "','0') "; } $i++; if ($i % 100 == 0 && $token_i > 0) { if ($dbml->query($query) === FALSE) { throw new \Exception('Error when saving into DB'); } $query = "INSERT IGNORE INTO aloja_ml.predictions (id_exec,exe_time,bench,net,disk,maps,iosf,replication,iofilebuf,comp,blk_size,id_cluster,datanodes,vm_OS,vm_cores,vm_RAM,provider,vm_size,type,bench_type,hadoop_version,pred_time,id_learner,instance,predict_code) VALUES "; $token = 0; $token_i = 0; } } if ($token_i > 0) { if ($dbml->query($query) === FALSE) { throw new \Exception('Error when saving into DB'); } } // Descriptive Tree $tree_descriptor = shell_exec(getcwd() . '/resources/aloja_cli.r -m aloja_representative_tree -p method=ordered:dump_file="' . getcwd() . '/cache/query/' . $tmp_file . '":output="html" -v 2> /dev/null'); $tree_descriptor = substr($tree_descriptor, 5, -2); $query = "INSERT INTO aloja_ml.trees (id_findattrs,id_learner,instance,model,tree_code) VALUES ('" . md5($config) . "','" . $current_model . "','" . $instance . "','" . $model_info . "','" . $tree_descriptor . "')"; if ($dbml->query($query) === FALSE) { throw new \Exception('Error when saving tree into DB'); } // remove remaining locks shell_exec('rm -f ' . getcwd() . '/cache/query/' . md5($config) . '*.lock'); // Remove temporal files $output = shell_exec('rm -f ' . getcwd() . '/cache/query/' . md5($config) . '.tmp'); $is_cached = true; } fclose($handle); } if (!$is_cached) { $jsonData = $jsonHeader = $jsonColumns = $jsonColor = '[]'; $must_wait = 'YES'; if (isset($dump)) { $dbml = null; echo "1"; exit(0); } if (isset($pass)) { $dbml = null; return "1"; } } else { if (isset($pass) && $pass == 2) { $dbml = null; return "2"; } // Fetch results and compose JSON $header = array('Benchmark', 'Net', 'Disk', 'Maps', 'IO.SFS', 'Rep', 'IO.FBuf', 'Comp', 'Blk.Size', 'Cluster', 'Datanodes', 'VM.OS', 'VM.Cores', 'VM.RAM', 'Provider', 'VM.Size', 'Type', 'Bench.Type', 'Version', 'Prediction', 'Observed'); $jsonHeader = '[{title:""}'; foreach ($header as $title) { $jsonHeader = $jsonHeader . ',{title:"' . $title . '"}'; } $jsonHeader = $jsonHeader . ']'; $query = "SELECT @i:=@i+1 as num, instance, AVG(pred_time) as pred_time, AVG(exe_time) as exe_time FROM aloja_ml.predictions, (SELECT @i:=0) d WHERE id_learner='" . $current_model . "' " . $where_configs . " GROUP BY instance"; $result = $dbml->query($query); $jsonData = '['; foreach ($result as $row) { if ($jsonData != '[') { $jsonData = $jsonData . ','; } $jsonData = $jsonData . "['" . $row['num'] . "','" . str_replace(",", "','", $row['instance']) . "','" . $row['pred_time'] . "','" . $row['exe_time'] . "']"; } $jsonData = $jsonData . ']'; foreach (range(1, 33) as $value) { $jsonData = str_replace('Cmp' . $value, Utils::getCompressionName($value), $jsonData); } // Fetch MAE & RAE values $query = "SELECT AVG(ABS(exe_time - pred_time)) AS MAE, AVG(ABS(exe_time - pred_time)/exe_time) AS RAE FROM aloja_ml.predictions WHERE id_learner='" . md5($config) . "' AND predict_code > 0"; $result = $dbml->query($query); $row = $result->fetch(); $mae = $row['MAE']; $rae = $row['RAE']; // Dump case if (isset($dump)) { echo "ID" . str_replace(array("[", "]", "{title:\"", "\"}"), array('', '', ''), $jsonHeader) . "\n"; echo str_replace(array('],[', '[[', ']]'), array("\n", '', ''), $jsonData); $dbml = null; exit(0); } if (isset($pass) && $pass == 1) { $retval = "ID" . str_replace(array("[", "]", "{title:\"", "\"}"), array('', '', ''), $jsonHeader) . "\n"; $retval .= str_replace(array('],[', '[[', ']]'), array("\n", '', ''), $jsonData); $dbml = null; return $retval; } // Display Descriptive Tree $query = "SELECT tree_code FROM aloja_ml.trees WHERE id_findattrs = '" . md5($config) . "'"; $result = $dbml->query($query); $row = $result->fetch(); $tree_descriptor = $row['tree_code']; } } else { $message = "There are no prediction models trained for such parameters. Train at least one model in 'ML Prediction' section."; $must_wait = 'NO'; if (isset($dump)) { echo "-1"; exit(0); } if (isset($pass)) { return "-1"; } } $dbml = null; } catch (\Exception $e) { $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n"); $jsonData = $jsonHeader = "[]"; $must_wait = 'NO'; $mae = $rae = 0; $dbml = null; if (isset($pass)) { return "-2"; } } $return_params = array('instance' => $instance, 'jsonData' => $jsonData, 'jsonHeader' => $jsonHeader, 'models' => $model_html, 'models_id' => $possible_models_id, 'other_models_id' => $other_models, 'current_model' => $current_model, 'message' => $message, 'mae' => $mae, 'rae' => $rae, 'must_wait' => $must_wait, 'instance' => $instance, 'instances' => implode("<br/> ", $instances), 'model_info' => $model_info, 'id_findattr' => md5($config), 'unseen' => $unseen, 'tree_descriptor' => $tree_descriptor); $this->filters->setCurrentChoices('current_model', array_merge($possible_models_id, array('---Other models---'), $other_models)); return $this->render('mltemplate/mlfindattributes.html.twig', $return_params); }
public static function changeParamOptions(&$paramOptions, $paramEval) { if ($paramEval == 'comp') { foreach ($paramOptions as &$option) { $option['param'] = Utils::getCompressionName($option['param']); } } }
public function mlparamEvaluationAction() { $rows = $categories = $series = $instance = $model_info = $config = $current_model = $slice_info = ''; $arrayBenchs_pred = $possible_models = $possible_models_id = $other_models = array(); $jsonData = $jsonHeader = "[]"; $must_wait = 'NO'; try { $dbml = new \PDO($this->container->get('config')['db_conn_chain'], $this->container->get('config')['mysql_user'], $this->container->get('config')['mysql_pwd']); $dbml->setAttribute(\PDO::ATTR_ERRMODE, \PDO::ERRMODE_EXCEPTION); $dbml->setAttribute(\PDO::ATTR_EMULATE_PREPARES, false); $db = $this->container->getDBUtils(); if (array_key_exists('parameval', $_GET)) { $paramEval = isset($_GET['parameval']) && Utils::get_GET_string('parameval') != '' ? Utils::get_GET_string('parameval') : 'maps'; unset($_GET["parameval"]); } $this->buildFilters(array('current_model' => array('type' => 'selectOne', 'default' => null, 'label' => 'Model to use: ', 'generateChoices' => function () { return array(); }, 'parseFunction' => function () { $choice = isset($_GET['current_model']) ? $_GET['current_model'] : array(""); return array('whereClause' => '', 'currentChoice' => $choice); }, 'filterGroup' => 'MLearning'), 'minExecs' => array('default' => 0, 'type' => 'inputNumber', 'label' => 'Minimum executions:', 'parseFunction' => function () { return 0; }, 'filterGroup' => 'basic'), 'minexetime' => array('default' => 0), 'valid' => array('default' => 0), 'filter' => array('default' => 0), 'prepares' => array('default' => 0))); $this->buildFilterGroups(array('MLearning' => array('label' => 'Machine Learning', 'tabOpenDefault' => true, 'filters' => array('current_model')))); $where_configs = $this->filters->getWhereClause(); $params = array(); $param_names = array('bench', 'net', 'disk', 'maps', 'iosf', 'replication', 'iofilebuf', 'comp', 'blk_size', 'id_cluster', 'datanodes', 'vm_OS', 'vm_cores', 'vm_RAM', 'provider', 'vm_size', 'type', 'bench_type', 'hadoop_version'); // Order is important $params = $this->filters->getFiltersSelectedChoices($param_names); foreach ($param_names as $p) { if (!is_null($params[$p]) && is_array($params[$p])) { sort($params[$p]); } } $params_additional = array(); $param_names_additional = array('datefrom', 'dateto', 'minexetime', 'maxexetime', 'valid', 'filter'); // Order is important $params_additional = $this->filters->getFiltersSelectedChoices($param_names_additional); $param_variables = $this->filters->getFiltersSelectedChoices(array('current_model', 'minExecs')); $param_current_model = $param_variables['current_model']; $minExecs = $param_variables['minExecs']; $where_configs = str_replace("AND .", "AND ", $where_configs); $where_configs = str_replace("id_cluster", "e.id_cluster", $where_configs); $minExecsFilter = ""; if ($minExecs > 0) { $minExecsFilter = "HAVING COUNT(*) > {$minExecs}"; } $filter_execs = DBUtils::getFilterExecs(); $options = $this->filters->getFilterChoices(); $paramOptions = array(); foreach ($options[$paramEval] as $option) { if ($paramEval == 'comp') { $paramOptions[] = Utils::getCompressionName($option); } else { if ($paramEval == 'net') { $paramOptions[] = Utils::getNetworkName($option); } else { if ($paramEval == 'disk') { $paramOptions[] = Utils::getDisksName($option); } else { $paramOptions[] = $option; } } } } $param_eval_query = $paramEval == 'id_cluster' ? 'e.id_cluster' : $paramEval; $benchOptions = $db->get_rows("SELECT DISTINCT bench FROM aloja2.execs e LEFT JOIN aloja2.clusters c ON e.id_cluster = c.id_cluster WHERE 1 {$filter_execs} {$where_configs} GROUP BY {$param_eval_query}, bench order by {$param_eval_query}"); // get the result rows $query = "SELECT count(*) as count, {$param_eval_query}, e.id_exec, exec as conf, bench, " . "exe_time, avg(exe_time) avg_exe_time, min(exe_time) min_exe_time " . "from aloja2.execs e LEFT JOIN aloja2.clusters c ON e.id_cluster = c.id_cluster WHERE 1 {$filter_execs} {$where_configs}" . "GROUP BY {$param_eval_query},bench {$minExecsFilter} order by bench,{$param_eval_query}"; $rows = $db->get_rows($query); if (!$rows) { throw new \Exception("No results for query!"); } $arrayBenchs = array(); foreach ($paramOptions as $param) { foreach ($benchOptions as $bench) { $arrayBenchs[$bench['bench']][$param] = null; $arrayBenchs[$bench['bench']][$param]['y'] = 0; $arrayBenchs[$bench['bench']][$param]['count'] = 0; } } $series = array(); $bench = ''; foreach ($rows as $row) { if ($paramEval == 'comp') { $row[$paramEval] = Utils::getCompressionName($row['comp']); } else { if ($paramEval == 'net') { $row[$paramEval] = Utils::getNetworkName($row['net']); } else { if ($paramEval == 'disk') { $row[$paramEval] = Utils::getDisksName($row['disk']); } else { if ($paramEval == 'iofilebuf') { $row[$paramEval] /= 1024; } } } } $arrayBenchs[$row['bench']][$row[$paramEval]]['y'] = round((int) $row['avg_exe_time'], 2); $arrayBenchs[$row['bench']][$row[$paramEval]]['count'] = (int) $row['count']; } // ---------------------------------------------------- // Add predictions to the series // ---------------------------------------------------- $param_variables = $this->filters->getFiltersSelectedChoices(array('current_model')); $param_current_model = $param_variables['current_model']; $where_configs = str_replace("AND .", "AND ", $where_configs); // compose instance $instance = MLUtils::generateSimpleInstance($this->filters, $param_names, $params, true); $model_info = MLUtils::generateModelInfo($this->filters, $param_names, $params, true); $slice_info = MLUtils::generateDatasliceInfo($this->filters, $param_names_additional, $params_additional); // model for filling MLUtils::findMatchingModels($model_info, $possible_models, $possible_models_id, $dbml); $current_model = ''; if (!is_null($possible_models_id) && in_array($param_current_model, $possible_models_id)) { $current_model = $param_current_model; } // Other models for filling $where_models = ''; if (!empty($possible_models_id)) { $where_models = " WHERE id_learner NOT IN ('" . implode("','", $possible_models_id) . "')"; } $result = $dbml->query("SELECT id_learner FROM aloja_ml.learners" . $where_models); foreach ($result as $row) { $other_models[] = $row['id_learner']; } if (!empty($possible_models_id)) { if ($current_model == "") { $query = "SELECT AVG(ABS(exe_time - pred_time)) AS MAE, AVG(ABS(exe_time - pred_time)/exe_time) AS RAE, p.id_learner FROM aloja_ml.predictions p, aloja_ml.learners l WHERE l.id_learner = p.id_learner AND p.id_learner IN ('" . implode("','", $possible_models_id) . "') AND predict_code > 0 ORDER BY MAE LIMIT 1"; $result = $dbml->query($query); $row = $result->fetch(); $current_model = $row['id_learner']; } $config = $instance . '-' . $current_model . ' ' . $slice_info . "-parameval"; $query_cache = "SELECT count(*) as total FROM aloja_ml.trees WHERE id_learner = '" . $current_model . "' AND model = '" . $model_info . "'"; $is_cached_mysql = $dbml->query($query_cache); $tmp_result = $is_cached_mysql->fetch(); $is_cached = $tmp_result['total'] > 0; $ret_data = null; if (!$is_cached) { // Call to MLFindAttributes, to fetch data $_GET['pass'] = 2; $_GET['unseen'] = 1; $_GET['current_model'] = $current_model; $mlfa1 = new MLFindAttributesController(); $mlfa1->container = $this->container; $ret_data = $mlfa1->mlfindattributesAction(); if ($ret_data == 1) { $must_wait = "YES"; $jsonData = $jsonHeader = '[]'; } else { $is_cached_mysql = $dbml->query($query_cache); $tmp_result = $is_cached_mysql->fetch(); $is_cached = $tmp_result['total'] > 0; } } if ($is_cached) { $must_wait = 'NO'; $query = "SELECT count(*) as count, {$param_eval_query}, bench, exe_time, avg(pred_time) avg_pred_time, min(pred_time) min_pred_time " . "FROM aloja_ml.predictions e WHERE e.id_learner = '" . $current_model . "' {$filter_execs} {$where_configs}" . "GROUP BY {$param_eval_query}, bench {$minExecsFilter} order by bench,{$param_eval_query}"; $result = $dbml->query($query); // Initialize array foreach ($paramOptions as $param) { foreach ($benchOptions as $bench) { $arrayBenchs_pred[$bench['bench'] . '_pred'][$param] = null; $arrayBenchs_pred[$bench['bench'] . '_pred'][$param]['y'] = 0; $arrayBenchs_pred[$bench['bench'] . '_pred'][$param]['count'] = 0; } } foreach ($result as $row) { $bench_n = $row['bench'] . '_pred'; $class = $row[$paramEval]; if ($paramEval == 'comp') { $value = Utils::getCompressionName($class); } else { if ($paramEval == 'id_cluster') { $value = Utils::getClusterName($class, $db); } else { if ($paramEval == 'net') { $value = Utils::getNetworkName($class); } else { if ($paramEval == 'disk') { $value = Utils::getDisksName($class); } else { if ($paramEval == 'iofilebuf') { $value = $class / 1024; } else { $value = $class; } } } } } if (!in_array($value, $paramOptions)) { $paramOptions[] = $value; foreach ($benchOptions as $bench) { $arrayBenchs_pred[$bench['bench'] . '_pred'][$value] = null; $arrayBenchs_pred[$bench['bench'] . '_pred'][$value]['y'] = 0; $arrayBenchs_pred[$bench['bench'] . '_pred'][$value]['count'] = 0; $arrayBenchs[$bench['bench']][$value] = null; $arrayBenchs[$bench['bench']][$value]['y'] = 0; $arrayBenchs[$bench['bench']][$value]['count'] = 0; } } $arrayBenchs_pred[$bench_n][$value]['y'] = (int) $row['avg_pred_time']; $arrayBenchs_pred[$bench_n][$value]['count'] = (int) $row['count']; } } } // ---------------------------------------------------- // END - Add predictions to the series // ---------------------------------------------------- asort($paramOptions); foreach ($arrayBenchs as $key => $arrayBench) { $caregories = ''; $data_a = null; $data_p = null; foreach ($paramOptions as $param) { if ($arrayBenchs[$key][$param]['count'] > 0 && empty($arrayBenchs_pred) || !empty($arrayBenchs_pred) && ($arrayBenchs_pred[$key . '_pred'][$param]['count'] > 0 || $arrayBenchs[$key][$param]['count'] > 0)) { $data_a[] = $arrayBenchs[$key][$param]; if (!empty($arrayBenchs_pred)) { $data_p[] = $arrayBenchs_pred[$key . '_pred'][$param]; } $categories = $categories . "'{$param} " . Utils::getParamevalUnit($paramEval) . "',"; // FIXME - Redundant n times performed... don't care now } } $series[] = array('name' => $key, 'data' => $data_a); if (!empty($arrayBenchs_pred)) { $series[] = array('name' => $key . '_pred', 'data' => $data_p); } } $series = json_encode($series); if (!empty($arrayBenchs_pred)) { $colors = "['#7cb5ec','#9cd5fc','#434348','#636368','#90ed7d','#b0fd9d','#f7a35c','#f7c37c','#8085e9','#a0a5f9','#f15c80','#f17ca0','#e4d354','#f4f374','#8085e8','#a0a5f8','#8d4653','#ad6673','#91e8e1','#b1f8f1']"; } else { $colors = "['#7cb5ec','#434348','#90ed7d','#f7a35c','#8085e9','#f15c80','#e4d354','#8085e8','#8d4653','#91e8e1']"; } } catch (\Exception $e) { $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n"); $series = $jsonHeader = $colors = '[]'; $must_wait = 'NO'; } $return_params = array('title' => 'Improvement of Hadoop Execution by SW and HW Configurations', 'categories' => $categories, 'series' => $series, 'paramEval' => $paramEval, 'instance' => $instance, 'models' => '<li>' . implode('</li><li>', $possible_models) . '</li>', 'models_id' => $possible_models_id, 'current_model' => $current_model, 'gammacolors' => $colors, 'model_info' => $model_info, 'slice_info' => $slice_info, 'must_wait' => $must_wait); $this->filters->setCurrentChoices('current_model', array_merge($possible_models_id, array('---Other models---'), $other_models)); return $this->render('mltemplate/mlparameval.html.twig', $return_params); }
public function mlobservedtreesAction() { $model_info = $instance = $slice_info = $message = $config = $tree_descriptor_ordered = $tree_descriptor_gini = ''; $jsonData = $jsonHeader = '[]'; $jsonObstrees = $jsonObstreesHeader = '[]'; $must_wait = 'NO'; try { $dbml = new \PDO($this->container->get('config')['db_conn_chain'], $this->container->get('config')['mysql_user'], $this->container->get('config')['mysql_pwd']); $dbml->setAttribute(\PDO::ATTR_ERRMODE, \PDO::ERRMODE_EXCEPTION); $dbml->setAttribute(\PDO::ATTR_EMULATE_PREPARES, false); $db = $this->container->getDBUtils(); // FIXME - This must be counted BEFORE building filters, as filters inject rubbish in GET when there are no parameters... $instructions = count($_GET) <= 1; $this->buildFilters(array('minexetime' => array('default' => 0), 'valid' => array('default' => 0), 'filter' => array('default' => 0), 'prepares' => array('default' => 1))); if ($instructions) { MLUtils::getIndexObsTrees($jsonObstrees, $jsonObstreesHeader, $dbml); return $this->render('mltemplate/mlobstrees.html.twig', array('obstrees' => $jsonObstrees, 'header_obstrees' => $jsonObstreesHeader, 'jsonData' => '[]', 'jsonHeader' => '[]', 'instructions' => 'YES')); } $param_names = array('bench', 'net', 'disk', 'maps', 'iosf', 'replication', 'iofilebuf', 'comp', 'blk_size', 'id_cluster', 'datanodes', 'vm_OS', 'vm_cores', 'vm_RAM', 'provider', 'vm_size', 'type', 'bench_type', 'hadoop_version', 'datasize', 'scale_factor'); // Order is important $params = $this->filters->getFiltersSelectedChoices($param_names); foreach ($param_names as $p) { if (!is_null($params[$p]) && is_array($params[$p])) { sort($params[$p]); } } $params_additional = array(); $param_names_additional = array('datefrom', 'dateto', 'minexetime', 'maxexetime', 'valid', 'filter'); // Order is important $params_additional = $this->filters->getFiltersSelectedChoices($param_names_additional); $where_configs = $this->filters->getWhereClause(); $where_configs = str_replace("AND .", "AND ", $where_configs); // compose instance $instance = MLUtils::generateSimpleInstance($this->filters, $param_names, $params, TRUE); $model_info = MLUtils::generateModelInfo($this->filters, $param_names, $params, TRUE); $slice_info = MLUtils::generateDatasliceInfo($this->filters, $param_names_additional, $params_additional); $config = $instance . '-' . $slice_info . '-obstree'; $is_cached_mysql = $dbml->query("SELECT count(*) as total FROM aloja_ml.observed_trees WHERE id_obstrees = '" . md5($config) . "'"); $tmp_result = $is_cached_mysql->fetch(); $is_cached = $tmp_result['total'] > 0; $in_process = file_exists(getcwd() . '/cache/ml/' . md5($config) . '.lock'); $finished_process = file_exists(getcwd() . '/cache/ml/' . md5($config) . '.fin'); $tmp_file = getcwd() . '/cache/ml/' . md5($config) . '.tmp'; // get headers for csv $header_names = array('bench' => 'Benchmark', 'net' => 'Net', 'disk' => 'Disk', 'maps' => 'Maps', 'iosf' => 'IO.SFac', 'replication' => 'Rep', 'iofilebuf' => 'IO.FBuf', 'comp' => 'Comp', 'blk_size' => 'Blk.size', 'e.id_cluster' => 'Cluster', 'datanodes' => 'Datanodes', 'vm_OS' => 'VM.OS', 'vm_cores' => 'VM.Cores', 'vm_RAM' => 'VM.RAM', 'provider' => 'Provider', 'vm_size' => 'VM.Size', 'type' => 'Type', 'bench_type' => 'Bench.Type', 'hadoop_version' => 'Hadoop.Version', 'IFNULL(datasize,0)' => 'Datasize', 'scale_factor' => 'Scale.Factor'); $special_header_names = array('id_exec' => 'ID', 'exe_time' => 'Exe.Time'); $headers = array_keys($header_names); $special_headers = array_keys($special_header_names); if (!$in_process && !$finished_process && !$is_cached) { // Dump the DB slice to csv $query = "SELECT " . implode(",", $headers) . ", " . implode(",", $special_headers) . " FROM aloja2.execs e LEFT JOIN aloja2.clusters c ON e.id_cluster = c.id_cluster WHERE hadoop_version IS NOT NULL" . $where_configs . ";"; $rows = $db->get_rows($query); if (empty($rows)) { throw new \Exception('No data matches with your critteria.'); } if (($key = array_search('e.id_cluster', $headers)) !== false) { $headers[$key] = 'id_cluster'; } $fp = fopen($tmp_file, 'w'); foreach ($rows as $row) { $row['id_cluster'] = "Cl" . $row['id_cluster']; // Cluster is numerically codified... $row['comp'] = "Cmp" . $row['comp']; // Compression is numerically codified... $line = ''; foreach ($headers as $hn) { $line = $line . ($line != '' ? ',' : '') . $row[$hn]; } $line = $row['id_exec'] . ' ' . $line . ' ' . $row['exe_time'] . "\n"; fputs($fp, $line); } fclose($fp); if (($key = array_search('id_cluster', $headers)) !== false) { $headers[$key] = 'e.id_cluster'; } // Execute R Engine $exe_query = 'cd ' . getcwd() . '/cache/ml;'; $exe_query = $exe_query . ' touch ' . md5($config) . '.lock;'; $exe_query = $exe_query . ' ../../resources/aloja_cli.r -m aloja_representative_tree -p method=ordered:dump_file=' . $tmp_file . ':output=nodejson -v >' . md5($config) . '-split.dat 2>/dev/null;'; $exe_query = $exe_query . ' ../../resources/aloja_cli.r -m aloja_representative_tree -p method=gini:dump_file=' . $tmp_file . ':output=nodejson -v >' . md5($config) . '-gini.dat 2>/dev/null;'; $exe_query = $exe_query . ' rm -f ' . md5($config) . '.lock; rm -f ' . $tmp_file . '; touch ' . md5($config) . '.fin'; exec(getcwd() . '/resources/queue -d -c "' . $exe_query . '" >/dev/null 2>&1 &'); } if (!$is_cached) { $finished_process = file_exists(getcwd() . '/cache/ml/' . md5($config) . '.fin'); if ($finished_process) { // Read results and dump to DB $tree_descriptor_ordered = ''; try { $file = fopen(getcwd() . '/cache/ml/' . md5($config) . '-split.dat', "r"); $tree_descriptor_ordered = fgets($file); $tree_descriptor_ordered = substr($tree_descriptor_ordered, 5, -2); $tree_descriptor_ordered = str_replace("\\\"", "\"", $tree_descriptor_ordered); $tree_descriptor_ordered = str_replace("desc:\"\"", "desc:\"---\"", $tree_descriptor_ordered); fclose($file); } catch (\Exception $e) { throw new \Exception("Error on retrieving result file. Check that R is working properly."); } $tree_descriptor_gini = ''; /* try { $file = fopen(getcwd().'/cache/ml/'.md5($config).'-gini.dat', "r"); $tree_descriptor_gini = fgets($file); $tree_descriptor_gini = substr($tree_descriptor_gini, 5, -2); $tree_descriptor_gini = str_replace("\\\"","\"",$tree_descriptor_gini); fclose($file); } catch (\Exception $e) { throw new \Exception ("Error on retrieving result file. Check that R is working properly."); } */ $query = "INSERT INTO aloja_ml.observed_trees (id_obstrees,instance,model,dataslice,tree_code_split,tree_code_gain) VALUES ('" . md5($config) . "','" . $instance . "','" . $model_info . "','" . $slice_info . "','" . $tree_descriptor_ordered . "','" . $tree_descriptor_gini . "')"; if ($dbml->query($query) === FALSE) { throw new \Exception('Error when saving tree into DB'); } // Remove temporal files $output = shell_exec('rm -f ' . getcwd() . '/cache/ml/' . md5($config) . '-*.dat'); $output = shell_exec('rm -f ' . getcwd() . '/cache/ml/' . md5($config) . '.fin'); } else { $must_wait = 'YES'; throw new \Exception('WAIT'); } } // Fetch results and compose JSON $header = array('Benchmark', 'Net', 'Disk', 'Maps', 'IO.SFS', 'Rep', 'IO.FBuf', 'Comp', 'Blk.Size', 'Cluster', 'Datanodes', 'VM.OS', 'VM.Cores', 'VM.RAM', 'Provider', 'VM.Size', 'Type', 'Bench.Type', 'Version', 'Datasize', 'Scale.Factor', 'Observed'); $jsonHeader = '[{title:""}'; foreach ($header as $title) { $jsonHeader = $jsonHeader . ',{title:"' . $title . '"}'; } $jsonHeader = $jsonHeader . ']'; // Fetch observed values $query = "SELECT " . implode(",", $headers) . ", " . implode(",", $special_headers) . " FROM aloja2.execs e LEFT JOIN aloja2.clusters c ON e.id_cluster = c.id_cluster WHERE hadoop_version IS NOT NULL" . $where_configs . ";"; $rows = $db->get_rows($query); if (empty($rows)) { throw new \Exception('No data matches with your critteria.'); } if (($key = array_search('e.id_cluster', $headers)) !== false) { $headers[$key] = 'id_cluster'; } $jsonData = '['; foreach ($rows as $row) { $row['id_cluster'] = "Cl" . $row['id_cluster']; // Cluster is numerically codified... $row['comp'] = "Cmp" . $row['comp']; // Compression is numerically codified... $line = ''; foreach ($headers as $hn) { $line = $line . ($line != '' ? ',' : '') . $row[$hn]; } $line = $row['id_exec'] . ',' . $line . ',' . $row['exe_time']; if ($jsonData != '[') { $jsonData = $jsonData . ','; } $jsonData = $jsonData . "['" . str_replace(",", "','", $line) . "']"; } $jsonData = $jsonData . ']'; foreach (range(1, 32) as $value) { $jsonData = str_replace('Cmp' . $value, Utils::getCompressionName($value), $jsonData); } if ($tree_descriptor_ordered == '') { // Display Descriptive Tree, if not processed yet $query = "SELECT tree_code_split, tree_code_gain FROM aloja_ml.observed_trees WHERE id_obstrees = '" . md5($config) . "'"; $result = $dbml->query($query); $row = $result->fetch(); $tree_descriptor_ordered = $row['tree_code_split']; $tree_descriptor_gini = $row['tree_code_gain']; } } catch (\Exception $e) { if ($e->getMessage() != "WAIT") { $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n"); } $jsonData = $jsonHeader = '[]'; } $dbml = null; $return_params = array('jsonData' => $jsonData, 'jsonHeader' => $jsonHeader, 'obstrees' => $jsonObstrees, 'header_obstrees' => $jsonObstreesHeader, 'message' => $message, 'must_wait' => $must_wait, 'instance' => $instance, 'model_info' => $model_info, 'slice_info' => $slice_info, 'id_obstrees' => md5($config), 'tree_descriptor_ordered' => $tree_descriptor_ordered, 'tree_descriptor_gini' => $tree_descriptor_gini); return $this->render('mltemplate/mlobstrees.html.twig', $return_params); }
public function paramEvaluationAction() { $db = $this->container->getDBUtils(); $this->buildFilters(array('minexecs' => array('default' => null, 'type' => 'inputNumber', 'label' => 'Minimum executions:', 'parseFunction' => function () { return 0; }, 'filterGroup' => 'basic'))); $whereClause = $this->filters->getWhereClause(); $model_html = ''; $model_info = $db->get_rows("SELECT id_learner, model, algorithm, dataslice FROM aloja_ml.learners"); foreach ($model_info as $row) { $model_html = $model_html . "<li><b>" . $row['id_learner'] . "</b> => " . $row['algorithm'] . " : " . $row['model'] . " : " . $row['dataslice'] . "</li>"; } $categories = ''; $series = ''; try { $paramEval = isset($_GET['parameval']) && Utils::get_GET_string('parameval') != '' ? Utils::get_GET_string('parameval') : 'maps'; $minExecs = isset($_GET['minexecs']) ? Utils::get_GET_int('minexecs') : -1; $this->filters->changeCurrentChoice('minexecs', $minExecs == -1 ? null : $minExecs); $shortAliasParamEval = array('maps' => 'e', 'comp' => 'e', 'id_cluster' => 'c', 'net' => 'e', 'disk' => 'e', 'replication' => 'e', 'iofilebuf' => 'e', 'blk_size' => 'e', 'iosf' => 'e', 'vm_size' => 'c', 'vm_cores' => 'c', 'vm_ram' => 'c', 'datanodes' => 'c', 'hadoop_version' => 'e', 'type' => 'c'); $minExecsFilter = ""; if ($minExecs > 0) { $minExecsFilter = "HAVING COUNT(*) > {$minExecs}"; } $filter_execs = DBUtils::getFilterExecs(); $options = $this->filters->getFiltersArray()[$paramEval]['choices']; $benchOptions = "SELECT DISTINCT e.bench FROM aloja2.execs e JOIN aloja2.clusters c USING (id_cluster) LEFT JOIN aloja_ml.predictions p USING (id_exec) WHERE 1 {$filter_execs} {$whereClause} GROUP BY {$shortAliasParamEval[$paramEval]}.{$paramEval}, e.bench order by {$shortAliasParamEval[$paramEval]}.{$paramEval}"; $params = $this->filters->getFiltersSelectedChoices(array('prediction_model', 'upred', 'uobsr')); $whereClauseML = str_replace("exe_time", "pred_time", $whereClause); $whereClauseML = str_replace("start_time", "creation_time", $whereClauseML); $query = "SELECT COUNT(*) AS count, {$shortAliasParamEval[$paramEval]}.{$paramEval}, e.bench, avg(e.exe_time) avg_exe_time, min(e.exe_time) min_exe_time\n\t\t\t\t\t FROM aloja2.execs AS e JOIN aloja2.clusters AS c USING (id_cluster)\n\t\t\t\t\t LEFT JOIN aloja_ml.predictions AS p USING (id_exec)\n\t\t\t\t\t WHERE 1 {$filter_execs} {$whereClause}\n\t\t\t\t\t GROUP BY {$shortAliasParamEval[$paramEval]}.{$paramEval}, e.bench {$minExecsFilter} ORDER BY e.bench, {$shortAliasParamEval[$paramEval]}.{$paramEval}"; $queryPredictions = "\n\t\t\t\t\tSELECT COUNT(*) AS count, {$shortAliasParamEval[$paramEval]}.{$paramEval}, CONCAT('pred_',e.bench) as bench,\n\t\t\t\t\t\tavg(e.pred_time) as avg_exe_time, min(e.pred_time) as min_exe_time\n\t\t\t\t\t\tFROM aloja_ml.predictions AS e\n\t\t\t\t\t\tJOIN clusters c USING (id_cluster)\n\t\t\t\t\t\tWHERE 1 {$filter_execs} " . str_replace("p.", "e.", $whereClauseML) . " AND e.id_learner = '" . $params['prediction_model'] . "'\n\t\t\t\t\t\tGROUP BY {$shortAliasParamEval[$paramEval]}.{$paramEval}, e.bench {$minExecsFilter} ORDER BY e.bench, {$shortAliasParamEval[$paramEval]}.{$paramEval}"; // get the result rows if ($params['uobsr'] == 1 && $params['upred'] == 1) { $query = "({$query}) UNION ({$queryPredictions})"; $benchOptions = "SELECT DISTINCT e.bench FROM aloja2.execs e JOIN aloja2.clusters c USING (id_cluster) LEFT JOIN aloja_ml.predictions p USING (id_exec) WHERE 1 {$filter_execs} {$whereClause} GROUP BY {$shortAliasParamEval[$paramEval]}.{$paramEval}, e.bench\n\t\t\t\t\t\t\t\t UNION\n\t\t\t\t\t\t\t\t (SELECT DISTINCT CONCAT('pred_', e.bench) as bench FROM aloja_ml.predictions AS e\n\t\t\t\t\t\t\t\t JOIN clusters c USING (id_cluster)\n\t\t\t\t\t\t\t\t WHERE 1 {$filter_execs} " . str_replace("p.", "e.", $whereClauseML) . " AND e.id_learner = '" . $params['prediction_model'] . "'\n\t\t\t\t\t\t\t\t GROUP BY {$shortAliasParamEval[$paramEval]}.{$paramEval}, e.bench {$minExecsFilter})\n\t\t\t\t\t\t\t\t ORDER BY bench"; $optionsPredictions = "SELECT DISTINCT {$shortAliasParamEval[$paramEval]}.{$paramEval} FROM aloja_ml.predictions AS e JOIN clusters c USING (id_cluster) WHERE 1 {$filter_execs} " . str_replace("p.", "e.", $whereClauseML) . " AND e.id_learner = '" . $params['prediction_model'] . "' ORDER BY {$shortAliasParamEval[$paramEval]}.{$paramEval}"; $optionsPredictions = $db->get_rows($optionsPredictions); foreach ($optionsPredictions as $predOption) { $options[] = $predOption[$paramEval]; } } else { if ($params['uobsr'] == 0 && $params['upred'] == 1) { $query = $queryPredictions; $benchOptions = "SELECT DISTINCT CONCAT('pred_', e.bench) as bench FROM aloja_ml.predictions AS e\n \t\t\t\t\t\t\t\t JOIN clusters c USING (id_cluster)\n\t\t\t\t\t\t\t\t WHERE 1 {$filter_execs} " . str_replace("p.", "e.", $whereClauseML) . " AND e.id_learner = '" . $params['prediction_model'] . "'\n\t\t\t\t\t\t\t\t GROUP BY {$shortAliasParamEval[$paramEval]}.{$paramEval}, e.bench {$minExecsFilter} ORDER BY e.bench, {$shortAliasParamEval[$paramEval]}.{$paramEval}"; $options = array(); $optionsPredictions = "SELECT DISTINCT {$shortAliasParamEval[$paramEval]}.{$paramEval} FROM aloja_ml.predictions AS e JOIN clusters c USING (id_cluster) WHERE 1 {$filter_execs} " . str_replace("p.", "e.", $whereClauseML) . " AND e.id_learner = '" . $params['prediction_model'] . "' ORDER BY {$shortAliasParamEval[$paramEval]}.{$paramEval}"; $optionsPredictions = $db->get_rows($optionsPredictions); foreach ($optionsPredictions as $predOption) { $options[] = $predOption[$paramEval]; } } else { if ($params['uobsr'] == 0 && $params['upred'] == 0) { $this->container->getTwig()->addGlobal('message', "Warning: No data selected (Predictions|Observations) from the ML Filters. Adding the Observed executions to the figure by default.\n"); } } } $rows = $db->get_rows($query); $benchOptions = $db->get_rows($benchOptions); if (!$rows) { throw new \Exception("No results for query!"); } $paramOptions = array(); foreach ($options as $option) { if ($paramEval == 'id_cluster') { $paramOptions[] = Utils::getClusterName($option, $db); } else { if ($paramEval == 'comp') { $paramOptions[] = Utils::getCompressionName($option); } else { if ($paramEval == 'net') { $paramOptions[] = Utils::getNetworkName($option); } else { if ($paramEval == 'disk') { $paramOptions[] = Utils::getDisksName($option); } else { if ($paramEval == 'vm_ram') { $paramOptions[] = Utils::getBeautyRam($option); } else { $paramOptions[] = $option; } } } } } } $categories = ''; $arrayBenchs = array(); foreach ($paramOptions as $param) { $categories .= "'{$param}" . Utils::getParamevalUnit($paramEval) . "',"; foreach ($benchOptions as $bench) { $arrayBenchs[$bench['bench']][$param] = null; } } $series = array(); foreach ($rows as $row) { if ($paramEval == 'comp') { $row[$paramEval] = Utils::getCompressionName($row['comp']); } else { if ($paramEval == 'id_cluster') { $row[$paramEval] = Utils::getClusterName($row[$paramEval], $db); } else { if ($paramEval == 'net') { $row[$paramEval] = Utils::getNetworkName($row['net']); } else { if ($paramEval == 'disk') { $row[$paramEval] = Utils::getDisksName($row['disk']); } else { if ($paramEval == 'vm_ram') { $row[$paramEval] = Utils::getBeautyRam($row['vm_ram']); } } } } } $arrayBenchs[strtolower($row['bench'])][$row[$paramEval]]['y'] = round((int) $row['avg_exe_time'], 2); $arrayBenchs[strtolower($row['bench'])][$row[$paramEval]]['count'] = (int) $row['count']; } foreach ($arrayBenchs as $key => $arrayBench) { $series[] = array('name' => $key, 'data' => array_values($arrayBench)); } $series = json_encode($series); } catch (\Exception $e) { $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n"); } return $this->render('configEvaluationViews/parameval.html.twig', array('title' => 'Improvement of Hadoop Execution by SW and HW Configurations', 'minexecs' => $minExecs, 'categories' => $categories, 'series' => $series, 'paramEval' => $paramEval, 'models' => $model_html)); }
public function paramEvaluationAction() { $db = $this->container->getDBUtils(); $preset = null; if (sizeof($_GET) <= 1) { $preset = Utils::setDefaultPreset($db, 'Parameter Evaluation'); } $selPreset = isset($_GET['presets']) ? $_GET['presets'] : "none"; $rows = ''; $categories = ''; $series = ''; try { $where_configs = ''; $concat_config = ""; if (!isset($_GET['benchs'])) { $_GET['benchs'] = array('wordcount', 'terasort', 'sort'); } $datefrom = Utils::read_params('datefrom', $where_configs); $dateto = Utils::read_params('dateto', $where_configs); $benchs = Utils::read_params('benchs', $where_configs); $nets = Utils::read_params('nets', $where_configs); $disks = Utils::read_params('disks', $where_configs); $blk_sizes = Utils::read_params('blk_sizes', $where_configs); $comps = Utils::read_params('comps', $where_configs); $id_clusters = Utils::read_params('id_clusters', $where_configs); $mapss = Utils::read_params('mapss', $where_configs); $replications = Utils::read_params('replications', $where_configs); $iosfs = Utils::read_params('iosfs', $where_configs); $iofilebufs = Utils::read_params('iofilebufs', $where_configs); $money = Utils::read_params('money', $where_configs); $datanodes = Utils::read_params('datanodess', $where_configs, false); $benchtype = Utils::read_params('bench_types', $where_configs); $vm_sizes = Utils::read_params('vm_sizes', $where_configs, false); $vm_coress = Utils::read_params('vm_coress', $where_configs, false); $vm_RAMs = Utils::read_params('vm_RAMs', $where_configs, false); $hadoop_versions = Utils::read_params('hadoop_versions', $where_configs, false); $types = Utils::read_params('types', $where_configs, false); $valid = Utils::read_params('valids', $where_configs); $filters = Utils::read_params('filters', $where_configs, false); $allunchecked = isset($_GET['allunchecked']) ? $_GET['allunchecked'] : ''; $minexetime = Utils::read_params('minexetime', $where_configs, false); $maxexetime = Utils::read_params('maxexetime', $where_configs, false); $provider = Utils::read_params('providers', $where_configs, false); $vm_OS = Utils::read_params('vm_OSs', $where_configs, false); $paramEval = isset($_GET['parameval']) && $_GET['parameval'] != '' ? $_GET['parameval'] : 'maps'; $minExecs = isset($_GET['minexecs']) ? $_GET['minexecs'] : -1; $minExecsFilter = ""; if ($minExecs > 0) { $minExecsFilter = "HAVING COUNT(*) > {$minExecs}"; } $filter_execs = DBUtils::getFilterExecs(); $options = Utils::getFilterOptions($db); $paramOptions = array(); foreach ($options[$paramEval] as $option) { if ($paramEval == 'id_cluster') { $paramOptions[] = $option['name']; } else { if ($paramEval == 'comp') { $paramOptions[] = Utils::getCompressionName($option[$paramEval]); } else { if ($paramEval == 'net') { $paramOptions[] = Utils::getNetworkName($option[$paramEval]); } else { if ($paramEval == 'disk') { $paramOptions[] = Utils::getDisksName($option[$paramEval]); } else { if ($paramEval == 'vm_ram') { $paramOptions[] = Utils::getBeautyRam($option['vm_RAM']); } else { $paramOptions[] = $option[$paramEval]; } } } } } } // if($paramEval == 'maps') // $paramOptions = array(4,6,8,10,12,16,24,32); // else if($paramEval == 'comp') // $paramOptions = array('None','ZLIB','BZIP2','Snappy'); // else if($paramEval == 'id_cluster') // $paramOptions = array('rl-06'); // else if($paramEval == 'net') // $paramOptions = array('Ethernet','Infiniband'); // else if($paramEval == 'disk') // $paramOptions = array('Hard-disk drive','1 HDFS remote(s)/tmp local','2 HDFS remote(s)/tmp local','3 HDFS remote(s)/tmp local','1 HDFS remote(s)', '2 HDFS remote(s)', '3 HDFS remote(s)', 'SSD'); // else if($paramEval == 'replication') // $paramOptions = array(1,2,3); // else if($paramEval == 'iofilebuf') // $paramOptions = array(1,4,16,32,64,128,256); // else if($paramEval == 'blk_size') // $paramOptions = array(32,64,128,256); // else if($paramEval == 'iosf') // $paramOptions = array(5,10,20,50); $benchOptions = $db->get_rows("SELECT DISTINCT bench FROM execs e JOIN clusters c USING (id_cluster) WHERE 1 {$filter_execs} {$where_configs} GROUP BY {$paramEval}, bench order by {$paramEval}"); // get the result rows $query = "SELECT count(*) as count, {$paramEval}, e.id_exec, exec as conf, bench, " . "exe_time, avg(exe_time) avg_exe_time, min(exe_time) min_exe_time " . "from execs e JOIN clusters c USING (id_cluster) WHERE 1 {$filter_execs} {$where_configs}" . "GROUP BY {$paramEval}, bench {$minExecsFilter} order by bench,{$paramEval}"; $rows = $db->get_rows($query); if (!$rows) { throw new \Exception("No results for query!"); } $categories = ''; $arrayBenchs = array(); foreach ($paramOptions as $param) { $categories .= "'{$param}" . Utils::getParamevalUnit($paramEval) . "',"; foreach ($benchOptions as $bench) { $arrayBenchs[$bench['bench']][$param] = null; } } $series = array(); $bench = ''; foreach ($rows as $row) { if ($paramEval == 'comp') { $row[$paramEval] = Utils::getCompressionName($row['comp']); } else { if ($paramEval == 'id_cluster') { $row[$paramEval] = Utils::getClusterName($row[$paramEval], $db); } else { if ($paramEval == 'net') { $row[$paramEval] = Utils::getNetworkName($row['net']); } else { if ($paramEval == 'disk') { $row[$paramEval] = Utils::getDisksName($row['disk']); } else { if ($paramEval == 'vm_ram') { $row[$paramEval] = Utils::getBeautyRam($row['vm_ram']); } } } } } $arrayBenchs[strtolower($row['bench'])][$row[$paramEval]]['y'] = round((int) $row['avg_exe_time'], 2); $arrayBenchs[strtolower($row['bench'])][$row[$paramEval]]['count'] = (int) $row['count']; } foreach ($arrayBenchs as $key => $arrayBench) { $series[] = array('name' => $key, 'data' => array_values($arrayBench)); } $series = json_encode($series); } catch (\Exception $e) { $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n"); } echo $this->container->getTwig()->render('parameval/parameval.html.twig', array('selected' => 'Parameter Evaluation', 'title' => 'Improvement of Hadoop Execution by SW and HW Configurations', 'categories' => $categories, 'series' => $series, 'datefrom' => $datefrom, 'dateto' => $dateto, 'benchs' => $benchs, 'nets' => $nets, 'disks' => $disks, 'blk_sizes' => $blk_sizes, 'comps' => $comps, 'id_clusters' => $id_clusters, 'mapss' => $mapss, 'replications' => $replications, 'iosfs' => $iosfs, 'iofilebufs' => $iofilebufs, 'money' => $money, 'datanodess' => $datanodes, 'bench_types' => $benchtype, 'vm_sizes' => $vm_sizes, 'vm_coress' => $vm_coress, 'vm_RAMs' => $vm_RAMs, 'vm_OS' => $vm_OS, 'hadoop_versions' => $hadoop_versions, 'types' => $types, 'providers' => $provider, 'filters' => $filters, 'allunchecked' => $allunchecked, 'minexetime' => $minexetime, 'maxexetime' => $maxexetime, 'preset' => $preset, 'selPreset' => $selPreset, 'paramEval' => $paramEval, 'options' => $options)); }