public function clustersAction() { $clusterNameSelected = null; if (isset($_GET['cluster_name'])) { $clusterNameSelected = Utils::get_GET_string('cluster_name'); } $filter_execs = DBUtils::getFilterExecs(); $db = $this->container->getDBUtils(); $clusters = $db->get_rows("SELECT * FROM aloja2.clusters c WHERE id_cluster IN (SELECT distinct(id_cluster) FROM aloja2.execs e WHERE 1 {$filter_execs});"); echo $this->container->getTwig()->render('defaultViews/clusters.html.twig', array('selected' => 'Clusters', 'clusters' => $clusters, 'clusterNameSelected' => $clusterNameSelected, 'title' => 'ALOJA Clusters')); }
public function BestCostPerfClusterEvaluationAction() { $filter_execs = DBUtils::getFilterExecs(); $dbUtils = $this->container->getDBUtils(); try { /* * 1. Get execs and cluster associated costs * 2. For each exec calculate cost, exe_time/3600 * (cost_cluster + clust_remote|ssd|ib|eth) * 3. Calculate max and minimum costs * 4. calculate max and minimum exe times * 5. Normalize costs and exe times * 6. Print results */ $minCost = -1; $maxCost = 0; $minExeTime = -1; $maxExeTime = 0; $innerQueryWhere = str_replace("e.", "e2.", $this->whereClause); $innerQueryWhere = str_replace("c.", "c2.", $innerQueryWhere); $innerQueryWhere = str_replace("p.", "p2.", $innerQueryWhere); $execs = "SELECT t.scount as count, e.exe_time,e.net,e.disk,e.bench,e.bench_type,e.maps,e.iosf,e.replication,e.iofilebuf,e.comp,e.blk_size,e.hadoop_version,e.exec, c.name as clustername,c.* \n \t\t FROM aloja2.execs e JOIN aloja2.clusters c USING (id_cluster)\n \t\t LEFT JOIN aloja_ml.predictions p USING (id_exec)\n \t\t INNER JOIN (SELECT count(*) as scount, MIN(e2.exe_time) minexe FROM aloja2.execs e2 JOIN aloja2.clusters c2 USING(id_cluster)\n \t\t\t\t\t LEFT JOIN aloja_ml.predictions p2 USING (id_exec) WHERE 1 {$filter_execs} {$innerQueryWhere} GROUP BY c2.name,e2.net,e2.disk ORDER BY c2.name ASC)\n \t\tt ON e.exe_time = t.minexe WHERE 1 {$filter_execs} {$this->whereClause}\n \t\t GROUP BY c.name,e.net,e.disk ORDER BY c.name ASC;"; $execs = $dbUtils->get_rows($execs); if (!$execs) { throw new \Exception("No results for query!"); } $minCostKey = 0; $tmpMinCost = -1; $previousCluster = "none"; $bestExecs = array(); $sumCount = 0; foreach ($execs as $key => &$exec) { if ($previousCluster != "none" && $previousCluster != $exec['name']) { $previousCluster = $exec['name']; $tmpMinCost = -1; if ($execs[$minCostKey]['cost_std'] > $maxCost) { $maxCost = $execs[$minCostKey]['cost_std']; } if ($execs[$minCostKey]['cost_std'] < $minCost || $minCost == -1) { $minCost = $execs[$minCostKey]['cost_std']; } if ($execs[$minCostKey]['exe_time'] < $minExeTime || $minExeTime == -1) { $minExeTime = $execs[$minCostKey]['exe_time']; } if ($execs[$minCostKey]['exe_time'] > $maxExeTime) { $maxExeTime = $execs[$minCostKey]['exe_time']; } $execs[$minCostKey]['countexecs'] = $sumCount; array_push($bestExecs, $execs[$minCostKey]); $sumCount = 0; } else { if ($previousCluster == "none") { $previousCluster = $exec['name']; } } $exec['cost_std'] = Utils::getExecutionCost($exec, $this->clusterCosts); if ($tmpMinCost == -1 || $exec['cost_std'] < $tmpMinCost) { $tmpMinCost = $exec['cost_std']; $minCostKey = $key; } $sumCount += $exec['count']; } if ($execs[$minCostKey]['cost_std'] > $maxCost) { $maxCost = $execs[$minCostKey]['cost_std']; } if ($execs[$minCostKey]['cost_std'] < $minCost || $minCost == -1) { $minCost = $execs[$minCostKey]['cost_std']; } if ($execs[$minCostKey]['exe_time'] < $minExeTime || $minExeTime == -1) { $minExeTime = $execs[$minCostKey]['exe_time']; } if ($execs[$minCostKey]['exe_time'] > $maxExeTime) { $maxExeTime = $execs[$minCostKey]['exe_time']; } $execs[$minCostKey]['countexecs'] = $sumCount; array_push($bestExecs, $execs[$minCostKey]); } catch (\Exception $e) { $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n"); } // (exe_time - $min_exe_time)/($max_exe_time - $min_exe_time) exe_time_std, // ($cost_per_run - $min_cost_per_run)/($max_cost_per_run - $min_cost_per_run) cost_std, $seriesData = ''; foreach ($bestExecs as $exec) { $exeTimeStd = 0.01; $costTimeStd = 0.01; if (count($bestExecs) > 1) { $exeTimeStd = ($exec['exe_time'] - $minExeTime) / ($maxExeTime - $minExeTime); $costTimeStd = ($exec['cost_std'] - $minCost) / ($maxCost - $minCost); if ($costTimeStd <= 0.01) { $costTimeStd = 0.01; } if ($exeTimeStd <= 0.01) { $exeTimeStd = 0.01; } } $clusterDesc = "{$exec['datanodes']} datanodes, " . round($exec['vm_RAM'], 0) . " GB memory, {$exec['vm_OS']}, {$exec['provider']} {$exec['type']}"; $seriesData .= "{\n name: '" . $exec['name'] . "',\n data: [[" . round($exeTimeStd, 3) . ", " . round($costTimeStd, 3) . ", " . $exec['countexecs'] . "]],\n clusterdesc: '{$clusterDesc}', countExecs: '{$exec['countexecs']}'\n },"; } $clusters = $dbUtils->get_rows("SELECT * FROM aloja2.clusters c WHERE id_cluster IN (SELECT DISTINCT(id_cluster) FROM aloja2.execs e WHERE 1 {$filter_execs});"); //Sorting clusters by size usort($bestExecs, function ($a, $b) { return $a['cost_std'] > $b['cost_std']; }); return $this->render('costPerfEvaluationViews/best_perf_by_cost_cluster.html.twig', array('highcharts_js' => HighCharts::getHeader(), 'clusterCosts' => $this->clusterCosts, 'seriesData' => $seriesData, 'bestExecs' => $bestExecs, 'clusters' => $clusters)); }
public static function multi_implode($array, $glue) { $ret = ''; if (!is_array($array)) { return $ret; } foreach ($array as $item) { if (is_array($item)) { $ret .= Utils::multi_implode($item, $glue) . $glue; } else { $ret .= $item . $glue; } } $ret = substr($ret, 0, 0 - strlen($glue)); return $ret; }
public function __construct() { $container = new PimpleContainer(); if (Utils::in_dev()) { ini_set('display_errors', 'On'); error_reporting(E_ALL); //ini_set('memory_limit', '512M'); $config_file = 'config/config.sample.yml'; if (!file_get_contents($config_file)) { throw new \Exception("Cannot read configuration file: {$config_file}, check that it exists!"); } $container['config'] = Yaml::parse(file_get_contents($config_file)); $container['env'] = 'dev'; } else { //ini_set('display_errors', 'On'); //error_reporting(E_ALL); //ini_set('memory_limit', '1024M'); $config_file = 'config/config.yml'; if (!file_get_contents($config_file)) { throw new Exception("Cannot read configuration file: {$config_file}, check that it exists!"); } $container['config'] = Yaml::parse(file_get_contents($config_file)); $container['env'] = 'prod'; } if (!$container['config']['show_warnings']) { error_reporting(E_ERROR | E_PARSE); } $container['log'] = function ($c) { $logLevel = $c['env'] == 'dev' ? Logger::DEBUG : Logger::WARNING; // create a log channel $log = new Logger('aloja'); $log->pushHandler(new StreamHandler("logs/aloja_{$c['env']}.log", $logLevel)); return $log; }; $container['db'] = function ($c) { $db = new DBUtils($c); return $db; }; $container['request'] = Request::createFromGlobals(); $container['router'] = function ($c) { $router = new Router($c['log'], $c['request']); return $router; }; $container['twig'] = function ($c) { $loader = new Twig_Loader_Filesystem($c['config']['twig_views_path']); $twigOptions = array('debug' => $c['config']['enable_debug']); if ($c['config']['in_cache']) { $twigOptions['cache'] = $c['config']['twig_cache_path']; } $twig = new Twig_Environment($loader, $twigOptions); $twig->addExtension(new AlojaTwigExtension($c['router'])); if ($c['config']['enable_debug']) { $twig->addExtension(new Twig_Extension_Debug()); } //Twig globals initialization $twig->addGlobal('request', $c['request']); $twig->addGlobal('PROD', $c['env'] === 'prod'); $twig->addGlobal('DEV', $c['env'] === 'dev'); // $twig->addGlobal('message',null); return $twig; }; $this->container = $container; }
public function nodesEvaluationAction() { $dbUtils = $this->container->getDBUtils(); $preset = null; if (sizeof($_GET) <= 1) { $preset = Utils::setDefaultPreset($dbUtils, 'Number of Nodes Evaluation'); } $selPreset = isset($_GET['presets']) ? $_GET['presets'] : "none"; try { $filter_execs = DBUtils::getFilterExecs(); $where_configs = ''; $datefrom = Utils::read_params('datefrom', $where_configs); $dateto = Utils::read_params('dateto', $where_configs); $benchs = Utils::read_params('benchs', $where_configs, true); $nets = Utils::read_params('nets', $where_configs, true); $disks = Utils::read_params('disks', $where_configs, true); $blk_sizes = Utils::read_params('blk_sizes', $where_configs, true); $comps = Utils::read_params('comps', $where_configs, true); $id_clusters = Utils::read_params('id_clusters', $where_configs, true); $mapss = Utils::read_params('mapss', $where_configs, true); $replications = Utils::read_params('replications', $where_configs, true); $iosfs = Utils::read_params('iosfs', $where_configs, true); $iofilebufs = Utils::read_params('iofilebufs', $where_configs, true); $money = Utils::read_params('money', $where_configs, true); $datanodes = Utils::read_params('datanodess', $where_configs, true); $benchtype = Utils::read_params('bench_types', $where_configs, true); $vm_sizes = Utils::read_params('vm_sizes', $where_configs, true); $vm_coress = Utils::read_params('vm_coress', $where_configs, true); $vm_RAMs = Utils::read_params('vm_RAMs', $where_configs, true); $hadoop_versions = Utils::read_params('hadoop_versions', $where_configs, true); $types = Utils::read_params('types', $where_configs, true); $filters = Utils::read_params('filters', $where_configs, true); $allunchecked = isset($_GET['allunchecked']) ? $_GET['allunchecked'] : ''; $minexetime = Utils::read_params('minexetime', $where_configs, true); $maxexetime = Utils::read_params('maxexetime', $where_configs, true); $provider = Utils::read_params('providers', $where_configs, false); $vm_OS = Utils::read_params('vm_OSs', $where_configs, false); if (!$benchs) { $where_configs .= 'AND bench IN (\'terasort\')'; } /* $selectedGroups = array(); if(isset($_GET['selected-groups']) && $_GET['selected-groups'] != "") { $selectedGroups = explode(",",$_GET['selected-groups']); } else { $selectedGroups[] = 'exec_type'; $selectedGroups[] = 'vm_OS'; } $selectedString = implode(',',Utils::getStandardGroupBy($selectedGroups)); $query = "SELECT ".str_replace("execTable","e",str_replace("clusterTable","c",$selectedString)).",c.vm_size,(e.exe_time * (c.cost_hour / 3600)) as cost, e.*, c.*". " FROM execs e JOIN clusters c USING (id_cluster) INNER JOIN ( SELECT ".str_replace("execTable","e2",str_replace("clusterTable","c2",$selectedString)).",c2.vm_size as vmsize,MIN(e2.exe_time) as minexe from execs e2 JOIN clusters c2 USING (id_cluster) WHERE 1 $where_configs GROUP BY ".str_replace("execTable","e2",str_replace("clusterTable","c2",$selectedString)).",c2.vm_size ) t ON"; $it = 0; foreach(explode(',',$selectedString) as $group) { if ($it != 0) $query .= " AND"; $tableName = str_replace("execTable", "e", $group); $tableName = str_replace("clusterTable", "c", $tableName); $withoutPrefixes = str_replace(array("execTable.", "clusterTable."), '', $group); $query .= " t.$withoutPrefixes = $tableName"; $it++; } $query .= " AND t.vmsize = c.vm_size WHERE 1 GROUP BY ".str_replace("execTable","e",str_replace("clusterTable","c",$selectedString)).",c.vm_size ORDER BY ".str_replace("execTable","e",str_replace("clusterTable","c",$selectedString)).",c.vm_size DESC;"; $execs = $dbUtils->get_rows($query); */ $execs = $dbUtils->get_rows("SELECT c.datanodes,e.exec_type,c.vm_OS,c.vm_size,(e.exe_time * (c.cost_hour/3600)) as cost,e.*,c.* FROM execs e JOIN clusters c USING (id_cluster) INNER JOIN ( SELECT c2.datanodes,e2.exec_type,c2.vm_OS,c2.vm_size as vmsize,MIN(e2.exe_time) as minexe from execs e2 JOIN clusters c2 USING (id_cluster) WHERE 1 {$where_configs} GROUP BY c2.datanodes,e2.exec_type,c2.vm_OS,c2.vm_size ) t ON t.minexe = e.exe_time AND t.datanodes = c.datanodes AND t.vmsize = c.vm_size WHERE 1 {$filter_execs} GROUP BY c.datanodes,e.exec_type,c.vm_OS,c.vm_size ORDER BY c.datanodes ASC,c.vm_OS,c.vm_size DESC;"); $vmSizes = array(); $categories = array(); $dataNodes = array(); $vmOS = array(); $execTypes = array(); foreach ($execs as &$exec) { if (!isset($dataNodes[$exec['datanodes']])) { $dataNodes[$exec['datanodes']] = 1; $categories[] = $exec['datanodes']; } if (!isset($vmOS[$exec['vm_OS']])) { $vmOS[$exec['vm_OS']] = 1; } if (!isset($execTypes[$exec['exec_type']])) { $execTypes[$exec['exec_type']] = 1; } $vmSizes[$exec['vm_size']][$exec['exec_type']][$exec['vm_OS']][$exec['datanodes']] = array(round($exec['exe_time'], 2), round($exec['cost'], 2)); } $i = 0; $seriesColors = array('#7cb5ec', '#434348', '#90ed7d', '#f7a35c', '#8085e9', '#f15c80', '#e4d354', '#2b908f', '#f45b5b', '#91e8e1'); $series = array(); foreach ($vmSizes as $vmSize => $value) { foreach ($execTypes as $execType => $typevalue) { foreach ($vmOS as $OS => $osvalue) { if (isset($vmSizes[$vmSize][$execType][$OS])) { if ($i == sizeof($seriesColors)) { $i = 0; } $costSeries = array('name' => "{$vmSize} {$execType} {$OS} Run cost", 'type' => 'spline', 'dashStyle' => 'longdash', 'yAxis' => 0, 'data' => array(), 'tooltip' => array('valueSuffix' => ' US$'), 'color' => $seriesColors[$i]); $timeSeries = array('name' => "{$vmSize} {$execType} {$OS} Run execution time", 'type' => 'spline', 'yAxis' => 1, 'data' => array(), 'tooltip' => array('valueSuffix' => ' s'), 'color' => $seriesColors[$i++]); foreach ($dataNodes as $datanodes => $dvalue) { if (!isset($value[$execType][$OS][$datanodes])) { $costSeries['data'][] = "null"; $timeSeries['data'][] = "null"; } else { $costSeries['data'][] = $value[$execType][$OS][$datanodes][1]; $timeSeries['data'][] = $value[$execType][$OS][$datanodes][0]; } } $series[] = $timeSeries; $series[] = $costSeries; } } } } } catch (\Exception $e) { $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n"); } echo $this->container->getTwig()->render('nodeseval/nodes_evaluation.html.twig', array('selected' => 'Number of Nodes Evaluation', 'highcharts_js' => HighCharts::getHeader(), 'categories' => json_encode($categories), 'seriesData' => str_replace('"null"', 'null', json_encode($series)), 'options' => Utils::getFilterOptions($dbUtils), 'datefrom' => $datefrom, 'dateto' => $dateto, 'benchs' => $benchs, 'nets' => $nets, 'disks' => $disks, 'blk_sizes' => $blk_sizes, 'comps' => $comps, 'id_clusters' => $id_clusters, 'mapss' => $mapss, 'replications' => $replications, 'iosfs' => $iosfs, 'iofilebufs' => $iofilebufs, 'money' => $money, 'datanodess' => $datanodes, 'bench_types' => $benchtype, 'vm_sizes' => $vm_sizes, 'vm_coress' => $vm_coress, 'vm_RAMs' => $vm_RAMs, 'vm_OS' => $vm_OS, 'hadoop_versions' => $hadoop_versions, 'types' => $types, 'providers' => $provider, 'filters' => $filters, 'allunchecked' => $allunchecked, 'select_multiple_benchs' => false, 'minexetime' => $minexetime, 'maxexetime' => $maxexetime, 'preset' => $preset, 'selPreset' => $selPreset, 'select_multiple_benchs' => false)); }
public function mlminconfigsAction() { $jsonData = array(); $message = $instance = ''; try { $dbml = new \PDO($this->container->get('config')['db_conn_chain_ml'], $this->container->get('config')['mysql_user'], $this->container->get('config')['mysql_pwd']); $dbml->setAttribute(\PDO::ATTR_ERRMODE, \PDO::ERRMODE_EXCEPTION); $dbml->setAttribute(\PDO::ATTR_EMULATE_PREPARES, false); $db = $this->container->getDBUtils(); $where_configs = ''; $preset = null; if (count($_GET) <= 1 || count($_GET) == 2 && array_key_exists('learn', $_GET)) { $preset = Utils::setDefaultPreset($db, 'mlminconfigs'); } $selPreset = isset($_GET['presets']) ? $_GET['presets'] : "none"; $params = array(); $param_names = array('benchs', 'nets', 'disks', 'mapss', 'iosfs', 'replications', 'iofilebufs', 'comps', 'blk_sizes', 'id_clusters', 'datanodess', 'bench_types', 'vm_sizes', 'vm_coress', 'vm_RAMs', 'types'); // Order is important foreach ($param_names as $p) { $params[$p] = Utils::read_params($p, $where_configs, FALSE); sort($params[$p]); } $learn_param = array_key_exists('learn', $_GET) ? $_GET['learn'] : 'regtree'; $unrestricted = array_key_exists('umodel', $_GET) && $_GET['umodel'] == 1; // FIXME PATCH FOR PARAM LIBRARIES WITHOUT LEGACY $where_configs = str_replace("`id_cluster`", "e.`id_cluster`", $where_configs); $where_configs = str_replace("AND .", "AND ", $where_configs); // compose instance $instance = MLUtils::generateSimpleInstance($param_names, $params, $unrestricted, $db); // Used only as indicator in the WEB $model_info = MLUtils::generateModelInfo($param_names, $params, $unrestricted, $db); $config = $model_info . ' ' . $learn_param . ' ' . ($unrestricted ? 'U' : 'R') . ' minconfs'; $learn_options = 'saveall=' . md5($config); if ($learn_param == 'regtree') { $learn_method = 'aloja_regtree'; $learn_options .= ':prange=0,20000'; } else { if ($learn_param == 'nneighbours') { $learn_method = 'aloja_nneighbors'; $learn_options .= ':kparam=3'; } else { if ($learn_param == 'nnet') { $learn_method = 'aloja_nnet'; $learn_options .= ':prange=0,20000'; } else { if ($learn_param == 'polyreg') { $learn_method = 'aloja_linreg'; $learn_options .= ':ppoly=3:prange=0,20000'; } } } } $cache_ds = getcwd() . '/cache/query/' . md5($config) . '-cache.csv'; $is_cached_mysql = $dbml->query("SELECT count(*) as num FROM learners WHERE id_learner = '" . md5($config) . "'"); $tmp_result = $is_cached_mysql->fetch(); $is_cached = $tmp_result['num'] > 0; $is_cached_mysql = $dbml->query("SELECT count(*) as num FROM minconfigs WHERE id_minconfigs = '" . md5($config . 'R') . "' AND id_learner = '" . md5($config) . "'"); $tmp_result = $is_cached_mysql->fetch(); $is_cached = $is_cached && $tmp_result['num'] > 0; $in_process = file_exists(getcwd() . '/cache/query/' . md5($config) . '.lock'); $finished_process = file_exists(getcwd() . '/cache/query/' . md5($config) . '.fin'); // Create Models and Predictions if (!$is_cached && !$in_process && !$finished_process) { // get headers for csv $header_names = array('id_exec' => 'ID', 'bench' => 'Benchmark', 'exe_time' => 'Exe.Time', 'net' => 'Net', 'disk' => 'Disk', 'maps' => 'Maps', 'iosf' => 'IO.SFac', 'replication' => 'Rep', 'iofilebuf' => 'IO.FBuf', 'comp' => 'Comp', 'blk_size' => 'Blk.size', 'e.id_cluster' => 'Cluster', 'name' => 'Cl.Name', 'datanodes' => 'Datanodes', 'headnodes' => 'Headnodes', 'vm_OS' => 'VM.OS', 'vm_cores' => 'VM.Cores', 'vm_RAM' => 'VM.RAM', 'provider' => 'Provider', 'vm_size' => 'VM.Size', 'type' => 'Type', 'bench_type' => 'Bench.Type'); $headers = array_keys($header_names); $names = array_values($header_names); // dump the result to csv $query = "SELECT " . implode(",", $headers) . " FROM execs e LEFT JOIN clusters c ON e.id_cluster = c.id_cluster WHERE e.valid = TRUE AND bench_type = 'HiBench' AND bench NOT LIKE 'prep_%' AND e.exe_time > 100" . $where_configs . ";"; $rows = $db->get_rows($query); if (empty($rows)) { throw new \Exception('No data matches with your critteria.'); } $fp = fopen($cache_ds, 'w'); fputcsv($fp, $names, ',', '"'); foreach ($rows as $row) { $row['id_cluster'] = "Cl" . $row['id_cluster']; // Cluster is numerically codified... $row['comp'] = "Cmp" . $row['comp']; // Compression is numerically codified... fputcsv($fp, array_values($row), ',', '"'); } // run the R processor exec('cd ' . getcwd() . '/cache/query; touch ' . md5($config) . '.lock'); $command = getcwd() . '/resources/queue -c "cd ' . getcwd() . '/cache/query; ../../resources/aloja_cli.r -d ' . $cache_ds . ' -m ' . $learn_method . ' -p ' . $learn_options . ' >/dev/null 2>&1 && '; $command = $command . '../../resources/aloja_cli.r -m aloja_minimal_instances -l ' . md5($config) . ' -p saveall=' . md5($config . 'R') . ':kmax=200 >/dev/null 2>&1; rm -f ' . md5($config) . '.lock; touch ' . md5($config) . '.fin" >/dev/null 2>&1 &'; exec($command); } $in_process = file_exists(getcwd() . '/cache/query/' . md5($config) . '.lock'); if ($in_process) { $jsonData = $jsonHeader = $configs = '[]'; $must_wait = "YES"; $max_x = $max_y = 0; } else { $must_wait = "NO"; // Save learning model to DB, with predictions $is_cached_mysql = $dbml->query("SELECT id_learner FROM learners WHERE id_learner = '" . md5($config) . "'"); $tmp_result = $is_cached_mysql->fetch(); if ($tmp_result['id_learner'] != md5($config)) { // register model to DB $query = "INSERT INTO learners (id_learner,instance,model,algorithm)"; $query = $query . " VALUES ('" . md5($config) . "','" . $instance . "','" . substr($model_info, 1) . "','" . $learn_param . "');"; if ($dbml->query($query) === FALSE) { throw new \Exception('Error when saving model into DB'); } // read results of the CSV and dump to DB foreach (array("tt", "tv", "tr") as $value) { if (($handle = fopen(getcwd() . '/cache/query/' . md5($config) . '-' . $value . '.csv', 'r')) !== FALSE) { $header = fgetcsv($handle, 1000, ","); $token = 0; $query = "INSERT INTO predictions (id_exec,exe_time,bench,net,disk,maps,iosf,replication,iofilebuf,comp,blk_size,id_cluster,name,datanodes,headnodes,vm_OS,vm_cores,vm_RAM,provider,vm_size,type,bench_type,pred_time,id_learner,instance,predict_code) VALUES "; while (($data = fgetcsv($handle, 1000, ",")) !== FALSE) { $specific_instance = implode(",", array_slice($data, 2, 19)); $specific_data = implode(",", $data); $specific_data = preg_replace('/,Cmp(\\d+),/', ',${1},', $specific_data); $specific_data = preg_replace('/,Cl(\\d+),/', ',${1},', $specific_data); $specific_data = str_replace(",", "','", $specific_data); $query_var = "SELECT count(*) as num FROM predictions WHERE instance = '" . $specific_instance . "' AND id_learner = '" . md5($config) . "'"; $result = $dbml->query($query_var); $row = $result->fetch(); // Insert instance values if ($row['num'] == 0) { if ($token != 0) { $query = $query . ","; } $token = 1; $query = $query . "('" . $specific_data . "','" . md5($config) . "','" . $specific_instance . "','" . ($value == 'tt' ? 3 : ($value == 'tv' ? 2 : 1)) . "') "; } } if ($dbml->query($query) === FALSE) { throw new \Exception('Error when saving into DB'); } fclose($handle); } } // Remove temporal files $output = shell_exec('rm -f ' . getcwd() . '/cache/query/' . md5($config) . '*.csv'); $output = shell_exec('rm -f ' . getcwd() . '/cache/query/' . md5($config) . '*.dat'); } // Save minconfigs to DB, with props and centers $is_cached_mysql = $dbml->query("SELECT id_minconfigs FROM minconfigs WHERE id_minconfigs = '" . md5($config . 'R') . "'"); $tmp_result = $is_cached_mysql->fetch(); if ($tmp_result['id_minconfigs'] != md5($config . 'R')) { // register minconfigs to DB $query = "INSERT INTO minconfigs (id_minconfigs,id_learner,instance,model)"; $query = $query . " VALUES ('" . md5($config . 'R') . "','" . md5($config) . "','" . $instance . "','" . substr($model_info, 1) . "');"; if ($dbml->query($query) === FALSE) { throw new \Exception('Error when saving minconfis into DB'); } $clusters = array(); // Save results of the CSV - MAE or RAE if (file_exists(getcwd() . '/cache/query/' . md5($config . 'R') . '-raes.csv')) { $error_file = 'raes.csv'; } else { $error_file = 'maes.csv'; } $handle = fopen(getcwd() . '/cache/query/' . md5($config . 'R') . '-' . $error_file, 'r'); while (($data = fgetcsv($handle, 1000, ",")) !== FALSE) { $cluster = (int) $data[0]; if ($error_file == 'raes.csv') { $error_mae = 'NULL'; $error_rae = (double) $data[1]; } if ($error_file == 'maes.csv') { $error_mae = (double) $data[1]; $error_rae = 'NULL'; } // register minconfigs_props to DB $query = "INSERT INTO minconfigs_props (id_minconfigs,cluster,MAE,RAE)"; $query = $query . " VALUES ('" . md5($config . 'R') . "','" . $cluster . "','" . $error_mae . "','" . $error_rae . "');"; if ($dbml->query($query) === FALSE) { throw new \Exception('Error when saving minconfis into DB'); } $clusters[] = $cluster; } fclose($handle); // Save results of the CSV - Configs $handle_sizes = fopen(getcwd() . '/cache/query/' . md5($config . 'R') . '-sizes.csv', 'r'); foreach ($clusters as $cluster) { // Get supports from sizes $sizes = fgetcsv($handle_sizes, 1000, ","); // Get clusters $handle = fopen(getcwd() . '/cache/query/' . md5($config . 'R') . '-dsk' . $cluster . '.csv', 'r'); $header = fgetcsv($handle, 1000, ","); $i = 0; while (($data = fgetcsv($handle, 1000, ",")) !== FALSE) { $subdata = array_slice($data, 0, 12); $specific_data = implode(',', $subdata); $specific_data = preg_replace('/,Cmp(\\d+),/', ',${1},', $specific_data); $specific_data = preg_replace('/,Cl(\\d+),/', ',${1},', $specific_data); $specific_data = preg_replace('/,Cl(\\d+)/', ',${1}', $specific_data); $specific_data = str_replace(",", "','", $specific_data); // register minconfigs_props to DB $query = "INSERT INTO minconfigs_centers (id_minconfigs,cluster,id_exec,exe_time,bench,net,disk,maps,iosf,replication,iofilebuf,comp,blk_size,id_cluster,support)"; $query = $query . " VALUES ('" . md5($config . 'R') . "','" . $cluster . "','" . $specific_data . "','" . $sizes[$i++] . "');"; if ($dbml->query($query) === FALSE) { throw new \Exception('Error when saving centers into DB'); } } fclose($handle); } fclose($handle_sizes); // Store file model to DB $filemodel = getcwd() . '/cache/query/' . md5($config) . '-object.rds'; $fp = fopen($filemodel, 'r'); $content = fread($fp, filesize($filemodel)); $content = addslashes($content); fclose($fp); $query = "INSERT INTO model_storage (id_hash,type,file) VALUES ('" . md5($config) . "','learner','" . $content . "');"; if ($dbml->query($query) === FALSE) { throw new \Exception('Error when saving file model into DB'); } $filemodel = getcwd() . '/cache/query/' . md5($config . 'R') . '-object.rds'; $fp = fopen($filemodel, 'r'); $content = fread($fp, filesize($filemodel)); $content = addslashes($content); fclose($fp); $query = "INSERT INTO model_storage (id_hash,type,file) VALUES ('" . md5($config . 'R') . "','minconf','" . $content . "');"; if ($dbml->query($query) === FALSE) { throw new \Exception('Error when saving file minconf into DB'); } // Remove temporal files $output = shell_exec('rm -f ' . getcwd() . '/cache/query/' . md5($config . 'R') . '*.csv'); $output = shell_exec('rm -f ' . getcwd() . '/cache/query/' . md5($config . 'R') . '*.rds'); $output = shell_exec('rm -f ' . getcwd() . '/cache/query/' . md5($config) . '*.rds'); $output = shell_exec('rm -f ' . getcwd() . '/cache/query/' . md5($config) . '.fin'); } // Retrieve minconfig progression results from DB $header = "id_exec,exe_time,bench,net,disk,maps,iosf,replication,iofilebuf,comp,blk_size,id_cluster,support"; $header_array = explode(",", $header); $last_y = 9000000000000000.0; $configs = '['; $query = "SELECT cluster, MAE, RAE FROM minconfigs_props WHERE id_minconfigs='" . md5($config . 'R') . "'"; $result = $dbml->query($query); foreach ($result as $row) { // Retrieve minconfig progression results from DB if ((double) $row['MAE'] > 0) { $error = (double) $row['MAE']; } else { $error = (double) $row['RAE']; } $cluster = (int) $row['cluster']; $new_val = array(); $new_val['x'] = $cluster; if ($error > $last_y) { $new_val['y'] = $last_y; } else { $last_y = $new_val['y'] = $error; } $jsonData[] = $new_val; // Retrieve minconfig centers from DB $query_2 = "SELECT " . $header . " FROM minconfigs_centers WHERE id_minconfigs='" . md5($config . 'R') . "' AND cluster='" . $cluster . "'"; $result_2 = $dbml->query($query_2); $jsonConfig = '['; foreach ($result_2 as $row_2) { $values = ''; foreach ($header_array as $ha) { $values = $values . ($values != '' ? ',' : '') . '\'' . $row_2[$ha] . '\''; } $jsonConfig = $jsonConfig . ($jsonConfig != '[' ? ',' : '') . '[' . $values . ']'; } $jsonConfig = $jsonConfig . ']'; $configs = $configs . ($configs != '[' ? ',' : '') . $jsonConfig; } $configs = $configs . ']'; $jsonData = json_encode($jsonData); $jsonHeader = '[{title:""},{title:"Est.Time"},{title:"Benchmark"},{title:"Network"},{title:"Disk"},{title:"Maps"},{title:"IO.SF"},{title:"Replicas"},{title:"IO.FBuf"},{title:"Compression"},{title:"Blk.Size"},{title:"Main Ref. Cluster"},{title:"Support"}]'; $is_cached_mysql = $dbml->query("SELECT MAX(cluster) as mcluster, MAX(MAE) as mmae, MAX(RAE) as mrae FROM minconfigs_props WHERE id_minconfigs='" . md5($config . 'R') . "'"); $tmp_result = $is_cached_mysql->fetch(); $max_x = (double) $tmp_result['mmae'] > 0 ? (double) $tmp_result['mmae'] : (double) $tmp_result['mrae']; $max_y = (double) $tmp_result['mcluster']; } } catch (\Exception $e) { $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n"); $jsonData = $jsonHeader = $configs = '[]'; $max_x = $max_y = 0; $must_wait = 'NO'; } echo $this->container->getTwig()->render('mltemplate/mlminconfigs.html.twig', array('selected' => 'mlminconfigs', 'jsonData' => $jsonData, 'jsonHeader' => $jsonHeader, 'configs' => $configs, 'max_p' => min(array($max_x, $max_y)), 'benchs' => $params['benchs'], 'nets' => $params['nets'], 'disks' => $params['disks'], 'blk_sizes' => $params['blk_sizes'], 'comps' => $params['comps'], 'id_clusters' => $params['id_clusters'], 'mapss' => $params['mapss'], 'replications' => $params['replications'], 'iosfs' => $params['iosfs'], 'iofilebufs' => $params['iofilebufs'], 'datanodess' => $params['datanodess'], 'bench_types' => $params['bench_types'], 'vm_sizes' => $params['vm_sizes'], 'vm_coress' => $params['vm_coress'], 'vm_RAMs' => $params['vm_RAMs'], 'types' => $params['types'], 'message' => $message, 'instance' => $instance, 'id_learner' => md5($config), 'id_minconf' => md5($config . 'R'), 'model_info' => $model_info, 'unrestricted' => $unrestricted, 'learn' => $learn_param, 'must_wait' => $must_wait, 'preset' => $preset, 'selPreset' => $selPreset, 'options' => Utils::getFilterOptions($db))); }
public function histogramHDIAction() { $idExec = ''; $dbConn = $this->getContainer()->getDBUtils(); try { $idExec = Utils::get_GET_string('id_exec'); if (!$idExec) { $idExec = @$dbConn->get_rows("SELECT id_exec FROM aloja2.execs WHERE perf_details = 1 AND valid = 1 AND filter = 0 AND hadoop_version = 2 LIMIT 5")[rand(0, 5)]['id_exec']; } } catch (\Exception $e) { $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n"); } if (!$idExec) { $this->container->getTwig()->addGlobal('message', 'No executions of Hadoop 2 with performance details available'); $exec = null; } else { $exec = $dbConn->get_rows("SELECT * FROM aloja2.execs JOIN aloja2.clusters USING (id_cluster) WHERE id_exec = {$idExec}")[0]; } return $this->render('perfDetailsViews/histogramhdi.html.twig', array('idExec' => $idExec, 'exec' => $exec)); }
public function getDisksName($diskName) { return Utils::getDisksName($diskName); }
public function mldatacollapseAction() { try { $dbml = new \PDO($this->container->get('config')['db_conn_chain'], $this->container->get('config')['mysql_user'], $this->container->get('config')['mysql_pwd']); $dbml->setAttribute(\PDO::ATTR_ERRMODE, \PDO::ERRMODE_EXCEPTION); $dbml->setAttribute(\PDO::ATTR_EMULATE_PREPARES, false); $db = $this->container->getDBUtils(); $where_configs = ''; $preset = null; if (count($_GET) <= 1 || count($_GET) == 2 && array_key_exists("current_model", $_GET)) { $preset = Utils::initDefaultPreset($db, 'mldatacollapse'); } $selPreset = isset($_GET['presets']) ? $_GET['presets'] : "none"; $params = array(); $param_names = array('benchs', 'nets', 'disks', 'mapss', 'iosfs', 'replications', 'iofilebufs', 'comps', 'blk_sizes', 'id_clusters', 'datanodess', 'bench_types', 'vm_sizes', 'vm_coress', 'vm_RAMs', 'types', 'hadoop_versions'); // Order is important foreach ($param_names as $p) { $params[$p] = Utils::read_params($p, $where_configs, FALSE); sort($params[$p]); } $params_additional = array(); $param_names_additional = array('datefrom', 'dateto', 'minexetime', 'maxexetime', 'valids', 'filters'); // Order is important foreach ($param_names_additional as $p) { $params_additional[$p] = Utils::read_params($p, $where_configs, FALSE); } $unseen = array_key_exists('unseen', $_GET) && $_GET['unseen'] == 1; // FIXME PATCH FOR PARAM LIBRARIES WITHOUT LEGACY $where_configs = str_replace("AND .", "AND ", $where_configs); $dims1 = (empty($params['nets']) ? '' : 'Net,') . (empty($params['disks']) ? '' : 'Disk,') . (empty($params['blk_sizes']) ? '' : 'Blk.size,') . (empty($params['comps']) ? '' : 'Comp,'); $dims1 = $dims1 . (empty($params['id_clusters']) ? '' : 'Cluster,') . (empty($params['mapss']) ? '' : 'Maps,') . (empty($params['replications']) ? '' : 'Rep,') . (empty($params['iosfs']) ? '' : 'IO.SFac,') . (empty($params['iofilebufs']) ? '' : 'IO.FBuf,'); $dims1 = $dims1 . (empty($params['hadoop_versionss']) ? '' : 'Version,') . (empty($params['bench_types']) ? '' : 'Bench.Type'); if (substr($dims1, -1) == ',') { $dims1 = substr($dims1, 0, -1); } $dims2 = "Benchmark"; // compose instance $instance = MLUtils::generateSimpleInstance($this->filters, $param_names, $params, true); $model_info = MLUtils::generateModelInfo($this->filters, $param_names, $params, true); $slice_info = MLUtils::generateDatasliceInfo($this->filters, $param_names_additional, $params_additional); // select model for filling $possible_models = $possible_models_id = array(); MLUtils::findMatchingModels($model_info, $possible_models, $possible_models_id, $dbml); $current_model = ''; /* if (array_key_exists('current_model',$_GET) && in_array($_GET['current_model'],$possible_models_id)) $current_model = $_GET['current_model']; // FIXME - Needs re-think logic if ($current_model == '') { $query = "SELECT AVG(ABS(exe_time - pred_time)) AS MAE, AVG(ABS(exe_time - pred_time)/exe_time) AS RAE, p.id_learner FROM predictions p, learners l WHERE l.id_learner = p.id_learner AND p.id_learner IN ('".implode("','",$possible_models_id)."') AND predict_code > 0 ORDER BY MAE LIMIT 1"; $result = $dbml->query($query); $row = $result->fetch(); $current_model = $row['id_learner']; } */ $learning_model = ''; if ($current_model != '' && file_exists(getcwd() . '/cache/query/' . $current_model . '-object.rds')) { $learning_model = ':model_name=' . $current_model . ':inst_general="' . $instance . '"'; } $config = $dims1 . '-' . $dims2 . '-' . $current_model . '-' . $model_info . '-' . $slice_info; // get headers for csv $header_names = array('id_exec' => 'ID', 'bench' => 'Benchmark', 'exe_time' => 'Exe.Time', 'net' => 'Net', 'disk' => 'Disk', 'maps' => 'Maps', 'iosf' => 'IO.SFac', 'replication' => 'Rep', 'iofilebuf' => 'IO.FBuf', 'comp' => 'Comp', 'blk_size' => 'Blk.size', 'e.id_cluster' => 'Cluster', 'name' => 'Cl.Name', 'datanodes' => 'Datanodes', 'headnodes' => 'Headnodes', 'vm_OS' => 'VM.OS', 'vm_cores' => 'VM.Cores', 'vm_RAM' => 'VM.RAM', 'provider' => 'Provider', 'vm_size' => 'VM.Size', 'type' => 'Type', 'bench_type' => 'Bench.Type', 'hadoop_version' => 'Hadoop.Version'); $headers = array_keys($header_names); $names = array_values($header_names); $dims1_array = explode(",", $dims1); $dims1_query = ''; $dims1_title = $dims1_concat = ''; foreach ($dims1_array as $d1value) { $dims1_query = $dims1_query . ($dims1_query == '' ? '' : ',') . array_search($d1value, $header_names); $dims1_title = $dims1_title . ($dims1_title == '' ? '' : ':') . array_search($d1value, $header_names); $dims1_concat = $dims1_concat . ($dims1_concat == '' ? '' : ',":",') . array_search($d1value, $header_names); } $query = "SELECT distinct bench FROM aloja2.execs e LEFT JOIN aloja2.clusters c ON e.id_cluster = c.id_cluster WHERE hadoop_version IS NOT NULL" . $where_configs . " ORDER BY bench;"; $rows = $db->get_rows($query); if (empty($rows)) { throw new \Exception('No data matches with your critteria.'); } $table = array(); $jsonHeader = '[{title:"' . $dims1_title . '"}'; foreach ($rows as $row) { $jsonHeader = $jsonHeader . ',{title:"' . $row['bench'] . '"}'; $table[$row['bench']] = array(); } $jsonHeader = $jsonHeader . ']'; $query = "SELECT CONCAT(" . $dims1_concat . ") as dim1, bench, avg(exe_time) as avg_exe_time FROM aloja2.execs e LEFT JOIN aloja2.clusters c ON e.id_cluster = c.id_cluster WHERE hadoop_version IS NOT NULL" . $where_configs . " GROUP BY bench," . $dims1_query . " ORDER BY dim1,bench;"; $rows = $db->get_rows($query); if (empty($rows)) { throw new \Exception('No data matches with your critteria.'); } foreach ($rows as $row) { $table[$row['bench']][$row['dim1']] = (int) $row['avg_exe_time']; } $row_ids = array(); foreach ($table as $bmk) { foreach ($bmk as $key => $value) { $row_ids[] = $key; } } $row_ids = array_unique($row_ids); $tableColor = array(); foreach ($table as $bmk => $values) { $tableColor[$bmk] = array(); foreach ($row_ids as $rid) { if (!array_key_exists($rid, $table[$bmk])) { $table[$bmk][$rid] = 0; $tableColor[$bmk][$rid] = 0; } else { $tableColor[$bmk][$rid] = 1; } } } $jsonData = '['; $jsonColor = '['; foreach ($row_ids as $rid) { $jsonData = $jsonData . ($jsonData == '[' ? '' : ',') . '[\'' . $rid . '\''; $jsonColor = $jsonColor . ($jsonColor == '[' ? '' : ',') . '[1'; foreach ($table as $bmk => $values) { $jsonData = $jsonData . ',' . $table[$bmk][$rid]; $jsonColor = $jsonColor . ',' . $tableColor[$bmk][$rid]; } $jsonData = $jsonData . ']'; $jsonColor = $jsonColor . ']'; } $jsonData = $jsonData . ']'; $jsonColor = $jsonColor . ']'; $jsonColumns = '['; for ($i = 1; $i <= count($table); $i++) { if ($jsonColumns != '[') { $jsonColumns = $jsonColumns . ','; } $jsonColumns = $jsonColumns . $i; } $jsonColumns = $jsonColumns . ']'; } catch (\Exception $e) { $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n"); $jsonData = $jsonHeader = $jsonColumns = $jsonColor = '[]'; } $return_params = array('selected' => 'mldatacollapse', 'jsonEncoded' => $jsonData, 'jsonHeader' => $jsonHeader, 'jsonColumns' => $jsonColumns, 'jsonColor' => $jsonColor, 'instance' => $instance, 'instance' => $instance, 'model_info' => $model_info, 'slice_info' => $slice_info, 'preset' => $preset, 'selPreset' => $selPreset, 'options' => Utils::getFilterOptions($db)); foreach ($param_names as $p) { $return_params[$p] = $params[$p]; } foreach ($param_names_additional as $p) { $return_params[$p] = $params_additional[$p]; } echo $this->container->getTwig()->render('mltemplate/mldatacollapse.html.twig', $return_params); }
public function mlsummariesAction() { $displaydata = $message = ''; try { $dbml = new \PDO($this->container->get('config')['db_conn_chain_ml'], $this->container->get('config')['mysql_user'], $this->container->get('config')['mysql_pwd']); $dbml->setAttribute(\PDO::ATTR_ERRMODE, \PDO::ERRMODE_EXCEPTION); $dbml->setAttribute(\PDO::ATTR_EMULATE_PREPARES, false); $db = $this->container->getDBUtils(); $where_configs = ''; $preset = null; if (count($_GET) <= 1) { $preset = Utils::setDefaultPreset($db, 'mlsummaries'); } $selPreset = isset($_GET['presets']) ? $_GET['presets'] : "none"; $params = array(); $param_names = array('benchs', 'nets', 'disks', 'mapss', 'iosfs', 'replications', 'iofilebufs', 'comps', 'blk_sizes', 'id_clusters', 'datanodess', 'bench_types', 'vm_sizes', 'vm_coress', 'vm_RAMs', 'types'); // Order is important foreach ($param_names as $p) { $params[$p] = Utils::read_params($p, $where_configs, FALSE); sort($params[$p]); } $separate_feat = 'joined'; if (array_key_exists('feature', $_GET)) { $separate_feat = $_GET['feature']; } // compose instance $instance = MLUtils::generateSimpleInstance($param_names, $params, true, $db); $model_info = MLUtils::generateModelInfo($param_names, $params, true, $db); $config = $model_info . ' ' . $separate_feat . ' SUMMARY'; $cache_ds = getcwd() . '/cache/query/' . md5($config) . '-cache.csv'; $is_cached_mysql = $dbml->query("SELECT count(*) as num FROM summaries WHERE id_summaries = '" . md5($config) . "'"); $tmp_result = $is_cached_mysql->fetch(); $is_cached = $tmp_result['num'] > 0; if (!$is_cached) { // get headers for csv $header_names = array('id_exec' => 'ID', 'bench' => 'Benchmark', 'exe_time' => 'Exe.Time', 'net' => 'Net', 'disk' => 'Disk', 'maps' => 'Maps', 'iosf' => 'IO.SFac', 'replication' => 'Rep', 'iofilebuf' => 'IO.FBuf', 'comp' => 'Comp', 'blk_size' => 'Blk.size', 'e.id_cluster' => 'Cluster', 'name' => 'Cl.Name', 'datanodes' => 'Datanodes', 'headnodes' => 'Headnodes', 'vm_OS' => 'VM.OS', 'vm_cores' => 'VM.Cores', 'vm_RAM' => 'VM.RAM', 'provider' => 'Provider', 'vm_size' => 'VM.Size', 'type' => 'Type', 'bench_type' => 'Bench.Type'); $headers = array_keys($header_names); $names = array_values($header_names); // dump the result to csv $query = "SELECT " . implode(",", $headers) . " FROM execs e LEFT JOIN clusters c ON e.id_cluster = c.id_cluster WHERE e.valid = TRUE AND e.exe_time > 100" . $where_configs . ";"; $rows = $db->get_rows($query); if (empty($rows)) { throw new \Exception('No data matches with your critteria.'); } $fp = fopen($cache_ds, 'w'); fputcsv($fp, $names, ',', '"'); foreach ($rows as $row) { $row['id_cluster'] = "Cl" . $row['id_cluster']; // Cluster is numerically codified... $row['comp'] = "Cmp" . $row['comp']; // Compression is numerically codified... fputcsv($fp, array_values($row), ',', '"'); } // launch query $command = 'cd ' . getcwd() . '/cache/query; ../../resources/aloja_cli.r -m aloja_print_summaries -d ' . $cache_ds . ' -p ' . ($separate_feat != 'joined' ? 'sname=' . $separate_feat . ':' : '') . 'fprint=' . md5($config) . ':fwidth=1000:html=1'; #fwidth=135 $output = shell_exec($command); // Save to DB if (($handle = fopen(getcwd() . '/cache/query/' . md5($config) . '-summary.data', 'r')) !== FALSE) { $displaydata = ""; while (($data = fgets($handle)) !== FALSE) { $displaydata = $displaydata . $data; } fclose($handle); $displaydata = str_replace('\'', '\\\'', $displaydata); // register model to DB $query = "INSERT INTO summaries (id_summaries,instance,model,summary)"; $query = $query . " VALUES ('" . md5($config) . "','" . $instance . "','" . substr($model_info, 1) . "','" . $displaydata . "');"; if ($dbml->query($query) === FALSE) { throw new \Exception('Error when saving model into DB'); } } // Remove temporal files $output = shell_exec('rm -f ' . getcwd() . '/cache/query/' . md5($config) . '-summary.data'); $output = shell_exec('rm -f ' . getcwd() . '/cache/query/' . md5($config) . '-cache.csv'); } // Read results of the DB $is_cached_mysql = $dbml->query("SELECT summary FROM summaries WHERE id_summaries = '" . md5($config) . "' LIMIT 1"); $tmp_result = $is_cached_mysql->fetch(); $displaydata = $tmp_result['summary']; } catch (\Exception $e) { $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n"); $displaydata = $separate_feat = ''; } echo $this->container->getTwig()->render('mltemplate/mlsummaries.html.twig', array('selected' => 'mlsummaries', 'displaydata' => $displaydata, 'benchs' => $params['benchs'], 'nets' => $params['nets'], 'disks' => $params['disks'], 'blk_sizes' => $params['blk_sizes'], 'comps' => $params['comps'], 'id_clusters' => $params['id_clusters'], 'mapss' => $params['mapss'], 'replications' => $params['replications'], 'iosfs' => $params['iosfs'], 'iofilebufs' => $params['iofilebufs'], 'datanodess' => $params['datanodess'], 'bench_types' => $params['bench_types'], 'vm_sizes' => $params['vm_sizes'], 'vm_coress' => $params['vm_coress'], 'vm_RAMs' => $params['vm_RAMs'], 'types' => $params['types'], 'feature' => $separate_feat, 'message' => $message, 'preset' => $preset, 'selPreset' => $selPreset, 'options' => Utils::getFilterOptions($db))); }
public function hdp2CountersDataAction() { $db = $this->container->getDBUtils(); $this->buildFilters(array('bench' => array('table' => 'execs', 'default' => null, 'type' => 'selectMultiple'))); $whereClause = $this->filters->getWhereClause(); try { //check the URL $execs = Utils::get_GET_intArray('execs'); if (!($type = Utils::get_GET_string('pageTab'))) { $type = 'SUMMARY'; } $join = "JOIN aloja2.execs e using (id_exec) LEFT JOIN aloja_ml.predictions p USING (id_exec) WHERE e.valid = 1 AND job_name NOT IN\n ('TeraGen', 'random-text-writer', 'mahout-examples-0.7-job.jar', 'Create pagerank nodes', 'Create pagerank links') {$whereClause}" . ($execs ? ' AND id_exec IN (' . join(',', $execs) . ') ' : '') . " LIMIT 10000"; if ($type == 'SUMMARY') { $query = "SELECT e.bench, e.exe_time, c.id_exec, c.JOB_ID, c.job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME, c.TOTAL_MAPS, c.FAILED_MAPS, c.FINISHED_MAPS, c.TOTAL_REDUCES, c.FAILED_REDUCES, c.job_name as CHARTS,\n \t\t\te.perf_details\n \t\t\tFROM aloja2.HDI_JOB_details c {$join}"; } elseif ($type == 'MAP') { $query = "SELECT e.bench, e.exe_time, c.id_exec, JOB_ID, job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME, c.TOTAL_MAPS, c.FAILED_MAPS, c.FINISHED_MAPS, `TOTAL_LAUNCHED_MAPS`,\n \t\t\t`RACK_LOCAL_MAPS`,\n \t\t\t`SPILLED_RECORDS`,\n \t\t\t`MAP_INPUT_RECORDS`,\n \t\t\t`MAP_OUTPUT_RECORDS`,\n \t\t\t`MAP_OUTPUT_BYTES`,\n \t\t\t`MAP_OUTPUT_MATERIALIZED_BYTES`,\n \t\t\te.perf_details\n \t\t\tFROM aloja2.HDI_JOB_details c {$join}"; } elseif ($type == 'REDUCE') { $query = "SELECT e.bench, e.exe_time, c.id_exec, c.JOB_ID, c.job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME, c.TOTAL_REDUCES, c.FAILED_REDUCES,\n \t\t\t`TOTAL_LAUNCHED_REDUCES`,\n \t\t\t`REDUCE_INPUT_GROUPS`,\n \t\t\t`REDUCE_INPUT_RECORDS`,\n \t\t\t`REDUCE_OUTPUT_RECORDS`,\n \t\t\t`REDUCE_SHUFFLE_BYTES`,\n \t\t\t`COMBINE_INPUT_RECORDS`,\n \t\t\t`COMBINE_OUTPUT_RECORDS`,\n \t\t\te.perf_details\n \t\t\tFROM aloja2.HDI_JOB_details c {$join}"; } elseif ($type == 'FILE-IO') { $query = "SELECT e.bench, e.exe_time, c.id_exec, c.JOB_ID, c.job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME,\n \t\t\t`SLOTS_MILLIS_MAPS`,\n \t\t\t`SLOTS_MILLIS_REDUCES`,\n \t\t\t`SPLIT_RAW_BYTES`,\n \t\t\t`FILE_BYTES_WRITTEN`,\n \t\t\t`FILE_BYTES_READ`,\n \t\t\t`WASB_BYTES_WRITTEN`,\n \t\t\t`WASB_BYTES_READ`,\n \t\t\t`BYTES_READ`,\n \t\t\t`BYTES_WRITTEN`,\n \t\t\te.perf_details\n \t\t\tFROM aloja2.HDI_JOB_details c {$join}"; } elseif ($type == 'DETAIL') { $query = "SELECT e.bench, e.exe_time, c.*,e.perf_details FROM aloja2.HDI_JOB_details c {$join}"; } elseif ($type == 'TASKS') { $query = "SELECT e.bench, e.exe_time, j.job_name, c.*,e.perf_details FROM aloja_logs.HDI_JOB_tasks c\n \t\t\tJOIN aloja2.HDI_JOB_details j USING(id_exec,JOB_ID) {$join} "; } else { throw new \Exception('Unknown type!'); } $exec_rows = $db->get_rows($query); if (count($exec_rows) > 0) { $show_in_result_counters = array('id_exec' => 'ID', 'JOB_ID' => 'JOBID', 'bench' => 'Bench', 'job_name' => 'JOBNAME'); $show_in_result_counters = Utils::generate_show($show_in_result_counters, $exec_rows, 4); $jsonData = Utils::generateJSONTable($exec_rows, $show_in_result_counters, 0, 'COUNTER'); header('Content-Type: application/json'); echo json_encode(array('aaData' => $jsonData)); // if (count($exec_rows) > 10000) { // $message .= 'WARNING, large resulset, please limit the query! Rows: '.count($exec_rows); // } } else { echo 'No data available'; } } catch (\Exception $e) { exit($e->getMessage()); echo 'No data available'; /*$noData = array(); for($i = 0; $i<=sizeof($show_in_result); ++$i) $noData[] = 'error'; echo json_encode(array('aaData' => array($noData)));*/ } }
public static function generateInstances($param_names, $params, $generalize, $db) { //$db = $this->container->getDBUtils(); $filter_options = Utils::getFilterOptions($db); // FIXME - FIXING STUFF OF OTHERS... if (array_key_exists("vm_ram", $filter_options)) { $filter_options['vm_RAM'] = $filter_options['vm_ram']; unset($filter_options['vm_ram']); } if (array_key_exists("benchtype", $filter_options)) { $filter_options['bench_type'] = $filter_options['benchtype']; unset($filter_options['benchtype']); } $paramAllOptions = $tokens = $instances = array(); // Get info from clusters (Part of header_names!) $cluster_header_names = array('id_cluster' => 'Cluster', 'name' => 'Cl.Name', 'datanodes' => 'Datanodes', 'headnodes' => 'Headnodes', 'vm_OS' => 'VM.OS', 'vm_cores' => 'VM.Cores', 'vm_RAM' => 'VM.RAM', 'provider' => 'Provider', 'vm_size' => 'VM.Size', 'type' => 'Type'); $cluster_descriptor = array(); $query = "select " . implode(",", array_keys($cluster_header_names)) . " from clusters;"; $rows = $db->get_rows($query); foreach ($rows as $row) { $cid = $row['id_cluster']; foreach (array_keys($cluster_header_names) as $cname) { $cluster_descriptor[$cid][$cname] = $row[$cname]; } } // If "No Clusters" -> All clusters if (empty($params['id_clusters'])) { $params['id_clusters'] = array(); $paramAllOptions['id_clusters'] = array_column($filter_options['id_cluster'], 'id_cluster'); foreach ($paramAllOptions['id_clusters'] as $par) { $params['id_clusters'][] = $par; } } // For each cluster selected, launch an instance... foreach ($params['id_clusters'] as $cl) { // Reduce the instance to the HW filter override, or even remove instance if no HW coincides $remove_if_no_props = FALSE; foreach (array_keys($cluster_header_names) as $cname) { if (!empty($params[$cname])) { // FIXME - When clusters have more than 1 characteristic, change this // Get only the current_props in params[cname] $current_props = $cluster_descriptor[$cl][$cname]; $current_props = array($current_props); $coincidences = array_intersect($current_props, $params[$cname]); if (empty($coincidences)) { $remove_if_no_props = TRUE; } else { $cluster_descriptor[$cl][$cname] = $coincidences; } } } if ($remove_if_no_props) { continue; } $cl_characteristics = "Cl" . implode(",", $cluster_descriptor[$cl]); $instance = ''; foreach ($param_names as $p) { // Ignore for now. Will be used at each cluster characteristics if (array_key_exists(substr($p, 0, -1), $cluster_header_names) && $p != "id_clusters") { continue; } if ($p != "id_clusters") { if (array_key_exists(substr($p, 0, -1), $filter_options)) { $paramAllOptions[$p] = array_column($filter_options[substr($p, 0, -1)], substr($p, 0, -1)); } $tokens[$p] = ''; if ($generalize && empty($params[$p])) { $tokens[$p] = '*'; } elseif (!$generalize && empty($params[$p])) { foreach ($paramAllOptions[$p] as $par) { $tokens[$p] = $tokens[$p] . ($tokens[$p] != '' ? '|' : '') . ($p == 'comps' ? 'Cmp' : '') . ($p == 'id_clusters' ? 'Cl' : '') . $par; } } else { foreach ($params[$p] as $par) { $tokens[$p] = $tokens[$p] . ($tokens[$p] != '' ? '|' : '') . ($p == 'comps' ? 'Cmp' : '') . ($p == 'id_clusters' ? 'Cl' : '') . $par; } } $instance = $instance . ($instance == '' ? '' : ',') . $tokens[$p]; } else { $instance = $instance . ($instance == '' ? '' : ',') . $cl_characteristics; } } $instances[] = $instance; } return $instances; }
public function mloutliersAction() { $jsonData = $jsonWarns = $jsonOuts = array(); $message = $instance = $jsonHeader = $jsonTable = ''; $max_x = $max_y = 0; $must_wait = 'NO'; try { $dbml = new \PDO($this->container->get('config')['db_conn_chain_ml'], $this->container->get('config')['mysql_user'], $this->container->get('config')['mysql_pwd']); $dbml->setAttribute(\PDO::ATTR_ERRMODE, \PDO::ERRMODE_EXCEPTION); $dbml->setAttribute(\PDO::ATTR_EMULATE_PREPARES, false); $db = $this->container->getDBUtils(); $where_configs = ''; $preset = null; if (count($_GET) <= 1 || count($_GET) == 2 && array_key_exists('current_model', $_GET) || count($_GET) == 2 && array_key_exists('dump', $_GET) || count($_GET) == 2 && array_key_exists('register', $_GET) || count($_GET) == 3 && array_key_exists('dump', $_GET) && array_key_exists('current_model', $_GET) || count($_GET) == 3 && array_key_exists('register', $_GET) && array_key_exists('current_model', $_GET)) { $preset = Utils::setDefaultPreset($db, 'mloutliers'); } $selPreset = isset($_GET['presets']) ? $_GET['presets'] : "none"; $params = array(); $param_names = array('benchs', 'nets', 'disks', 'mapss', 'iosfs', 'replications', 'iofilebufs', 'comps', 'blk_sizes', 'id_clusters', 'datanodess', 'bench_types', 'vm_sizes', 'vm_coress', 'vm_RAMs', 'types'); // Order is important foreach ($param_names as $p) { $params[$p] = Utils::read_params($p, $where_configs, FALSE); sort($params[$p]); } $sigma_param = array_key_exists('sigma', $_GET) ? (int) $_GET['sigma'] : 1; // FIXME PATCH FOR PARAM LIBRARIES WITHOUT LEGACY $where_configs = str_replace("AND .", "AND ", $where_configs); // compose instance $instance = MLUtils::generateSimpleInstance($param_names, $params, true, $db); // Used only as indicator for WEB $model_info = MLUtils::generateModelInfo($param_names, $params, true, $db); // model for filling MLUtils::findMatchingModels($model_info, $possible_models, $possible_models_id, $dbml); $current_model = ""; if (array_key_exists('current_model', $_GET) && in_array($_GET['current_model'], $possible_models_id)) { $current_model = $_GET['current_model']; } if (!empty($possible_models_id)) { if ($current_model == "") { $query = "SELECT AVG(ABS(exe_time - pred_time)) AS MAE, AVG(ABS(exe_time - pred_time)/exe_time) AS RAE, p.id_learner FROM predictions p, learners l WHERE l.id_learner = p.id_learner AND p.id_learner IN ('" . implode("','", $possible_models_id) . "') AND predict_code > 0 ORDER BY MAE LIMIT 1"; $result = $dbml->query($query); $row = $result->fetch(); $current_model = $row['id_learner']; } $config = $instance . '-' . $current_model . '-' . $sigma_param . '-outliers'; $is_cached_mysql = $dbml->query("SELECT count(*) as total FROM resolutions WHERE id_resolution = '" . md5($config) . "'"); $tmp_result = $is_cached_mysql->fetch(); $is_cached = $tmp_result['total'] > 0; $cache_ds = getcwd() . '/cache/query/' . md5($config) . '-cache.csv'; $in_process = file_exists(getcwd() . '/cache/query/' . md5($config) . '.lock'); $finished_process = file_exists(getcwd() . '/cache/query/' . md5($config) . '-resolutions.csv'); if (!$is_cached && !$in_process && !$finished_process) { // get headers for csv $header_names = array('id_exec' => 'ID', 'bench' => 'Benchmark', 'exe_time' => 'Exe.Time', 'net' => 'Net', 'disk' => 'Disk', 'maps' => 'Maps', 'iosf' => 'IO.SFac', 'replication' => 'Rep', 'iofilebuf' => 'IO.FBuf', 'comp' => 'Comp', 'blk_size' => 'Blk.size', 'e.id_cluster' => 'Cluster', 'name' => 'Cl.Name', 'datanodes' => 'Datanodes', 'headnodes' => 'Headnodes', 'vm_OS' => 'VM.OS', 'vm_cores' => 'VM.Cores', 'vm_RAM' => 'VM.RAM', 'provider' => 'Provider', 'vm_size' => 'VM.Size', 'type' => 'Type', 'bench_type' => 'Bench.Type'); $headers = array_keys($header_names); $names = array_values($header_names); // dump the result to csv $query = "SELECT " . implode(",", $headers) . " FROM execs e LEFT JOIN clusters c ON e.id_cluster = c.id_cluster WHERE e.valid = TRUE AND e.exe_time > 100" . $where_configs . ";"; $rows = $db->get_rows($query); if (empty($rows)) { throw new \Exception('No data matches with your critteria.'); } $fp = fopen($cache_ds, 'w'); fputcsv($fp, $names, ',', '"'); foreach ($rows as $row) { $row['id_cluster'] = "Cl" . $row['id_cluster']; // Cluster is numerically codified... $row['comp'] = "Cmp" . $row['comp']; // Compression is numerically codified... fputcsv($fp, array_values($row), ',', '"'); } // Retrieve file model from DB $query = "SELECT file FROM model_storage WHERE id_hash='" . $current_model . "' AND type='learner';"; $result = $dbml->query($query); $row = $result->fetch(); $content = $row['file']; $filemodel = getcwd() . '/cache/query/' . $current_model . '-object.rds'; $fp = fopen($filemodel, 'w'); fwrite($fp, $content); fclose($fp); // launch query exec('cd ' . getcwd() . '/cache/query ; touch ' . md5($config) . '.lock'); exec(getcwd() . '/resources/queue -c "cd ' . getcwd() . '/cache/query ; ' . getcwd() . '/resources/aloja_cli.r -m aloja_outlier_dataset -d ' . $cache_ds . ' -l ' . $current_model . ' -p sigma=' . $sigma_param . ':hdistance=3:saveall=' . md5($config) . ' > /dev/null 2>&1 ; rm -f ' . md5($config) . '.lock" > /dev/null 2>&1 &'); } $finished_process = file_exists(getcwd() . '/cache/query/' . md5($config) . '-resolutions.csv'); if ($finished_process && !$is_cached) { if (($handle = fopen(getcwd() . '/cache/query/' . md5($config) . '-resolutions.csv', 'r')) !== FALSE) { $header = fgetcsv($handle, 1000, ","); $token = 0; $query = "REPLACE INTO resolutions (id_resolution,id_learner,id_exec,instance,model,sigma,outlier_code,predicted,observed) VALUES "; while (($data = fgetcsv($handle, 1000, ",")) !== FALSE) { $resolution = $data[0]; $pred_value = (int) $data[1] >= 100 ? (int) $data[1] : 100; $exec_value = (int) $data[2]; $selected_instance_pre = preg_replace('/\\s+/', '', $data[3]); $selected_instance_pre = str_replace(':', ',', $selected_instance_pre); $specific_id = $data[4]; if ($token > 0) { $query = $query . ","; } $token = 1; $query = $query . "('" . md5($config) . "','" . $current_model . "','" . $specific_id . "','" . $selected_instance_pre . "','" . $model_info . "','" . $sigma_param . "','" . $resolution . "','" . $pred_value . "','" . $exec_value . "') "; } if ($dbml->query($query) === FALSE) { throw new \Exception('Error when saving tree into DB'); } } // Store file model to DB $filemodel = getcwd() . '/cache/query/' . md5($config) . '-object.rds'; $fp = fopen($filemodel, 'r'); $content = fread($fp, filesize($filemodel)); $content = addslashes($content); fclose($fp); $query = "INSERT INTO model_storage (id_hash,type,file) VALUES ('" . md5($config) . "','resolution','" . $content . "');"; if ($dbml->query($query) === FALSE) { throw new \Exception('Error when saving file resolution into DB'); } // Remove temporary files $output = shell_exec('rm -f ' . getcwd() . '/cache/query/' . md5($config) . '-*.csv'); $output = shell_exec('rm -f ' . getcwd() . '/cache/query/' . $current_model . '-object.rds'); $output = shell_exec('rm -f ' . getcwd() . '/cache/query/' . md5($config) . '-object.rds'); $is_cached = true; } if (!$is_cached) { $jsonData = $jsonOuts = $jsonWarns = $jsonHeader = $jsonTable = '[]'; $must_wait = 'YES'; if (isset($_GET['dump'])) { echo "1"; exit(0); } } else { $must_wait = 'NO'; $query = "SELECT predicted, observed, outlier_code, id_exec, instance FROM resolutions WHERE id_resolution = '" . md5($config) . "' LIMIT 5000"; // FIXME - CLUMSY PATCH FOR BYPASS THE BUG FROM HIGHCHARTS... REMEMBER TO ERASE THIS LINE WHEN THE BUG IS SOLVED $result = $dbml->query($query); foreach ($result as $row) { $entry = array('x' => (int) $row['predicted'], 'y' => (int) $row['observed'], 'name' => $row['instance'], 'id' => (int) $row['id_exec']); if ($row['outlier_code'] == 0) { $jsonData[] = $entry; } if ($row['outlier_code'] == 1) { $jsonWarns[] = $entry; } if ($row['outlier_code'] == 2) { $jsonOuts[] = $entry; } $jsonTable .= ($jsonTable == '' ? '' : ',') . '["' . ($row['outlier_code'] == 0 ? 'Legitimate' : ($row['outlier_code'] == 1 ? 'Warning' : 'Outlier')) . '","' . $row['predicted'] . '","' . $row['observed'] . '","' . str_replace(",", "\",\"", $row['instance']) . '","' . $row['id_exec'] . '"]'; } $query_var = "SELECT MAX(predicted) as max_x, MAX(observed) as max_y FROM resolutions WHERE id_resolution = '" . md5($config) . "' LIMIT 5000"; $result = $dbml->query($query_var); $row = $result->fetch(); $max_x = $row['max_x']; $max_y = $row['max_y']; $header = array('Prediction', 'Observed', 'Benchmark', 'Net', 'Disk', 'Maps', 'IO.SFS', 'Rep', 'IO.FBuf', 'Comp', 'Blk.Size', 'Cluster', 'Cl.Name', 'Datanodes', 'Headnodes', 'VM.OS', 'VM.Cores', 'VM.RAM', 'Provider', 'VM.Size', 'Type', 'Bench.Type', 'ID'); $jsonHeader = '[{title:""}'; foreach ($header as $title) { $jsonHeader = $jsonHeader . ',{title:"' . $title . '"}'; } $jsonHeader = $jsonHeader . ']'; $jsonData = json_encode($jsonData); $jsonWarns = json_encode($jsonWarns); $jsonOuts = json_encode($jsonOuts); $jsonTable = '[' . $jsonTable . ']'; // Dump case if (isset($_GET['dump'])) { echo str_replace(array("[", "]", "{title:\"", "\"}"), array('', '', ''), $jsonHeader) . "\n"; echo str_replace(array('],[', '[[', ']]'), array("\n", '', ''), $jsonOuts); echo str_replace(array('],[', '[[', ']]'), array("\n", '', ''), $jsonWarns); echo str_replace(array('],[', '[[', ']]'), array("\n", '', ''), $jsonData); exit(0); } // Register case if (isset($_GET['register'])) { // Update the predictions table $query_var = "UPDATE predictions as p, resolutions as r\n\t\t\t\t\t\t\t\tSET p.outlier = r.outlier_code\n\t\t\t\t\t\t\t\tWHERE r.id_exec = p.id_exec\n\t\t\t\t\t\t\t\t\tAND r.id_resolution = '" . md5($config) . "'\n\t\t\t\t\t\t\t\t\tAND p.id_learner = '" . $current_model . "'"; if ($dbml->query($query_var) === FALSE) { throw new \Exception('Error when updating predictions in DB'); } } } } else { $message = "There are no prediction models trained for such parameters. Train at least one model in 'ML Prediction' section."; $must_wait = "NO"; } $dbml = null; } catch (\Exception $e) { $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n"); $jsonData = $jsonOuts = $jsonWarns = $jsonHeader = $jsonTable = '[]'; $model = ''; $possible_models_id = $possible_models = array(); $dbml = null; } echo $this->container->getTwig()->render('mltemplate/mloutliers.html.twig', array('selected' => 'mloutliers', 'jsonData' => $jsonData, 'jsonWarns' => $jsonWarns, 'jsonOuts' => $jsonOuts, 'jsonHeader' => $jsonHeader, 'jsonTable' => $jsonTable, 'max_p' => min(array($max_x, $max_y)), 'benchs' => $params['benchs'], 'nets' => $params['nets'], 'disks' => $params['disks'], 'blk_sizes' => $params['blk_sizes'], 'comps' => $params['comps'], 'id_clusters' => $params['id_clusters'], 'mapss' => $params['mapss'], 'replications' => $params['replications'], 'iosfs' => $params['iosfs'], 'iofilebufs' => $params['iofilebufs'], 'datanodess' => $params['datanodess'], 'bench_types' => $params['bench_types'], 'vm_sizes' => $params['vm_sizes'], 'vm_coress' => $params['vm_coress'], 'vm_RAMs' => $params['vm_RAMs'], 'types' => $params['types'], 'must_wait' => $must_wait, 'models' => '<li>' . implode('</li><li>', $possible_models) . '</li>', 'models_id' => $possible_models_id, 'current_model' => $current_model, 'resolution_id' => md5($config), 'sigma' => $sigma_param, 'message' => $message, 'instance' => $instance, 'preset' => $preset, 'selPreset' => $selPreset, 'options' => Utils::getFilterOptions($db))); }
private function nodesEvalCore($scalabilityType, $dbUtils) { $categories = array(); $series = array(); $datanodes = array(); try { $filter_execs = DBUtils::getFilterExecs(); $innerQueryWhere = str_replace("e.", "e2.", $this->whereClause); $innerQueryWhere = str_replace("c.", "c2.", $innerQueryWhere); $innerQueryWhere = str_replace("p.", "p2.", $innerQueryWhere); $execs = "SELECT c.datanodes as 'category',e.exec_type,c.vm_OS,c.vm_size,(e.exe_time * (c.cost_hour/3600)) as cost,e.exe_time,c.*\n\t\t\t\t\tFROM aloja2.execs e JOIN aloja2.clusters c USING (id_cluster)\n\t\t\t\t\tLEFT JOIN aloja_ml.predictions p USING (id_exec)\n\t\t\t\t\tINNER JOIN ( SELECT c2.datanodes as 'category',e2.exec_type,c2.vm_OS,c2.vm_size as vmsize,MIN(e2.exe_time) as minexe\n\t\t\t\t\t\t\t\tfrom execs e2 JOIN aloja2.clusters c2 USING (id_cluster)\n\t\t\t\t\t\t\t\tLEFT JOIN aloja_ml.predictions p2 USING (id_exec)\n\t\t\t\t\t\t\t\tWHERE 1 {$innerQueryWhere} GROUP BY c2.datanodes,e2.exec_type,c2.vm_OS,c2.vm_size ) t ON t.minexe = e.exe_time\n\t\t\t\t\tAND t.category = category AND t.vmsize = c.vm_size\n\t\t\t\t\tWHERE 1 {$filter_execs} GROUP BY c.datanodes,e.exec_type,c.vm_OS,c.vm_size\n\t\t\t\t\tORDER BY c.datanodes ASC,c.vm_OS,c.vm_size DESC"; $predExecs = "SELECT c.datanodes as 'category','predicted' as 'exec_type',c.vm_OS,c.vm_size,(e.exe_time * (c.cost_hour/3600)) as cost,e.exe_time,c.*\n\t\t\t\t\tFROM aloja_ml.predictions e JOIN aloja2.clusters c USING (id_cluster)\n\t\t\t\t\tINNER JOIN ( SELECT c2.datanodes as 'category','default' as 'exec_type',c2.vm_OS,c2.vm_size as vmsize,MIN(p2.exe_time) as minexe\n\t\t\t\t\t\t\t\tfrom aloja_ml.predictions p2 JOIN aloja2.clusters c2 USING (id_cluster)\n\t\t\t\t\t\t\t\tWHERE 1 " . str_replace("e2.", "p2.", $innerQueryWhere) . " GROUP BY c2.datanodes,exec_type,c2.vm_OS,c2.vm_size ) t ON t.minexe = e.exe_time\n\t\t\t\t\tAND t.category = category AND t.vmsize = c.vm_size\n\t\t\t\t\tWHERE 1 {$filter_execs} GROUP BY c.datanodes,exec_type,c.vm_OS,c.vm_size\n\t\t\t\t\tORDER BY c.datanodes ASC,c.vm_OS,c.vm_size DESC"; if ($scalabilityType == 'Datasize') { $execs = str_replace("c2.datanodes", "e2.datasize", $execs); $execs = str_replace("c.datanodes", "e.datasize", $execs); $predExecs = str_replace("c2.datanodes", "p2.datasize", $predExecs); $predExecs = str_replace("c.datanodes", "e.datasize", $predExecs); } $params = $this->filters->getFiltersSelectedChoices(array('upred', 'uobsr')); if ($params['uobsr'] == 1 && $params['upred'] == 1) { $execs = "({$execs}) UNION ({$predExecs})"; } else { if ($params['uobsr'] == 0 && $params['upred'] == 1) { $execs = $predExecs; } } $execs = $dbUtils->get_rows($execs); $vmSizes = array(); $dataNodes = array(); $vmOS = array(); $execTypes = array(); foreach ($execs as &$exec) { if (!isset($dataNodes[$exec['category']])) { $dataNodes[$exec['category']] = 1; $categories[] = $exec['category']; } if (!isset($vmOS[$exec['vm_OS']])) { $vmOS[$exec['vm_OS']] = 1; } if (!isset($execTypes[$exec['exec_type']])) { $execTypes[$exec['exec_type']] = 1; } $vmSizes[$exec['vm_size']][$exec['exec_type']][$exec['vm_OS']][$exec['category']] = array(round($exec['exe_time'], 2), round($exec['cost'], 2)); } $i = 0; $seriesColors = array('#7cb5ec', '#434348', '#90ed7d', '#f7a35c', '#8085e9', '#f15c80', '#e4d354', '#2b908f', '#f45b5b', '#91e8e1'); foreach ($vmSizes as $vmSize => $value) { foreach ($execTypes as $execType => $typevalue) { foreach ($vmOS as $OS => $osvalue) { if (isset($vmSizes[$vmSize][$execType][$OS])) { if ($i == sizeof($seriesColors)) { $i = 0; } $costSeries = array('name' => "{$vmSize} {$execType} {$OS} Run cost", 'type' => 'spline', 'dashStyle' => 'longdash', 'yAxis' => 0, 'data' => array(), 'tooltip' => array('valueSuffix' => ' US$'), 'color' => $seriesColors[$i]); $timeSeries = array('name' => "{$vmSize} {$execType} {$OS} Run execution time", 'type' => 'spline', 'yAxis' => 1, 'data' => array(), 'tooltip' => array('valueSuffix' => ' s'), 'color' => $seriesColors[$i++]); foreach ($dataNodes as $datanode => $dvalue) { $datanodes[] = $datanode; if (!isset($value[$execType][$OS][$datanode])) { $costSeries['data'][] = "null"; $timeSeries['data'][] = "null"; } else { $costSeries['data'][] = $value[$execType][$OS][$datanode][1]; $timeSeries['data'][] = $value[$execType][$OS][$datanode][0]; } } $series[] = $timeSeries; $series[] = $costSeries; } } } } } catch (\Exception $e) { $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n"); } if ($scalabilityType == 'Datasize') { foreach ($categories as &$category) { $category = Utils::beautifyDatasize($category); } } return array('categories' => json_encode($categories), 'seriesData' => str_replace('"null"', 'null', json_encode($series)), 'datanodess' => $datanodes); }
public function mlparamEvaluationAction() { $rows = $categories = $series = ''; $must_wait = 'NO'; try { $dbml = new \PDO($this->container->get('config')['db_conn_chain_ml'], $this->container->get('config')['mysql_user'], $this->container->get('config')['mysql_pwd']); $dbml->setAttribute(\PDO::ATTR_ERRMODE, \PDO::ERRMODE_EXCEPTION); $dbml->setAttribute(\PDO::ATTR_EMULATE_PREPARES, false); $db = $this->container->getDBUtils(); $where_configs = ''; $preset = null; if (count($_GET) <= 1 || count($_GET) == 2 && array_key_exists('parameval', $_GET) || count($_GET) == 2 && array_key_exists('current_model', $_GET)) { $preset = Utils::setDefaultPreset($db, 'mlparameval'); } $selPreset = isset($_GET['presets']) ? $_GET['presets'] : "none"; $params = array(); $param_names = array('benchs', 'nets', 'disks', 'mapss', 'iosfs', 'replications', 'iofilebufs', 'comps', 'blk_sizes', 'id_clusters', 'datanodess', 'bench_types', 'vm_sizes', 'vm_coress', 'vm_RAMs', 'types'); // Order is important foreach ($param_names as $p) { $params[$p] = Utils::read_params($p, $where_configs, FALSE); sort($params[$p]); } $money = Utils::read_params('money', $where_configs); $paramEval = isset($_GET['parameval']) && $_GET['parameval'] != '' ? $_GET['parameval'] : 'maps'; $minExecs = isset($_GET['minexecs']) ? $_GET['minexecs'] : -1; $minExecsFilter = ""; // FIXME PATCH FOR PARAM LIBRARIES WITHOUT LEGACY $where_configs = str_replace("AND .", "AND ", $where_configs); $where_configs = str_replace("`id_cluster`", "e.`id_cluster`", $where_configs); if ($minExecs > 0) { $minExecsFilter = "HAVING COUNT(*) > {$minExecs}"; } $filter_execs = DBUtils::getFilterExecs(); $options = Utils::getFilterOptions($db); $paramOptions = array(); foreach ($options[$paramEval] as $option) { if ($paramEval == 'id_cluster') { $paramOptions[] = $option['name']; } else { if ($paramEval == 'comp') { $paramOptions[] = Utils::getCompressionName($option[$paramEval]); } else { if ($paramEval == 'net') { $paramOptions[] = Utils::getNetworkName($option[$paramEval]); } else { if ($paramEval == 'disk') { $paramOptions[] = Utils::getDisksName($option[$paramEval]); } else { $paramOptions[] = $option[$paramEval]; } } } } } $benchOptions = $db->get_rows("SELECT DISTINCT bench FROM execs e LEFT JOIN clusters c ON e.id_cluster = c.id_cluster WHERE 1 {$filter_execs} {$where_configs} GROUP BY {$paramEval}, bench order by {$paramEval}"); // get the result rows $query = "SELECT count(*) as count, {$paramEval}, e.id_exec, exec as conf, bench, " . "exe_time, avg(exe_time) avg_exe_time, min(exe_time) min_exe_time " . "from execs e LEFT JOIN clusters c ON e.id_cluster = c.id_cluster WHERE 1 {$filter_execs} {$where_configs}" . "GROUP BY {$paramEval}, bench {$minExecsFilter} order by bench,{$paramEval}"; $rows = $db->get_rows($query); if (!$rows) { throw new \Exception("No results for query!"); } $arrayBenchs = array(); foreach ($paramOptions as $param) { foreach ($benchOptions as $bench) { $arrayBenchs[$bench['bench']][$param] = null; $arrayBenchs[$bench['bench']][$param]['y'] = 0; $arrayBenchs[$bench['bench']][$param]['count'] = 0; } } $series = array(); $bench = ''; foreach ($rows as $row) { if ($paramEval == 'comp') { $row[$paramEval] = Utils::getCompressionName($row['comp']); } else { if ($paramEval == 'id_cluster') { $row[$paramEval] = Utils::getClusterName($row[$paramEval], $db); } else { if ($paramEval == 'net') { $row[$paramEval] = Utils::getNetworkName($row['net']); } else { if ($paramEval == 'disk') { $row[$paramEval] = Utils::getDisksName($row['disk']); } else { if ($paramEval == 'iofilebuf') { $row[$paramEval] /= 1024; } } } } } $arrayBenchs[$row['bench']][$row[$paramEval]]['y'] = round((int) $row['avg_exe_time'], 2); $arrayBenchs[$row['bench']][$row[$paramEval]]['count'] = (int) $row['count']; } // ---------------------------------------------------- // Add predictions to the series // ---------------------------------------------------- $jsonData = $jsonHeader = "[]"; $instance = ""; $arrayBenchs_pred = array(); // FIXME PATCH FOR PARAM LIBRARIES WITHOUT LEGACY $where_configs = str_replace("AND .", "AND ", $where_configs); // compose instance $instance = MLUtils::generateSimpleInstance($param_names, $params, true, $db); $model_info = MLUtils::generateModelInfo($param_names, $params, true, $db); $instances = MLUtils::generateInstances($param_names, $params, true, $db); // model for filling $possible_models = $possible_models_id = array(); MLUtils::findMatchingModels($model_info, $possible_models, $possible_models_id, $dbml); $current_model = ""; if (array_key_exists('current_model', $_GET) && in_array($_GET['current_model'], $possible_models_id)) { $current_model = $_GET['current_model']; } if (!empty($possible_models_id)) { if ($current_model == "") { $query = "SELECT AVG(ABS(exe_time - pred_time)) AS MAE, AVG(ABS(exe_time - pred_time)/exe_time) AS RAE, p.id_learner FROM predictions p, learners l WHERE l.id_learner = p.id_learner AND p.id_learner IN ('" . implode("','", $possible_models_id) . "') AND predict_code > 0 ORDER BY MAE LIMIT 1"; $result = $dbml->query($query); $row = $result->fetch(); $current_model = $row['id_learner']; } $config = $instance . '-' . $current_model . "-parameval"; $query_cache = "SELECT count(*) as total FROM trees WHERE id_learner = '" . $current_model . "' AND model = '" . $model_info . "'"; $is_cached_mysql = $dbml->query($query_cache); $tmp_result = $is_cached_mysql->fetch(); $is_cached = $tmp_result['total'] > 0; $ret_data = null; if (!$is_cached) { // Call to MLFindAttributes, to fetch data $_GET['pass'] = 2; $_GET['unseen'] = 1; $_GET['current_model'] = $current_model; $mlfa1 = new MLFindAttributesController(); $mlfa1->container = $this->container; $ret_data = $mlfa1->mlfindattributesAction(); if ($ret_data == 1) { $must_wait = "YES"; $jsonData = $jsonHeader = '[]'; } else { $is_cached_mysql = $dbml->query($query_cache); $tmp_result = $is_cached_mysql->fetch(); $is_cached = $tmp_result['total'] > 0; } } if ($is_cached) { $must_wait = 'NO'; $query = "SELECT count(*) as count, {$paramEval}, bench, exe_time, avg(pred_time) avg_pred_time, min(pred_time) min_pred_time " . "FROM predictions e WHERE e.id_learner = '" . $current_model . "' {$filter_execs} {$where_configs}" . "GROUP BY {$paramEval}, bench {$minExecsFilter} order by bench, {$paramEval}"; $result = $dbml->query($query); // Initialize array foreach ($paramOptions as $param) { foreach ($benchOptions as $bench) { $arrayBenchs_pred[$bench['bench'] . '_pred'][$param] = null; $arrayBenchs_pred[$bench['bench'] . '_pred'][$param]['y'] = 0; $arrayBenchs_pred[$bench['bench'] . '_pred'][$param]['count'] = 0; } } foreach ($result as $row) { $bench_n = $row['bench'] . '_pred'; $class = $row[$paramEval]; if ($paramEval == 'comp') { $value = Utils::getCompressionName($class); } else { if ($paramEval == 'id_cluster') { $value = Utils::getClusterName($class, $db); } else { if ($paramEval == 'net') { $value = Utils::getNetworkName($class); } else { if ($paramEval == 'disk') { $value = Utils::getDisksName($class); } else { if ($paramEval == 'iofilebuf') { $value = $class / 1024; } else { $value = $class; } } } } } if (!in_array($value, $paramOptions)) { $paramOptions[] = $value; foreach ($benchOptions as $bench) { $arrayBenchs_pred[$bench['bench'] . '_pred'][$value] = null; $arrayBenchs_pred[$bench['bench'] . '_pred'][$value]['y'] = 0; $arrayBenchs_pred[$bench['bench'] . '_pred'][$value]['count'] = 0; $arrayBenchs[$bench['bench']][$value] = null; $arrayBenchs[$bench['bench']][$value]['y'] = 0; $arrayBenchs[$bench['bench']][$value]['count'] = 0; } } $arrayBenchs_pred[$bench_n][$value]['y'] = (int) $row['avg_pred_time']; $arrayBenchs_pred[$bench_n][$value]['count'] = (int) $row['count']; } } } // ---------------------------------------------------- // END - Add predictions to the series // ---------------------------------------------------- asort($paramOptions); foreach ($arrayBenchs as $key => $arrayBench) { $caregories = ''; $data_a = null; $data_p = null; foreach ($paramOptions as $param) { if ($arrayBenchs[$key][$param]['count'] > 0 && empty($arrayBenchs_pred) || !empty($arrayBenchs_pred) && ($arrayBenchs_pred[$key . '_pred'][$param]['count'] > 0 || $arrayBenchs[$key][$param]['count'] > 0)) { $data_a[] = $arrayBenchs[$key][$param]; if (!empty($arrayBenchs_pred)) { $data_p[] = $arrayBenchs_pred[$key . '_pred'][$param]; } $categories = $categories . "'{$param} " . Utils::getParamevalUnit($paramEval) . "',"; // FIXME - Redundant n times performed... don't care now } } $series[] = array('name' => $key, 'data' => $data_a); if (!empty($arrayBenchs_pred)) { $series[] = array('name' => $key . '_pred', 'data' => $data_p); } } $series = json_encode($series); if (!empty($arrayBenchs_pred)) { $colors = "['#7cb5ec','#9cd5fc','#434348','#636368','#90ed7d','#b0fd9d','#f7a35c','#f7c37c','#8085e9','#a0a5f9','#f15c80','#f17ca0','#e4d354','#f4f374','#8085e8','#a0a5f8','#8d4653','#ad6673','#91e8e1','#b1f8f1']"; } else { $colors = "['#7cb5ec','#434348','#90ed7d','#f7a35c','#8085e9','#f15c80','#e4d354','#8085e8','#8d4653','#91e8e1']"; } } catch (\Exception $e) { $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n"); $series = $jsonHeader = $colors = '[]'; $instance = $current_model = ''; $possible_models = $possible_models_id = array(); $must_wait = 'NO'; } echo $this->container->getTwig()->render('mltemplate/mlconfigperf.html.twig', array('selected' => 'mlparameval', 'title' => 'Improvement of Hadoop Execution by SW and HW Configurations', 'categories' => $categories, 'series' => $series, 'benchs' => $params['benchs'], 'nets' => $params['nets'], 'disks' => $params['disks'], 'blk_sizes' => $params['blk_sizes'], 'comps' => $params['comps'], 'id_clusters' => $params['id_clusters'], 'mapss' => $params['mapss'], 'replications' => $params['replications'], 'iosfs' => $params['iosfs'], 'iofilebufs' => $params['iofilebufs'], 'datanodess' => $params['datanodess'], 'bench_types' => $params['bench_types'], 'vm_sizes' => $params['vm_sizes'], 'vm_coress' => $params['vm_coress'], 'vm_RAMs' => $params['vm_RAMs'], 'types' => $params['types'], 'money' => $money, 'paramEval' => $paramEval, 'instance' => $instance, 'models' => '<li>' . implode('</li><li>', $possible_models) . '</li>', 'models_id' => $possible_models_id, 'current_model' => $current_model, 'gammacolors' => $colors, 'must_wait' => $must_wait, 'preset' => $preset, 'selPreset' => $selPreset, 'options' => Utils::getFilterOptions($db))); }
public function mlcrossvar3dfaAction() { $jsonData = array(); $message = $instance = $possible_models_id = ''; $maxx = $minx = $maxy = $miny = $maxz = $minz = 0; $must_wait = 'NO'; try { $dbml = new \PDO($this->container->get('config')['db_conn_chain_ml'], $this->container->get('config')['mysql_user'], $this->container->get('config')['mysql_pwd']); $dbml->setAttribute(\PDO::ATTR_ERRMODE, \PDO::ERRMODE_EXCEPTION); $dbml->setAttribute(\PDO::ATTR_EMULATE_PREPARES, false); $db = $this->container->getDBUtils(); $where_configs = ''; $preset = null; if (count($_GET) <= 1 || count($_GET) == 2 && array_key_exists('current_model', $_GET) || count($_GET) == 3 && array_key_exists('variable1', $_GET) && array_key_exists('variable2', $_GET) || count($_GET) == 4 && array_key_exists('current_model', $_GET) && array_key_exists('variable1', $_GET) && array_key_exists('variable2', $_GET)) { $preset = Utils::setDefaultPreset($db, 'mlcrossvar3dfa'); } $selPreset = isset($_GET['presets']) ? $_GET['presets'] : "none"; $params = array(); $param_names = array('benchs', 'nets', 'disks', 'mapss', 'iosfs', 'replications', 'iofilebufs', 'comps', 'blk_sizes', 'id_clusters', 'datanodess', 'bench_types', 'vm_sizes', 'vm_coress', 'vm_RAMs', 'types'); // Order is important foreach ($param_names as $p) { $params[$p] = Utils::read_params($p, $where_configs, FALSE); sort($params[$p]); } $cross_var1 = array_key_exists('variable1', $_GET) ? $_GET['variable1'] : 'maps'; $cross_var2 = array_key_exists('variable2', $_GET) ? $_GET['variable2'] : 'net'; $unseen = array_key_exists('unseen', $_GET) && $_GET['unseen'] == 1; $where_configs = str_replace("AND .", "AND ", $where_configs); $cross_var1 = str_replace("id_cluster", "e.id_cluster", $cross_var1); $cross_var2 = str_replace("id_cluster", "e.id_cluster", $cross_var2); // compose instance $instance = MLUtils::generateSimpleInstance($param_names, $params, $unseen, $db); $model_info = MLUtils::generateModelInfo($param_names, $params, $unseen, $db); // Model for filling MLUtils::findMatchingModels($model_info, $possible_models, $possible_models_id, $dbml); $current_model = ""; if (array_key_exists('current_model', $_GET) && in_array($_GET['current_model'], $possible_models_id)) { $current_model = $_GET['current_model']; } // Call to MLFindAttributes, to fetch data $_GET['pass'] = 1; $_GET['unseen'] = $unseen; $mlfa1 = new MLFindAttributesController(); $mlfa1->container = $this->container; $ret_data = $mlfa1->mlfindattributesAction(); $rows = null; if ($ret_data == 1) { $must_wait = "YES"; $jsonData = '[]'; $categories1 = $categories2 = "''"; } else { if ($ret_data == -1) { $must_wait = "NO"; $jsonData = '[]'; $categories1 = $categories2 = "''"; $message = "There are no prediction models trained for such parameters. Train at least one model in 'ML Prediction' section. [" . $instance . "]"; } else { // Get stuff from the DB $query = "SELECT " . $cross_var1 . " AS V1, " . $cross_var2 . " AS V2, AVG(p.pred_time) as V3, p.instance\n\t\t\t\t\tFROM predictions as p\n\t\t\t\t\tWHERE p.id_learner " . ($current_model != '' ? "='" . $current_model . "'" : "IN (SELECT id_learner FROM trees WHERE model='" . $model_info . "')") . $where_configs . "\n\t\t\t\t\tGROUP BY p.instance\n\t\t\t\t\tORDER BY RAND() LIMIT 5000;"; // FIXME - CLUMPSY PATCH FOR BYPASS THE BUG FROM HIGHCHARTS... REMEMBER TO ERASE THIS LINE WHEN THE BUG IS SOLVED $rows = $dbml->query($query); if (empty($rows)) { throw new \Exception('No data matches with your critteria.'); } } } if ($must_wait == "NO") { $map_var1 = $map_var2 = array(); $count_var1 = $count_var2 = 0; $categories1 = $categories2 = ''; $var1_categorical = in_array($cross_var1, array("net", "disk", "bench", "vm_OS", "provider", "vm_size", "type", "bench_type")); $var2_categorical = in_array($cross_var2, array("net", "disk", "bench", "vm_OS", "provider", "vm_size", "type", "bench_type")); foreach ($rows as $row) { $entry = array(); if ($var1_categorical) { if (!array_key_exists($row['V1'], $map_var1)) { $map_var1[$row['V1']] = $count_var1++; $categories1 = $categories1 . ($categories1 != '' ? "," : "") . "\"" . $row['V1'] . "\""; } $entry['y'] = $map_var1[$row['V1']] * (rand(990, 1010) / 1000); } else { $entry['y'] = (int) $row['V1'] * (rand(990, 1010) / 1000); } if ($entry['y'] > $maxy) { $maxy = $entry['y']; } if ($entry['y'] < $miny) { $miny = $entry['y']; } if ($var2_categorical) { if (!array_key_exists($row['V2'], $map_var2)) { $map_var2[$row['V2']] = $count_var2++; $categories2 = $categories2 . ($categories2 != '' ? "," : "") . "\"" . $row['V2'] . "\""; } $entry['x'] = $map_var2[$row['V2']] * (rand(990, 1010) / 1000); } else { $entry['x'] = (int) $row['V2'] * (rand(990, 1010) / 1000); } if ($entry['x'] > $maxx) { $maxx = $entry['x']; } if ($entry['x'] < $minx) { $minx = $entry['x']; } $entry['z'] = -1 * (int) $row['V3'] * (rand(990, 1010) / 1000); if ($entry['z'] > $maxz) { $maxz = $entry['z']; } if ($entry['z'] < $minz) { $minz = $entry['z']; } $entry['name'] = $row['instance']; //$row['V1']." - ".$row['V2']." - ".max(100,(int)$row['V3']); $jsonData[] = $entry; } $jsonData = json_encode($jsonData); if ($categories1 != '') { $categories1 = "[" . $categories1 . "]"; } else { $categories1 = "''"; } if ($categories2 != '') { $categories2 = "[" . $categories2 . "]"; } else { $categories2 = "''"; } } $dbml = null; $cross_var1 = str_replace("e.id_cluster", "id_cluster", $cross_var1); $cross_var2 = str_replace("e.id_cluster", "id_cluster", $cross_var2); } catch (\Exception $e) { $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n"); $jsonData = '[]'; $cross_var1 = $cross_var2 = ''; $categories1 = $categories2 = ''; $maxx = $minx = $maxy = $miny = $maxz = $minz = 0; $must_wait = "NO"; $dbml = null; $possible_models = $possible_models_id = array(); } echo $this->container->getTwig()->render('mltemplate/mlcrossvar3dfa.html.twig', array('selected' => 'mlcrossvar3dfa', 'jsonData' => $jsonData, 'variable1' => $cross_var1, 'variable2' => $cross_var2, 'categories1' => $categories1, 'categories2' => $categories2, 'maxx' => $maxx, 'minx' => $minx, 'maxy' => $maxy, 'miny' => $miny, 'maxz' => $maxz, 'minz' => $minz, 'benchs' => $params['benchs'], 'nets' => $params['nets'], 'disks' => $params['disks'], 'blk_sizes' => $params['blk_sizes'], 'comps' => $params['comps'], 'id_clusters' => $params['id_clusters'], 'mapss' => $params['mapss'], 'replications' => $params['replications'], 'iosfs' => $params['iosfs'], 'iofilebufs' => $params['iofilebufs'], 'datanodess' => $params['datanodess'], 'bench_types' => $params['bench_types'], 'vm_sizes' => $params['vm_sizes'], 'vm_coress' => $params['vm_coress'], 'vm_RAMs' => $params['vm_RAMs'], 'types' => $params['types'], 'message' => $message, 'instance' => $instance, 'model_info' => $model_info, 'current_model' => $current_model, 'unseen' => $unseen, 'models' => '<li>' . implode('</li><li>', $possible_models) . '</li>', 'models_id' => $possible_models_id, 'must_wait' => $must_wait, 'preset' => $preset, 'selPreset' => $selPreset, 'options' => Utils::getFilterOptions($db))); }
public function hdp2CountersAction() { try { $db = $this->container->getDBUtils(); $this->buildFilters(array('bench' => array('default' => null))); $whereClause = $this->filters->getWhereClause(); $benchOptions = $db->get_rows("SELECT DISTINCT bench FROM aloja2.execs e JOIN aloja2.HDI_JOB_details USING (id_exec) WHERE valid = 1"); $discreteOptions = array(); $discreteOptions['bench'][] = 'All'; foreach ($benchOptions as $option) { $discreteOptions['bench'][] = array_shift($option); } $dbUtil = $this->container->getDBUtils(); $message = null; //check the URL $execs = Utils::get_GET_intArray('execs'); if (Utils::get_GET_string('pageTab')) { $type = Utils::get_GET_string('pageTab'); } else { $type = 'SUMMARY'; } $join = "JOIN aloja2.execs e using (id_exec) JOIN aloja2.clusters USING (id_cluster) LEFT JOIN aloja_ml.predictions p USING (id_exec) WHERE job_name NOT IN\n ('TeraGen', 'random-text-writer', 'mahout-examples-0.7-job.jar', 'Create pagerank nodes', 'Create pagerank links') {$whereClause}" . ($execs ? ' AND id_exec IN (' . join(',', $execs) . ') ' : ''); if (isset($_GET['jobid'])) { $join .= " AND JOB_ID = '" . Utils::get_GET_string('jobid') . "' "; } $join .= " LIMIT 10000"; $query = ""; if ($type == 'SUMMARY') { $query = "SELECT e.bench, e.exe_time, c.id_exec, c.JOB_ID, c.job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME, c.TOTAL_MAPS, c.FAILED_MAPS, c.FINISHED_MAPS, c.TOTAL_REDUCES, c.FAILED_REDUCES, c.job_name as CHARTS\n \t\t\tFROM aloja2.HDI_JOB_details c {$join}"; } else { if ($type == "MAP") { $query = "SELECT e.bench, e.exe_time, c.id_exec, JOB_ID, job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME, c.TOTAL_MAPS, c.FAILED_MAPS, c.FINISHED_MAPS, `TOTAL_LAUNCHED_MAPS`,\n \t\t\t`RACK_LOCAL_MAPS`,\n \t\t\t`SPILLED_RECORDS`,\n \t\t\t`MAP_INPUT_RECORDS`,\n \t\t\t`MAP_OUTPUT_RECORDS`,\n \t\t\t`MAP_OUTPUT_BYTES`,\n \t\t\t`MAP_OUTPUT_MATERIALIZED_BYTES`\n \t\t\tFROM aloja2.HDI_JOB_details c {$join}"; } else { if ($type == 'REDUCE') { $query = "SELECT e.bench, e.exe_time, c.id_exec, c.JOB_ID, c.job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME, c.TOTAL_REDUCES, c.FAILED_REDUCES,\n \t\t\t`TOTAL_LAUNCHED_REDUCES`,\n \t\t\t`REDUCE_INPUT_GROUPS`,\n \t\t\t`REDUCE_INPUT_RECORDS`,\n \t\t\t`REDUCE_OUTPUT_RECORDS`,\n \t\t\t`REDUCE_SHUFFLE_BYTES`,\n \t\t\t`COMBINE_INPUT_RECORDS`,\n \t\t\t`COMBINE_OUTPUT_RECORDS`\n \t\t\tFROM aloja2.HDI_JOB_details c {$join}"; } else { if ($type == 'FILE-IO') { $query = "SELECT e.bench, e.exe_time, c.id_exec, c.JOB_ID, c.job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME,\n \t\t\t`SLOTS_MILLIS_MAPS`,\n \t\t\t`SLOTS_MILLIS_REDUCES`,\n \t\t\t`SPLIT_RAW_BYTES`,\n \t\t\t`FILE_BYTES_WRITTEN`,\n \t\t\t`FILE_BYTES_READ`,\n \t\t\t`WASB_BYTES_WRITTEN`,\n \t\t\t`WASB_BYTES_READ`,\n \t\t\t`BYTES_READ`,\n \t\t\t`BYTES_WRITTEN`\n \t\t\tFROM aloja2.HDI_JOB_details c {$join}"; } else { if ($type == 'DETAIL') { $query = "SELECT e.bench, e.exe_time, c.* FROM aloja_logs.aloja2 c {$join}"; } else { if ($type == "TASKS") { $query = "SELECT e.bench, e.exe_time, j.job_name, c.* FROM aloja_logs.HDI_JOB_tasks c\n \t\t\tJOIN aloja2.HDI_JOB_details j USING(id_exec,JOB_ID) {$join} "; // $taskStatusOptions = $db->get_rows("SELECT DISTINCT TASK_STATUS FROM HDI_JOB_tasks JOIN aloja2.execs USING (id_exec) WHERE valid = 1"); // $typeOptions = $db->get_rows("SELECT DISTINCT TASK_TYPE FROM HDI_JOB_tasks JOIN aloja2.execs USING (id_exec) WHERE valid = 1"); // // $discreteOptions['TASK_STATUS'][] = 'All'; // $discreteOptions['TASK_TYPE'][] = 'All'; // foreach($taskStatusOptions as $option) { // $discreteOptions['TASK_STATUS'][] = array_shift($option); // } // foreach($typeOptions as $option) { // $discreteOptions['TASK_TYPE'][] = array_shift($option); // } $discreteOptions['TASK_STATUS'] = array('All', 'SUCCEEDED'); $discreteOptions['TASK_TYPE'] = array('All', 'MAP', 'REDUCE'); } else { throw new \Exception('Unknown type!'); } } } } } } $exec_rows = $dbUtil->get_rows($query); if (count($exec_rows) > 0) { $show_in_result_counters = array('id_exec' => 'ID', 'JOB_ID' => 'JOBID', 'bench' => 'Bench', 'job_name' => 'JOBNAME'); $show_in_result_counters = Utils::generate_show($show_in_result_counters, $exec_rows, 4); } } catch (\Exception $e) { $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n"); } return $this->render('repositoryViews/hdp2counters.html.twig', array('theaders' => isset($show_in_result_counters) ? $show_in_result_counters : array(), 'message' => $message, 'title' => 'Hadoop Jobs and Tasks Execution Counters', 'type' => $type, 'execs' => $execs, 'execsParam' => isset($_GET['execs']) ? Utils::get_GET_intArray('execs') : '', 'discreteOptions' => $discreteOptions, 'hdp2' => true)); }
public function paramEvaluationAction() { $db = $this->container->getDBUtils(); $this->buildFilters(array('minexecs' => array('default' => null, 'type' => 'inputNumber', 'label' => 'Minimum executions:', 'parseFunction' => function () { return 0; }, 'filterGroup' => 'basic'))); $whereClause = $this->filters->getWhereClause(); $model_html = ''; $model_info = $db->get_rows("SELECT id_learner, model, algorithm, dataslice FROM aloja_ml.learners"); foreach ($model_info as $row) { $model_html = $model_html . "<li><b>" . $row['id_learner'] . "</b> => " . $row['algorithm'] . " : " . $row['model'] . " : " . $row['dataslice'] . "</li>"; } $categories = ''; $series = ''; try { $paramEval = isset($_GET['parameval']) && Utils::get_GET_string('parameval') != '' ? Utils::get_GET_string('parameval') : 'maps'; $minExecs = isset($_GET['minexecs']) ? Utils::get_GET_int('minexecs') : -1; $this->filters->changeCurrentChoice('minexecs', $minExecs == -1 ? null : $minExecs); $shortAliasParamEval = array('maps' => 'e', 'comp' => 'e', 'id_cluster' => 'c', 'net' => 'e', 'disk' => 'e', 'replication' => 'e', 'iofilebuf' => 'e', 'blk_size' => 'e', 'iosf' => 'e', 'vm_size' => 'c', 'vm_cores' => 'c', 'vm_ram' => 'c', 'datanodes' => 'c', 'hadoop_version' => 'e', 'type' => 'c'); $minExecsFilter = ""; if ($minExecs > 0) { $minExecsFilter = "HAVING COUNT(*) > {$minExecs}"; } $filter_execs = DBUtils::getFilterExecs(); $options = $this->filters->getFiltersArray()[$paramEval]['choices']; $benchOptions = "SELECT DISTINCT e.bench FROM aloja2.execs e JOIN aloja2.clusters c USING (id_cluster) LEFT JOIN aloja_ml.predictions p USING (id_exec) WHERE 1 {$filter_execs} {$whereClause} GROUP BY {$shortAliasParamEval[$paramEval]}.{$paramEval}, e.bench order by {$shortAliasParamEval[$paramEval]}.{$paramEval}"; $params = $this->filters->getFiltersSelectedChoices(array('prediction_model', 'upred', 'uobsr')); $whereClauseML = str_replace("exe_time", "pred_time", $whereClause); $whereClauseML = str_replace("start_time", "creation_time", $whereClauseML); $query = "SELECT COUNT(*) AS count, {$shortAliasParamEval[$paramEval]}.{$paramEval}, e.bench, avg(e.exe_time) avg_exe_time, min(e.exe_time) min_exe_time\n\t\t\t\t\t FROM aloja2.execs AS e JOIN aloja2.clusters AS c USING (id_cluster)\n\t\t\t\t\t LEFT JOIN aloja_ml.predictions AS p USING (id_exec)\n\t\t\t\t\t WHERE 1 {$filter_execs} {$whereClause}\n\t\t\t\t\t GROUP BY {$shortAliasParamEval[$paramEval]}.{$paramEval}, e.bench {$minExecsFilter} ORDER BY e.bench, {$shortAliasParamEval[$paramEval]}.{$paramEval}"; $queryPredictions = "\n\t\t\t\t\tSELECT COUNT(*) AS count, {$shortAliasParamEval[$paramEval]}.{$paramEval}, CONCAT('pred_',e.bench) as bench,\n\t\t\t\t\t\tavg(e.pred_time) as avg_exe_time, min(e.pred_time) as min_exe_time\n\t\t\t\t\t\tFROM aloja_ml.predictions AS e\n\t\t\t\t\t\tJOIN clusters c USING (id_cluster)\n\t\t\t\t\t\tWHERE 1 {$filter_execs} " . str_replace("p.", "e.", $whereClauseML) . " AND e.id_learner = '" . $params['prediction_model'] . "'\n\t\t\t\t\t\tGROUP BY {$shortAliasParamEval[$paramEval]}.{$paramEval}, e.bench {$minExecsFilter} ORDER BY e.bench, {$shortAliasParamEval[$paramEval]}.{$paramEval}"; // get the result rows if ($params['uobsr'] == 1 && $params['upred'] == 1) { $query = "({$query}) UNION ({$queryPredictions})"; $benchOptions = "SELECT DISTINCT e.bench FROM aloja2.execs e JOIN aloja2.clusters c USING (id_cluster) LEFT JOIN aloja_ml.predictions p USING (id_exec) WHERE 1 {$filter_execs} {$whereClause} GROUP BY {$shortAliasParamEval[$paramEval]}.{$paramEval}, e.bench\n\t\t\t\t\t\t\t\t UNION\n\t\t\t\t\t\t\t\t (SELECT DISTINCT CONCAT('pred_', e.bench) as bench FROM aloja_ml.predictions AS e\n\t\t\t\t\t\t\t\t JOIN clusters c USING (id_cluster)\n\t\t\t\t\t\t\t\t WHERE 1 {$filter_execs} " . str_replace("p.", "e.", $whereClauseML) . " AND e.id_learner = '" . $params['prediction_model'] . "'\n\t\t\t\t\t\t\t\t GROUP BY {$shortAliasParamEval[$paramEval]}.{$paramEval}, e.bench {$minExecsFilter})\n\t\t\t\t\t\t\t\t ORDER BY bench"; $optionsPredictions = "SELECT DISTINCT {$shortAliasParamEval[$paramEval]}.{$paramEval} FROM aloja_ml.predictions AS e JOIN clusters c USING (id_cluster) WHERE 1 {$filter_execs} " . str_replace("p.", "e.", $whereClauseML) . " AND e.id_learner = '" . $params['prediction_model'] . "' ORDER BY {$shortAliasParamEval[$paramEval]}.{$paramEval}"; $optionsPredictions = $db->get_rows($optionsPredictions); foreach ($optionsPredictions as $predOption) { $options[] = $predOption[$paramEval]; } } else { if ($params['uobsr'] == 0 && $params['upred'] == 1) { $query = $queryPredictions; $benchOptions = "SELECT DISTINCT CONCAT('pred_', e.bench) as bench FROM aloja_ml.predictions AS e\n \t\t\t\t\t\t\t\t JOIN clusters c USING (id_cluster)\n\t\t\t\t\t\t\t\t WHERE 1 {$filter_execs} " . str_replace("p.", "e.", $whereClauseML) . " AND e.id_learner = '" . $params['prediction_model'] . "'\n\t\t\t\t\t\t\t\t GROUP BY {$shortAliasParamEval[$paramEval]}.{$paramEval}, e.bench {$minExecsFilter} ORDER BY e.bench, {$shortAliasParamEval[$paramEval]}.{$paramEval}"; $options = array(); $optionsPredictions = "SELECT DISTINCT {$shortAliasParamEval[$paramEval]}.{$paramEval} FROM aloja_ml.predictions AS e JOIN clusters c USING (id_cluster) WHERE 1 {$filter_execs} " . str_replace("p.", "e.", $whereClauseML) . " AND e.id_learner = '" . $params['prediction_model'] . "' ORDER BY {$shortAliasParamEval[$paramEval]}.{$paramEval}"; $optionsPredictions = $db->get_rows($optionsPredictions); foreach ($optionsPredictions as $predOption) { $options[] = $predOption[$paramEval]; } } else { if ($params['uobsr'] == 0 && $params['upred'] == 0) { $this->container->getTwig()->addGlobal('message', "Warning: No data selected (Predictions|Observations) from the ML Filters. Adding the Observed executions to the figure by default.\n"); } } } $rows = $db->get_rows($query); $benchOptions = $db->get_rows($benchOptions); if (!$rows) { throw new \Exception("No results for query!"); } $paramOptions = array(); foreach ($options as $option) { if ($paramEval == 'id_cluster') { $paramOptions[] = Utils::getClusterName($option, $db); } else { if ($paramEval == 'comp') { $paramOptions[] = Utils::getCompressionName($option); } else { if ($paramEval == 'net') { $paramOptions[] = Utils::getNetworkName($option); } else { if ($paramEval == 'disk') { $paramOptions[] = Utils::getDisksName($option); } else { if ($paramEval == 'vm_ram') { $paramOptions[] = Utils::getBeautyRam($option); } else { $paramOptions[] = $option; } } } } } } $categories = ''; $arrayBenchs = array(); foreach ($paramOptions as $param) { $categories .= "'{$param}" . Utils::getParamevalUnit($paramEval) . "',"; foreach ($benchOptions as $bench) { $arrayBenchs[$bench['bench']][$param] = null; } } $series = array(); foreach ($rows as $row) { if ($paramEval == 'comp') { $row[$paramEval] = Utils::getCompressionName($row['comp']); } else { if ($paramEval == 'id_cluster') { $row[$paramEval] = Utils::getClusterName($row[$paramEval], $db); } else { if ($paramEval == 'net') { $row[$paramEval] = Utils::getNetworkName($row['net']); } else { if ($paramEval == 'disk') { $row[$paramEval] = Utils::getDisksName($row['disk']); } else { if ($paramEval == 'vm_ram') { $row[$paramEval] = Utils::getBeautyRam($row['vm_ram']); } } } } } $arrayBenchs[strtolower($row['bench'])][$row[$paramEval]]['y'] = round((int) $row['avg_exe_time'], 2); $arrayBenchs[strtolower($row['bench'])][$row[$paramEval]]['count'] = (int) $row['count']; } foreach ($arrayBenchs as $key => $arrayBench) { $series[] = array('name' => $key, 'data' => array_values($arrayBench)); } $series = json_encode($series); } catch (\Exception $e) { $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n"); } return $this->render('configEvaluationViews/parameval.html.twig', array('title' => 'Improvement of Hadoop Execution by SW and HW Configurations', 'minexecs' => $minExecs, 'categories' => $categories, 'series' => $series, 'paramEval' => $paramEval, 'models' => $model_html)); }
public function mlfindattributesAction() { $instance = $instances = $message = $tree_descriptor = $model_html = $config = ''; $possible_models = $possible_models_id = $other_models = array(); $must_wait = 'NO'; try { $dbml = new \PDO($this->container->get('config')['db_conn_chain'], $this->container->get('config')['mysql_user'], $this->container->get('config')['mysql_pwd']); $dbml->setAttribute(\PDO::ATTR_ERRMODE, \PDO::ERRMODE_EXCEPTION); $dbml->setAttribute(\PDO::ATTR_EMULATE_PREPARES, false); $db = $this->container->getDBUtils(); if (array_key_exists('dump', $_GET)) { $dump = $_GET["dump"]; unset($_GET["dump"]); } if (array_key_exists('pass', $_GET)) { $pass = $_GET["pass"]; unset($_GET["pass"]); } $this->buildFilters(array('current_model' => array('type' => 'selectOne', 'default' => null, 'label' => 'Model to use: ', 'generateChoices' => function () { return array(); }, 'parseFunction' => function () { $choice = isset($_GET['current_model']) ? $_GET['current_model'] : array(""); return array('whereClause' => '', 'currentChoice' => $choice); }, 'filterGroup' => 'MLearning'), 'unseen' => array('type' => 'checkbox', 'default' => 1, 'label' => 'Predict with unseen atributes ⚠', 'parseFunction' => function () { $choice = isset($_GET['unseen']) && !isset($_GET['unseen']) ? 0 : 1; return array('whereClause' => '', 'currentChoice' => $choice); }, 'filterGroup' => 'MLearning'), 'minexetime' => array('default' => 0), 'valid' => array('default' => 0), 'filter' => array('default' => 0), 'prepares' => array('default' => 1))); $this->buildFilterGroups(array('MLearning' => array('label' => 'Machine Learning', 'tabOpenDefault' => true, 'filters' => array('current_model', 'unseen')))); $where_configs = $this->filters->getWhereClause(); $param_names = array('bench', 'net', 'disk', 'maps', 'iosf', 'replication', 'iofilebuf', 'comp', 'blk_size', 'id_cluster', 'datanodes', 'vm_OS', 'vm_cores', 'vm_RAM', 'provider', 'vm_size', 'type', 'bench_type', 'hadoop_version'); // Order is important $params = $this->filters->getFiltersSelectedChoices($param_names); foreach ($param_names as $p) { if (!is_null($params[$p]) && is_array($params[$p])) { sort($params[$p]); } } $learnParams = $this->filters->getFiltersSelectedChoices(array('current_model', 'unseen')); $param_current_model = $learnParams['current_model']; $unseen = $learnParams['unseen'] ? true : false; // FIXME PATCH FOR PARAM LIBRARIES WITHOUT LEGACY $where_configs = str_replace("AND .", "AND ", $where_configs); $jsonData = $jsonHeader = "[]"; $mae = $rae = 0; // compose instance $model_info = MLUtils::generateModelInfo($this->filters, $param_names, $params, $unseen); $instance = MLUtils::generateSimpleInstance($this->filters, $param_names, $params, $unseen); $instances = MLUtils::generateInstances($this->filters, $param_names, $params, $unseen, $db); // Model for filling MLUtils::findMatchingModels($model_info, $possible_models, $possible_models_id, $dbml); $current_model = ''; if (!is_null($possible_models_id) && in_array($param_current_model, $possible_models_id)) { $current_model = $param_current_model; } // Other models for filling $where_models = ''; if (!empty($possible_models_id)) { $where_models = " WHERE id_learner NOT IN ('" . implode("','", $possible_models_id) . "')"; } $result = $dbml->query("SELECT id_learner FROM aloja_ml.learners" . $where_models); foreach ($result as $row) { $other_models[] = $row['id_learner']; } if (!empty($possible_models_id) || $current_model != "") { $result = $dbml->query("SELECT id_learner, model, algorithm, CASE WHEN `id_learner` IN ('" . implode("','", $possible_models_id) . "') THEN 'COMPATIBLE' ELSE 'NOT MATCHED' END AS compatible FROM aloja_ml.learners"); foreach ($result as $row) { $model_html = $model_html . "<li>" . $row['id_learner'] . " => " . $row['algorithm'] . " : " . $row['compatible'] . " : " . $row['model'] . "</li>"; } if ($current_model == "") { $query = "SELECT AVG(ABS(exe_time - pred_time)) AS MAE, AVG(ABS(exe_time - pred_time)/exe_time) AS RAE, p.id_learner FROM aloja_ml.predictions p, aloja_ml.learners l WHERE l.id_learner = p.id_learner AND p.id_learner IN ('" . implode("','", $possible_models_id) . "') AND predict_code > 0 ORDER BY MAE LIMIT 1"; $result = $dbml->query($query); $row = $result->fetch(); $current_model = $row['id_learner']; } $config = $instance . '-' . $current_model . '-' . ($unseen ? 'U' : 'R'); $is_cached_mysql = $dbml->query("SELECT count(*) as total FROM aloja_ml.trees WHERE id_findattrs = '" . md5($config) . "'"); $tmp_result = $is_cached_mysql->fetch(); $is_cached = $tmp_result['total'] > 0; $tmp_file = md5($config) . '.tmp'; $in_process = file_exists(getcwd() . '/cache/query/' . md5($config) . '.lock'); $finished_process = $in_process && (int) shell_exec('ls ' . getcwd() . '/cache/query/' . md5($config) . '-*.lock | wc -w ') == count($instances); if (!$in_process && !$finished_process && !$is_cached) { // Retrieve file model from DB $query = "SELECT file FROM aloja_ml.model_storage WHERE id_hash='" . $current_model . "' AND type='learner';"; $result = $dbml->query($query); $row = $result->fetch(); $content = $row['file']; $filemodel = getcwd() . '/cache/query/' . $current_model . '-object.rds'; $fp = fopen($filemodel, 'w'); fwrite($fp, $content); fclose($fp); // Run the predictor exec('cd ' . getcwd() . '/cache/query ; touch ' . md5($config) . '.lock ; rm -f ' . $tmp_file); $count = 1; foreach ($instances as $inst) { exec(getcwd() . '/resources/queue -d -c "cd ' . getcwd() . '/cache/query ; ../../resources/aloja_cli.r -m aloja_predict_instance -l ' . $current_model . ' -p inst_predict=\'' . $inst . '\' -v | grep -v \'Prediction\' >>' . $tmp_file . ' 2>/dev/null; touch ' . md5($config) . '-' . $count++ . '.lock" >/dev/null 2>&1 &'); } } $finished_process = (int) shell_exec('ls ' . getcwd() . '/cache/query/' . md5($config) . '-*.lock | wc -w ') == count($instances); if ($finished_process && !$is_cached) { // Read results and dump to DB $i = 0; $token = 0; $token_i = 0; $query = "INSERT IGNORE INTO aloja_ml.predictions (id_exec,exe_time,bench,net,disk,maps,iosf,replication,iofilebuf,comp,blk_size,id_cluster,datanodes,vm_OS,vm_cores,vm_RAM,provider,vm_size,type,bench_type,hadoop_version,pred_time,id_learner,instance,predict_code) VALUES "; if (($handle = fopen(getcwd() . '/cache/query/' . $tmp_file, "r")) !== FALSE) { while (($line = fgets($handle, 1000)) !== FALSE && $i < 1000) { if ($line == '') { break; } // Fetch Real Value $inst_aux = preg_split("/\\s+/", $line); $query_var = "SELECT AVG(exe_time) as AVG, id_exec, outlier FROM aloja_ml.predictions WHERE instance = '" . $inst_aux[1] . "' AND predict_code > 0"; $result = $dbml->query($query_var); $row = $result->fetch(); $realexecval = is_null($row['AVG']) || $row['outlier'] == 2 ? 0 : $row['AVG']; $realid_exec = is_null($row['id_exec']) || $row['outlier'] == 2 ? 0 : $row['id_exec']; $query_var = "SELECT count(*) as num FROM aloja_ml.predictions WHERE instance = '" . $inst_aux[1] . "' AND id_learner = '" . $current_model . "'"; $result = $dbml->query($query_var); $row = $result->fetch(); // Insert instance values if ($row['num'] == 0) { $token_i = 1; $selected_instance = preg_replace('/,Cmp(\\d+),/', ',${1},', $inst_aux[1]); $selected_instance = preg_replace('/,Cl(\\d+),/', ',${1},', $selected_instance); if ($token > 0) { $query = $query . ","; } $token = 1; $query = $query . "('" . $realid_exec . "','" . $realexecval . "','" . str_replace(",", "','", $selected_instance) . "','" . $inst_aux[2] . "','" . $current_model . "','" . $inst_aux[1] . "','0') "; } $i++; if ($i % 100 == 0 && $token_i > 0) { if ($dbml->query($query) === FALSE) { throw new \Exception('Error when saving into DB'); } $query = "INSERT IGNORE INTO aloja_ml.predictions (id_exec,exe_time,bench,net,disk,maps,iosf,replication,iofilebuf,comp,blk_size,id_cluster,datanodes,vm_OS,vm_cores,vm_RAM,provider,vm_size,type,bench_type,hadoop_version,pred_time,id_learner,instance,predict_code) VALUES "; $token = 0; $token_i = 0; } } if ($token_i > 0) { if ($dbml->query($query) === FALSE) { throw new \Exception('Error when saving into DB'); } } // Descriptive Tree $tree_descriptor = shell_exec(getcwd() . '/resources/aloja_cli.r -m aloja_representative_tree -p method=ordered:dump_file="' . getcwd() . '/cache/query/' . $tmp_file . '":output="html" -v 2> /dev/null'); $tree_descriptor = substr($tree_descriptor, 5, -2); $query = "INSERT INTO aloja_ml.trees (id_findattrs,id_learner,instance,model,tree_code) VALUES ('" . md5($config) . "','" . $current_model . "','" . $instance . "','" . $model_info . "','" . $tree_descriptor . "')"; if ($dbml->query($query) === FALSE) { throw new \Exception('Error when saving tree into DB'); } // remove remaining locks shell_exec('rm -f ' . getcwd() . '/cache/query/' . md5($config) . '*.lock'); // Remove temporal files $output = shell_exec('rm -f ' . getcwd() . '/cache/query/' . md5($config) . '.tmp'); $is_cached = true; } fclose($handle); } if (!$is_cached) { $jsonData = $jsonHeader = $jsonColumns = $jsonColor = '[]'; $must_wait = 'YES'; if (isset($dump)) { $dbml = null; echo "1"; exit(0); } if (isset($pass)) { $dbml = null; return "1"; } } else { if (isset($pass) && $pass == 2) { $dbml = null; return "2"; } // Fetch results and compose JSON $header = array('Benchmark', 'Net', 'Disk', 'Maps', 'IO.SFS', 'Rep', 'IO.FBuf', 'Comp', 'Blk.Size', 'Cluster', 'Datanodes', 'VM.OS', 'VM.Cores', 'VM.RAM', 'Provider', 'VM.Size', 'Type', 'Bench.Type', 'Version', 'Prediction', 'Observed'); $jsonHeader = '[{title:""}'; foreach ($header as $title) { $jsonHeader = $jsonHeader . ',{title:"' . $title . '"}'; } $jsonHeader = $jsonHeader . ']'; $query = "SELECT @i:=@i+1 as num, instance, AVG(pred_time) as pred_time, AVG(exe_time) as exe_time FROM aloja_ml.predictions, (SELECT @i:=0) d WHERE id_learner='" . $current_model . "' " . $where_configs . " GROUP BY instance"; $result = $dbml->query($query); $jsonData = '['; foreach ($result as $row) { if ($jsonData != '[') { $jsonData = $jsonData . ','; } $jsonData = $jsonData . "['" . $row['num'] . "','" . str_replace(",", "','", $row['instance']) . "','" . $row['pred_time'] . "','" . $row['exe_time'] . "']"; } $jsonData = $jsonData . ']'; foreach (range(1, 33) as $value) { $jsonData = str_replace('Cmp' . $value, Utils::getCompressionName($value), $jsonData); } // Fetch MAE & RAE values $query = "SELECT AVG(ABS(exe_time - pred_time)) AS MAE, AVG(ABS(exe_time - pred_time)/exe_time) AS RAE FROM aloja_ml.predictions WHERE id_learner='" . md5($config) . "' AND predict_code > 0"; $result = $dbml->query($query); $row = $result->fetch(); $mae = $row['MAE']; $rae = $row['RAE']; // Dump case if (isset($dump)) { echo "ID" . str_replace(array("[", "]", "{title:\"", "\"}"), array('', '', ''), $jsonHeader) . "\n"; echo str_replace(array('],[', '[[', ']]'), array("\n", '', ''), $jsonData); $dbml = null; exit(0); } if (isset($pass) && $pass == 1) { $retval = "ID" . str_replace(array("[", "]", "{title:\"", "\"}"), array('', '', ''), $jsonHeader) . "\n"; $retval .= str_replace(array('],[', '[[', ']]'), array("\n", '', ''), $jsonData); $dbml = null; return $retval; } // Display Descriptive Tree $query = "SELECT tree_code FROM aloja_ml.trees WHERE id_findattrs = '" . md5($config) . "'"; $result = $dbml->query($query); $row = $result->fetch(); $tree_descriptor = $row['tree_code']; } } else { $message = "There are no prediction models trained for such parameters. Train at least one model in 'ML Prediction' section."; $must_wait = 'NO'; if (isset($dump)) { echo "-1"; exit(0); } if (isset($pass)) { return "-1"; } } $dbml = null; } catch (\Exception $e) { $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n"); $jsonData = $jsonHeader = "[]"; $must_wait = 'NO'; $mae = $rae = 0; $dbml = null; if (isset($pass)) { return "-2"; } } $return_params = array('instance' => $instance, 'jsonData' => $jsonData, 'jsonHeader' => $jsonHeader, 'models' => $model_html, 'models_id' => $possible_models_id, 'other_models_id' => $other_models, 'current_model' => $current_model, 'message' => $message, 'mae' => $mae, 'rae' => $rae, 'must_wait' => $must_wait, 'instance' => $instance, 'instances' => implode("<br/> ", $instances), 'model_info' => $model_info, 'id_findattr' => md5($config), 'unseen' => $unseen, 'tree_descriptor' => $tree_descriptor); $this->filters->setCurrentChoices('current_model', array_merge($possible_models_id, array('---Other models---'), $other_models)); return $this->render('mltemplate/mlfindattributes.html.twig', $return_params); }
public function mlobservedtreesAction() { $model_info = $instance = $slice_info = $message = $config = $tree_descriptor_ordered = $tree_descriptor_gini = ''; $jsonData = $jsonHeader = '[]'; $jsonObstrees = $jsonObstreesHeader = '[]'; $must_wait = 'NO'; try { $dbml = new \PDO($this->container->get('config')['db_conn_chain'], $this->container->get('config')['mysql_user'], $this->container->get('config')['mysql_pwd']); $dbml->setAttribute(\PDO::ATTR_ERRMODE, \PDO::ERRMODE_EXCEPTION); $dbml->setAttribute(\PDO::ATTR_EMULATE_PREPARES, false); $db = $this->container->getDBUtils(); // FIXME - This must be counted BEFORE building filters, as filters inject rubbish in GET when there are no parameters... $instructions = count($_GET) <= 1; $this->buildFilters(array('minexetime' => array('default' => 0), 'valid' => array('default' => 0), 'filter' => array('default' => 0), 'prepares' => array('default' => 1))); if ($instructions) { MLUtils::getIndexObsTrees($jsonObstrees, $jsonObstreesHeader, $dbml); return $this->render('mltemplate/mlobstrees.html.twig', array('obstrees' => $jsonObstrees, 'header_obstrees' => $jsonObstreesHeader, 'jsonData' => '[]', 'jsonHeader' => '[]', 'instructions' => 'YES')); } $param_names = array('bench', 'net', 'disk', 'maps', 'iosf', 'replication', 'iofilebuf', 'comp', 'blk_size', 'id_cluster', 'datanodes', 'vm_OS', 'vm_cores', 'vm_RAM', 'provider', 'vm_size', 'type', 'bench_type', 'hadoop_version', 'datasize', 'scale_factor'); // Order is important $params = $this->filters->getFiltersSelectedChoices($param_names); foreach ($param_names as $p) { if (!is_null($params[$p]) && is_array($params[$p])) { sort($params[$p]); } } $params_additional = array(); $param_names_additional = array('datefrom', 'dateto', 'minexetime', 'maxexetime', 'valid', 'filter'); // Order is important $params_additional = $this->filters->getFiltersSelectedChoices($param_names_additional); $where_configs = $this->filters->getWhereClause(); $where_configs = str_replace("AND .", "AND ", $where_configs); // compose instance $instance = MLUtils::generateSimpleInstance($this->filters, $param_names, $params, TRUE); $model_info = MLUtils::generateModelInfo($this->filters, $param_names, $params, TRUE); $slice_info = MLUtils::generateDatasliceInfo($this->filters, $param_names_additional, $params_additional); $config = $instance . '-' . $slice_info . '-obstree'; $is_cached_mysql = $dbml->query("SELECT count(*) as total FROM aloja_ml.observed_trees WHERE id_obstrees = '" . md5($config) . "'"); $tmp_result = $is_cached_mysql->fetch(); $is_cached = $tmp_result['total'] > 0; $in_process = file_exists(getcwd() . '/cache/ml/' . md5($config) . '.lock'); $finished_process = file_exists(getcwd() . '/cache/ml/' . md5($config) . '.fin'); $tmp_file = getcwd() . '/cache/ml/' . md5($config) . '.tmp'; // get headers for csv $header_names = array('bench' => 'Benchmark', 'net' => 'Net', 'disk' => 'Disk', 'maps' => 'Maps', 'iosf' => 'IO.SFac', 'replication' => 'Rep', 'iofilebuf' => 'IO.FBuf', 'comp' => 'Comp', 'blk_size' => 'Blk.size', 'e.id_cluster' => 'Cluster', 'datanodes' => 'Datanodes', 'vm_OS' => 'VM.OS', 'vm_cores' => 'VM.Cores', 'vm_RAM' => 'VM.RAM', 'provider' => 'Provider', 'vm_size' => 'VM.Size', 'type' => 'Type', 'bench_type' => 'Bench.Type', 'hadoop_version' => 'Hadoop.Version', 'IFNULL(datasize,0)' => 'Datasize', 'scale_factor' => 'Scale.Factor'); $special_header_names = array('id_exec' => 'ID', 'exe_time' => 'Exe.Time'); $headers = array_keys($header_names); $special_headers = array_keys($special_header_names); if (!$in_process && !$finished_process && !$is_cached) { // Dump the DB slice to csv $query = "SELECT " . implode(",", $headers) . ", " . implode(",", $special_headers) . " FROM aloja2.execs e LEFT JOIN aloja2.clusters c ON e.id_cluster = c.id_cluster WHERE hadoop_version IS NOT NULL" . $where_configs . ";"; $rows = $db->get_rows($query); if (empty($rows)) { throw new \Exception('No data matches with your critteria.'); } if (($key = array_search('e.id_cluster', $headers)) !== false) { $headers[$key] = 'id_cluster'; } $fp = fopen($tmp_file, 'w'); foreach ($rows as $row) { $row['id_cluster'] = "Cl" . $row['id_cluster']; // Cluster is numerically codified... $row['comp'] = "Cmp" . $row['comp']; // Compression is numerically codified... $line = ''; foreach ($headers as $hn) { $line = $line . ($line != '' ? ',' : '') . $row[$hn]; } $line = $row['id_exec'] . ' ' . $line . ' ' . $row['exe_time'] . "\n"; fputs($fp, $line); } fclose($fp); if (($key = array_search('id_cluster', $headers)) !== false) { $headers[$key] = 'e.id_cluster'; } // Execute R Engine $exe_query = 'cd ' . getcwd() . '/cache/ml;'; $exe_query = $exe_query . ' touch ' . md5($config) . '.lock;'; $exe_query = $exe_query . ' ../../resources/aloja_cli.r -m aloja_representative_tree -p method=ordered:dump_file=' . $tmp_file . ':output=nodejson -v >' . md5($config) . '-split.dat 2>/dev/null;'; $exe_query = $exe_query . ' ../../resources/aloja_cli.r -m aloja_representative_tree -p method=gini:dump_file=' . $tmp_file . ':output=nodejson -v >' . md5($config) . '-gini.dat 2>/dev/null;'; $exe_query = $exe_query . ' rm -f ' . md5($config) . '.lock; rm -f ' . $tmp_file . '; touch ' . md5($config) . '.fin'; exec(getcwd() . '/resources/queue -d -c "' . $exe_query . '" >/dev/null 2>&1 &'); } if (!$is_cached) { $finished_process = file_exists(getcwd() . '/cache/ml/' . md5($config) . '.fin'); if ($finished_process) { // Read results and dump to DB $tree_descriptor_ordered = ''; try { $file = fopen(getcwd() . '/cache/ml/' . md5($config) . '-split.dat', "r"); $tree_descriptor_ordered = fgets($file); $tree_descriptor_ordered = substr($tree_descriptor_ordered, 5, -2); $tree_descriptor_ordered = str_replace("\\\"", "\"", $tree_descriptor_ordered); $tree_descriptor_ordered = str_replace("desc:\"\"", "desc:\"---\"", $tree_descriptor_ordered); fclose($file); } catch (\Exception $e) { throw new \Exception("Error on retrieving result file. Check that R is working properly."); } $tree_descriptor_gini = ''; /* try { $file = fopen(getcwd().'/cache/ml/'.md5($config).'-gini.dat', "r"); $tree_descriptor_gini = fgets($file); $tree_descriptor_gini = substr($tree_descriptor_gini, 5, -2); $tree_descriptor_gini = str_replace("\\\"","\"",$tree_descriptor_gini); fclose($file); } catch (\Exception $e) { throw new \Exception ("Error on retrieving result file. Check that R is working properly."); } */ $query = "INSERT INTO aloja_ml.observed_trees (id_obstrees,instance,model,dataslice,tree_code_split,tree_code_gain) VALUES ('" . md5($config) . "','" . $instance . "','" . $model_info . "','" . $slice_info . "','" . $tree_descriptor_ordered . "','" . $tree_descriptor_gini . "')"; if ($dbml->query($query) === FALSE) { throw new \Exception('Error when saving tree into DB'); } // Remove temporal files $output = shell_exec('rm -f ' . getcwd() . '/cache/ml/' . md5($config) . '-*.dat'); $output = shell_exec('rm -f ' . getcwd() . '/cache/ml/' . md5($config) . '.fin'); } else { $must_wait = 'YES'; throw new \Exception('WAIT'); } } // Fetch results and compose JSON $header = array('Benchmark', 'Net', 'Disk', 'Maps', 'IO.SFS', 'Rep', 'IO.FBuf', 'Comp', 'Blk.Size', 'Cluster', 'Datanodes', 'VM.OS', 'VM.Cores', 'VM.RAM', 'Provider', 'VM.Size', 'Type', 'Bench.Type', 'Version', 'Datasize', 'Scale.Factor', 'Observed'); $jsonHeader = '[{title:""}'; foreach ($header as $title) { $jsonHeader = $jsonHeader . ',{title:"' . $title . '"}'; } $jsonHeader = $jsonHeader . ']'; // Fetch observed values $query = "SELECT " . implode(",", $headers) . ", " . implode(",", $special_headers) . " FROM aloja2.execs e LEFT JOIN aloja2.clusters c ON e.id_cluster = c.id_cluster WHERE hadoop_version IS NOT NULL" . $where_configs . ";"; $rows = $db->get_rows($query); if (empty($rows)) { throw new \Exception('No data matches with your critteria.'); } if (($key = array_search('e.id_cluster', $headers)) !== false) { $headers[$key] = 'id_cluster'; } $jsonData = '['; foreach ($rows as $row) { $row['id_cluster'] = "Cl" . $row['id_cluster']; // Cluster is numerically codified... $row['comp'] = "Cmp" . $row['comp']; // Compression is numerically codified... $line = ''; foreach ($headers as $hn) { $line = $line . ($line != '' ? ',' : '') . $row[$hn]; } $line = $row['id_exec'] . ',' . $line . ',' . $row['exe_time']; if ($jsonData != '[') { $jsonData = $jsonData . ','; } $jsonData = $jsonData . "['" . str_replace(",", "','", $line) . "']"; } $jsonData = $jsonData . ']'; foreach (range(1, 32) as $value) { $jsonData = str_replace('Cmp' . $value, Utils::getCompressionName($value), $jsonData); } if ($tree_descriptor_ordered == '') { // Display Descriptive Tree, if not processed yet $query = "SELECT tree_code_split, tree_code_gain FROM aloja_ml.observed_trees WHERE id_obstrees = '" . md5($config) . "'"; $result = $dbml->query($query); $row = $result->fetch(); $tree_descriptor_ordered = $row['tree_code_split']; $tree_descriptor_gini = $row['tree_code_gain']; } } catch (\Exception $e) { if ($e->getMessage() != "WAIT") { $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n"); } $jsonData = $jsonHeader = '[]'; } $dbml = null; $return_params = array('jsonData' => $jsonData, 'jsonHeader' => $jsonHeader, 'obstrees' => $jsonObstrees, 'header_obstrees' => $jsonObstreesHeader, 'message' => $message, 'must_wait' => $must_wait, 'instance' => $instance, 'model_info' => $model_info, 'slice_info' => $slice_info, 'id_obstrees' => md5($config), 'tree_descriptor_ordered' => $tree_descriptor_ordered, 'tree_descriptor_gini' => $tree_descriptor_gini); return $this->render('mltemplate/mlobstrees.html.twig', $return_params); }
private function parseDatasize() { $values = Utils::get_GET_intArray('datasize'); $this->filters['datasize']['currentChoice'] = $values; foreach ($values as $value) { $definition = $this->filters['datasize']; $DBreference = $definition['table'] != 'mixed' ? "{$definition['table']}Alias." : ''; $DBreference .= isset($definition['field']) ? $definition['field'] : 'datasize'; $errorMargin = $this->getErrorMargin($value); $maxValue = $value + $errorMargin; $minValue = $value - $errorMargin; $this->whereClause .= " AND {$DBreference} >= {$minValue} AND {$DBreference} <= {$maxValue}"; } }
public function mlparamEvaluationAction() { $rows = $categories = $series = $instance = $model_info = $config = $current_model = $slice_info = ''; $arrayBenchs_pred = $possible_models = $possible_models_id = $other_models = array(); $jsonData = $jsonHeader = "[]"; $must_wait = 'NO'; try { $dbml = new \PDO($this->container->get('config')['db_conn_chain'], $this->container->get('config')['mysql_user'], $this->container->get('config')['mysql_pwd']); $dbml->setAttribute(\PDO::ATTR_ERRMODE, \PDO::ERRMODE_EXCEPTION); $dbml->setAttribute(\PDO::ATTR_EMULATE_PREPARES, false); $db = $this->container->getDBUtils(); if (array_key_exists('parameval', $_GET)) { $paramEval = isset($_GET['parameval']) && Utils::get_GET_string('parameval') != '' ? Utils::get_GET_string('parameval') : 'maps'; unset($_GET["parameval"]); } $this->buildFilters(array('current_model' => array('type' => 'selectOne', 'default' => null, 'label' => 'Model to use: ', 'generateChoices' => function () { return array(); }, 'parseFunction' => function () { $choice = isset($_GET['current_model']) ? $_GET['current_model'] : array(""); return array('whereClause' => '', 'currentChoice' => $choice); }, 'filterGroup' => 'MLearning'), 'minExecs' => array('default' => 0, 'type' => 'inputNumber', 'label' => 'Minimum executions:', 'parseFunction' => function () { return 0; }, 'filterGroup' => 'basic'), 'minexetime' => array('default' => 0), 'valid' => array('default' => 0), 'filter' => array('default' => 0), 'prepares' => array('default' => 0))); $this->buildFilterGroups(array('MLearning' => array('label' => 'Machine Learning', 'tabOpenDefault' => true, 'filters' => array('current_model')))); $where_configs = $this->filters->getWhereClause(); $params = array(); $param_names = array('bench', 'net', 'disk', 'maps', 'iosf', 'replication', 'iofilebuf', 'comp', 'blk_size', 'id_cluster', 'datanodes', 'vm_OS', 'vm_cores', 'vm_RAM', 'provider', 'vm_size', 'type', 'bench_type', 'hadoop_version'); // Order is important $params = $this->filters->getFiltersSelectedChoices($param_names); foreach ($param_names as $p) { if (!is_null($params[$p]) && is_array($params[$p])) { sort($params[$p]); } } $params_additional = array(); $param_names_additional = array('datefrom', 'dateto', 'minexetime', 'maxexetime', 'valid', 'filter'); // Order is important $params_additional = $this->filters->getFiltersSelectedChoices($param_names_additional); $param_variables = $this->filters->getFiltersSelectedChoices(array('current_model', 'minExecs')); $param_current_model = $param_variables['current_model']; $minExecs = $param_variables['minExecs']; $where_configs = str_replace("AND .", "AND ", $where_configs); $where_configs = str_replace("id_cluster", "e.id_cluster", $where_configs); $minExecsFilter = ""; if ($minExecs > 0) { $minExecsFilter = "HAVING COUNT(*) > {$minExecs}"; } $filter_execs = DBUtils::getFilterExecs(); $options = $this->filters->getFilterChoices(); $paramOptions = array(); foreach ($options[$paramEval] as $option) { if ($paramEval == 'comp') { $paramOptions[] = Utils::getCompressionName($option); } else { if ($paramEval == 'net') { $paramOptions[] = Utils::getNetworkName($option); } else { if ($paramEval == 'disk') { $paramOptions[] = Utils::getDisksName($option); } else { $paramOptions[] = $option; } } } } $param_eval_query = $paramEval == 'id_cluster' ? 'e.id_cluster' : $paramEval; $benchOptions = $db->get_rows("SELECT DISTINCT bench FROM aloja2.execs e LEFT JOIN aloja2.clusters c ON e.id_cluster = c.id_cluster WHERE 1 {$filter_execs} {$where_configs} GROUP BY {$param_eval_query}, bench order by {$param_eval_query}"); // get the result rows $query = "SELECT count(*) as count, {$param_eval_query}, e.id_exec, exec as conf, bench, " . "exe_time, avg(exe_time) avg_exe_time, min(exe_time) min_exe_time " . "from aloja2.execs e LEFT JOIN aloja2.clusters c ON e.id_cluster = c.id_cluster WHERE 1 {$filter_execs} {$where_configs}" . "GROUP BY {$param_eval_query},bench {$minExecsFilter} order by bench,{$param_eval_query}"; $rows = $db->get_rows($query); if (!$rows) { throw new \Exception("No results for query!"); } $arrayBenchs = array(); foreach ($paramOptions as $param) { foreach ($benchOptions as $bench) { $arrayBenchs[$bench['bench']][$param] = null; $arrayBenchs[$bench['bench']][$param]['y'] = 0; $arrayBenchs[$bench['bench']][$param]['count'] = 0; } } $series = array(); $bench = ''; foreach ($rows as $row) { if ($paramEval == 'comp') { $row[$paramEval] = Utils::getCompressionName($row['comp']); } else { if ($paramEval == 'net') { $row[$paramEval] = Utils::getNetworkName($row['net']); } else { if ($paramEval == 'disk') { $row[$paramEval] = Utils::getDisksName($row['disk']); } else { if ($paramEval == 'iofilebuf') { $row[$paramEval] /= 1024; } } } } $arrayBenchs[$row['bench']][$row[$paramEval]]['y'] = round((int) $row['avg_exe_time'], 2); $arrayBenchs[$row['bench']][$row[$paramEval]]['count'] = (int) $row['count']; } // ---------------------------------------------------- // Add predictions to the series // ---------------------------------------------------- $param_variables = $this->filters->getFiltersSelectedChoices(array('current_model')); $param_current_model = $param_variables['current_model']; $where_configs = str_replace("AND .", "AND ", $where_configs); // compose instance $instance = MLUtils::generateSimpleInstance($this->filters, $param_names, $params, true); $model_info = MLUtils::generateModelInfo($this->filters, $param_names, $params, true); $slice_info = MLUtils::generateDatasliceInfo($this->filters, $param_names_additional, $params_additional); // model for filling MLUtils::findMatchingModels($model_info, $possible_models, $possible_models_id, $dbml); $current_model = ''; if (!is_null($possible_models_id) && in_array($param_current_model, $possible_models_id)) { $current_model = $param_current_model; } // Other models for filling $where_models = ''; if (!empty($possible_models_id)) { $where_models = " WHERE id_learner NOT IN ('" . implode("','", $possible_models_id) . "')"; } $result = $dbml->query("SELECT id_learner FROM aloja_ml.learners" . $where_models); foreach ($result as $row) { $other_models[] = $row['id_learner']; } if (!empty($possible_models_id)) { if ($current_model == "") { $query = "SELECT AVG(ABS(exe_time - pred_time)) AS MAE, AVG(ABS(exe_time - pred_time)/exe_time) AS RAE, p.id_learner FROM aloja_ml.predictions p, aloja_ml.learners l WHERE l.id_learner = p.id_learner AND p.id_learner IN ('" . implode("','", $possible_models_id) . "') AND predict_code > 0 ORDER BY MAE LIMIT 1"; $result = $dbml->query($query); $row = $result->fetch(); $current_model = $row['id_learner']; } $config = $instance . '-' . $current_model . ' ' . $slice_info . "-parameval"; $query_cache = "SELECT count(*) as total FROM aloja_ml.trees WHERE id_learner = '" . $current_model . "' AND model = '" . $model_info . "'"; $is_cached_mysql = $dbml->query($query_cache); $tmp_result = $is_cached_mysql->fetch(); $is_cached = $tmp_result['total'] > 0; $ret_data = null; if (!$is_cached) { // Call to MLFindAttributes, to fetch data $_GET['pass'] = 2; $_GET['unseen'] = 1; $_GET['current_model'] = $current_model; $mlfa1 = new MLFindAttributesController(); $mlfa1->container = $this->container; $ret_data = $mlfa1->mlfindattributesAction(); if ($ret_data == 1) { $must_wait = "YES"; $jsonData = $jsonHeader = '[]'; } else { $is_cached_mysql = $dbml->query($query_cache); $tmp_result = $is_cached_mysql->fetch(); $is_cached = $tmp_result['total'] > 0; } } if ($is_cached) { $must_wait = 'NO'; $query = "SELECT count(*) as count, {$param_eval_query}, bench, exe_time, avg(pred_time) avg_pred_time, min(pred_time) min_pred_time " . "FROM aloja_ml.predictions e WHERE e.id_learner = '" . $current_model . "' {$filter_execs} {$where_configs}" . "GROUP BY {$param_eval_query}, bench {$minExecsFilter} order by bench,{$param_eval_query}"; $result = $dbml->query($query); // Initialize array foreach ($paramOptions as $param) { foreach ($benchOptions as $bench) { $arrayBenchs_pred[$bench['bench'] . '_pred'][$param] = null; $arrayBenchs_pred[$bench['bench'] . '_pred'][$param]['y'] = 0; $arrayBenchs_pred[$bench['bench'] . '_pred'][$param]['count'] = 0; } } foreach ($result as $row) { $bench_n = $row['bench'] . '_pred'; $class = $row[$paramEval]; if ($paramEval == 'comp') { $value = Utils::getCompressionName($class); } else { if ($paramEval == 'id_cluster') { $value = Utils::getClusterName($class, $db); } else { if ($paramEval == 'net') { $value = Utils::getNetworkName($class); } else { if ($paramEval == 'disk') { $value = Utils::getDisksName($class); } else { if ($paramEval == 'iofilebuf') { $value = $class / 1024; } else { $value = $class; } } } } } if (!in_array($value, $paramOptions)) { $paramOptions[] = $value; foreach ($benchOptions as $bench) { $arrayBenchs_pred[$bench['bench'] . '_pred'][$value] = null; $arrayBenchs_pred[$bench['bench'] . '_pred'][$value]['y'] = 0; $arrayBenchs_pred[$bench['bench'] . '_pred'][$value]['count'] = 0; $arrayBenchs[$bench['bench']][$value] = null; $arrayBenchs[$bench['bench']][$value]['y'] = 0; $arrayBenchs[$bench['bench']][$value]['count'] = 0; } } $arrayBenchs_pred[$bench_n][$value]['y'] = (int) $row['avg_pred_time']; $arrayBenchs_pred[$bench_n][$value]['count'] = (int) $row['count']; } } } // ---------------------------------------------------- // END - Add predictions to the series // ---------------------------------------------------- asort($paramOptions); foreach ($arrayBenchs as $key => $arrayBench) { $caregories = ''; $data_a = null; $data_p = null; foreach ($paramOptions as $param) { if ($arrayBenchs[$key][$param]['count'] > 0 && empty($arrayBenchs_pred) || !empty($arrayBenchs_pred) && ($arrayBenchs_pred[$key . '_pred'][$param]['count'] > 0 || $arrayBenchs[$key][$param]['count'] > 0)) { $data_a[] = $arrayBenchs[$key][$param]; if (!empty($arrayBenchs_pred)) { $data_p[] = $arrayBenchs_pred[$key . '_pred'][$param]; } $categories = $categories . "'{$param} " . Utils::getParamevalUnit($paramEval) . "',"; // FIXME - Redundant n times performed... don't care now } } $series[] = array('name' => $key, 'data' => $data_a); if (!empty($arrayBenchs_pred)) { $series[] = array('name' => $key . '_pred', 'data' => $data_p); } } $series = json_encode($series); if (!empty($arrayBenchs_pred)) { $colors = "['#7cb5ec','#9cd5fc','#434348','#636368','#90ed7d','#b0fd9d','#f7a35c','#f7c37c','#8085e9','#a0a5f9','#f15c80','#f17ca0','#e4d354','#f4f374','#8085e8','#a0a5f8','#8d4653','#ad6673','#91e8e1','#b1f8f1']"; } else { $colors = "['#7cb5ec','#434348','#90ed7d','#f7a35c','#8085e9','#f15c80','#e4d354','#8085e8','#8d4653','#91e8e1']"; } } catch (\Exception $e) { $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n"); $series = $jsonHeader = $colors = '[]'; $must_wait = 'NO'; } $return_params = array('title' => 'Improvement of Hadoop Execution by SW and HW Configurations', 'categories' => $categories, 'series' => $series, 'paramEval' => $paramEval, 'instance' => $instance, 'models' => '<li>' . implode('</li><li>', $possible_models) . '</li>', 'models_id' => $possible_models_id, 'current_model' => $current_model, 'gammacolors' => $colors, 'model_info' => $model_info, 'slice_info' => $slice_info, 'must_wait' => $must_wait); $this->filters->setCurrentChoices('current_model', array_merge($possible_models_id, array('---Other models---'), $other_models)); return $this->render('mltemplate/mlparameval.html.twig', $return_params); }
public static function generateInstances(\alojaweb\Filters\Filters $filters, $param_names, $params, $generalize, $db = null) { $filter_options = $filters->getFilterChoices(); $paramAllOptions = $tokens = $instances = array(); // Get info from clusters (Part of header_names!) $cluster_header_names = array('id_cluster' => 'Cluster', 'datanodes' => 'Datanodes', 'vm_OS' => 'VM.OS', 'vm_cores' => 'VM.Cores', 'vm_RAM' => 'VM.RAM', 'provider' => 'Provider', 'vm_size' => 'VM.Size', 'type' => 'Type'); $cluster_descriptor = array(); $query = "select " . implode(",", array_keys($cluster_header_names)) . " from aloja2.clusters;"; $rows = $db->get_rows($query); foreach ($rows as $row) { $cid = $row['id_cluster']; foreach (array_keys($cluster_header_names) as $cname) { $cluster_descriptor[$cid][$cname] = $row[$cname]; } } // If "No Clusters" -> All clusters if (empty($params['id_cluster'])) { $params['id_cluster'] = array(); $paramAllOptions['id_cluster'] = $filter_options['id_cluster']; foreach ($paramAllOptions['id_cluster'] as $par) { $params['id_cluster'][] = $par; } } // For each cluster selected, launch an instance... foreach ($params['id_cluster'] as $cl) { // Reduce the instance to the HW filter override, or even remove instance if no HW coincides $remove_if_no_props = FALSE; foreach (array_keys($cluster_header_names) as $cname) { if (!empty($params[$cname])) { // FIXME - When clusters have more than 1 characteristic, change this // Get only the current_props in params[cname] $current_props = $cluster_descriptor[$cl][$cname]; $current_props = array($current_props); $coincidences = array_intersect($current_props, $params[$cname]); if (empty($coincidences)) { $remove_if_no_props = TRUE; } else { $cluster_descriptor[$cl][$cname] = $coincidences; } } } if ($remove_if_no_props) { continue; } $cl_characteristics = "Cl" . Utils::multi_implode($cluster_descriptor[$cl], ','); $instance = ''; foreach ($param_names as $p) { // Ignore for now. Will be used at each cluster characteristics if (array_key_exists($p, $cluster_header_names) && $p != "id_cluster") { continue; } if ($p != "id_cluster") { if (array_key_exists($p, $filter_options)) { $paramAllOptions[$p] = $filter_options[$p]; } $tokens[$p] = ''; if ($generalize && empty($params[$p])) { $tokens[$p] = '*'; } elseif (!$generalize && empty($params[$p])) { foreach ($paramAllOptions[$p] as $par) { $tokens[$p] = $tokens[$p] . ($tokens[$p] != '' ? '|' : '') . ($p == 'comp' ? 'Cmp' : '') . ($p == 'id_cluster' ? 'Cl' : '') . $par; } } else { if (is_array($params[$p])) { foreach ($params[$p] as $par) { $tokens[$p] = $tokens[$p] . ($tokens[$p] != '' ? '|' : '') . ($p == 'comp' ? 'Cmp' : '') . ($p == 'id_cluster' ? 'Cl' : '') . $par; } } else { $tokens[$p] = $params[$p]; } } $instance = $instance . ($instance == '' ? '' : ',') . $tokens[$p]; } else { $instance = $instance . ($instance == '' ? '' : ',') . $cl_characteristics; } } $instances[] = $instance; } return MLUtils::completeInstances($filters, $instances, $param_names, $params, $db); }
public function mlpredictionAction() { $jsonExecs = array(); $instance = $error_stats = ''; try { $dbml = new \PDO($this->container->get('config')['db_conn_chain_ml'], $this->container->get('config')['mysql_user'], $this->container->get('config')['mysql_pwd']); $dbml->setAttribute(\PDO::ATTR_ERRMODE, \PDO::ERRMODE_EXCEPTION); $dbml->setAttribute(\PDO::ATTR_EMULATE_PREPARES, false); $db = $this->container->getDBUtils(); $where_configs = ''; $preset = null; if (count($_GET) <= 1 || count($_GET) == 2 && array_key_exists("dump", $_GET) || count($_GET) == 2 && array_key_exists("pass", $_GET) || count($_GET) == 3 && array_key_exists("dump", $_GET) && array_key_exists("pass", $_GET)) { $preset = Utils::setDefaultPreset($db, 'mlprediction'); } $selPreset = isset($_GET['presets']) ? $_GET['presets'] : "none"; $params = array(); $param_names = array('benchs', 'nets', 'disks', 'mapss', 'iosfs', 'replications', 'iofilebufs', 'comps', 'blk_sizes', 'id_clusters', 'datanodess', 'bench_types', 'vm_sizes', 'vm_coress', 'vm_RAMs', 'types'); // Order is important foreach ($param_names as $p) { $params[$p] = Utils::read_params($p, $where_configs, FALSE); sort($params[$p]); } $learn_param = array_key_exists('learn', $_GET) ? $_GET['learn'] : 'regtree'; $unrestricted = array_key_exists('umodel', $_GET) && $_GET['umodel'] == 1; // FIXME PATCH FOR PARAM LIBRARIES WITHOUT LEGACY $where_configs = str_replace("`id_cluster`", "e.`id_cluster`", $where_configs); $where_configs = str_replace("AND .", "AND ", $where_configs); // compose instance $instance = MLUtils::generateSimpleInstance($param_names, $params, $unrestricted, $db); $model_info = MLUtils::generateModelInfo($param_names, $params, $unrestricted, $db); $config = $model_info . ' ' . $learn_param . ' ' . ($unrestricted ? 'U' : 'R'); $learn_options = 'saveall=' . md5($config); if ($learn_param == 'regtree') { $learn_method = 'aloja_regtree'; $learn_options .= ':prange=0,20000'; } else { if ($learn_param == 'nneighbours') { $learn_method = 'aloja_nneighbors'; $learn_options .= ':kparam=3'; } else { if ($learn_param == 'nnet') { $learn_method = 'aloja_nnet'; $learn_options .= ':prange=0,20000'; } else { if ($learn_param == 'polyreg') { $learn_method = 'aloja_linreg'; $learn_options .= ':ppoly=3:prange=0,20000'; } } } } $cache_ds = getcwd() . '/cache/query/' . md5($config) . '-cache.csv'; $is_cached_mysql = $dbml->query("SELECT count(*) as num FROM learners WHERE id_learner = '" . md5($config) . "'"); $tmp_result = $is_cached_mysql->fetch(); $is_cached = $tmp_result['num'] > 0; $in_process = file_exists(getcwd() . '/cache/query/' . md5($config) . '.lock'); $finished_process = file_exists(getcwd() . '/cache/query/' . md5($config) . '.fin'); if (!$is_cached && !$in_process && !$finished_process) { // get headers for csv $header_names = array('id_exec' => 'ID', 'bench' => 'Benchmark', 'exe_time' => 'Exe.Time', 'net' => 'Net', 'disk' => 'Disk', 'maps' => 'Maps', 'iosf' => 'IO.SFac', 'replication' => 'Rep', 'iofilebuf' => 'IO.FBuf', 'comp' => 'Comp', 'blk_size' => 'Blk.size', 'e.id_cluster' => 'Cluster', 'name' => 'Cl.Name', 'datanodes' => 'Datanodes', 'headnodes' => 'Headnodes', 'vm_OS' => 'VM.OS', 'vm_cores' => 'VM.Cores', 'vm_RAM' => 'VM.RAM', 'provider' => 'Provider', 'vm_size' => 'VM.Size', 'type' => 'Type', 'bench_type' => 'Bench.Type'); $headers = array_keys($header_names); $names = array_values($header_names); // dump the result to csv $query = "SELECT " . implode(",", $headers) . " FROM execs e LEFT JOIN clusters c ON e.id_cluster = c.id_cluster WHERE e.valid = TRUE AND e.exe_time > 100" . $where_configs . ";"; $rows = $db->get_rows($query); if (empty($rows)) { throw new \Exception('No data matches with your critteria.'); } $fp = fopen($cache_ds, 'w'); fputcsv($fp, $names, ',', '"'); foreach ($rows as $row) { $row['id_cluster'] = "Cl" . $row['id_cluster']; // Cluster is numerically codified... $row['comp'] = "Cmp" . $row['comp']; // Compression is numerically codified... fputcsv($fp, array_values($row), ',', '"'); } // run the R processor exec('cd ' . getcwd() . '/cache/query ; touch ' . getcwd() . '/cache/query/' . md5($config) . '.lock'); exec('cd ' . getcwd() . '/cache/query ; ' . getcwd() . '/resources/queue -c "' . getcwd() . '/resources/aloja_cli.r -d ' . $cache_ds . ' -m ' . $learn_method . ' -p ' . $learn_options . ' > /dev/null 2>&1; rm -f ' . getcwd() . '/cache/query/' . md5($config) . '.lock; touch ' . md5($config) . '.fin" > /dev/null 2>&1 -p 1 &'); } $in_process = file_exists(getcwd() . '/cache/query/' . md5($config) . '.lock'); $finished_process = file_exists(getcwd() . '/cache/query/' . md5($config) . '.fin'); if ($in_process) { $jsonExecs = "[]"; $must_wait = "YES"; $max_x = $max_y = 0; if (isset($_GET['dump'])) { echo "1"; exit(0); } if (isset($_GET['pass'])) { return 1; } } else { $is_cached_mysql = $dbml->query("SELECT count(*) as num FROM learners WHERE id_learner = '" . md5($config) . "'"); $tmp_result = $is_cached_mysql->fetch(); $is_cached = $tmp_result['num'] > 0; if (!$is_cached) { // register model to DB $query = "INSERT IGNORE INTO learners (id_learner,instance,model,algorithm)"; $query = $query . " VALUES ('" . md5($config) . "','" . $instance . "','" . substr($model_info, 1) . "','" . $learn_param . "');"; if ($dbml->query($query) === FALSE) { throw new \Exception('Error when saving model into DB'); } // read results of the CSV and dump to DB foreach (array("tt", "tv", "tr") as $value) { if (($handle = fopen(getcwd() . '/cache/query/' . md5($config) . '-' . $value . '.csv', 'r')) !== FALSE) { $header = fgetcsv($handle, 1000, ","); $token = 0; $insertions = 0; $query = "INSERT IGNORE INTO predictions (id_exec,exe_time,bench,net,disk,maps,iosf,replication,iofilebuf,comp,blk_size,id_cluster,name,datanodes,headnodes,vm_OS,vm_cores,vm_RAM,provider,vm_size,type,bench_type,pred_time,id_learner,instance,predict_code) VALUES "; while (($data = fgetcsv($handle, 1000, ",")) !== FALSE) { $specific_instance = implode(",", array_slice($data, 2, 20)); $specific_data = implode(",", $data); $specific_data = preg_replace('/,Cmp(\\d+),/', ',${1},', $specific_data); $specific_data = preg_replace('/,Cl(\\d+),/', ',${1},', $specific_data); $specific_data = str_replace(",", "','", $specific_data); $query_var = "SELECT count(*) as num FROM predictions WHERE instance = '" . $specific_instance . "' AND id_learner = '" . md5($config) . "'"; $result = $dbml->query($query_var); $row = $result->fetch(); // Insert instance values if ($row['num'] == 0) { if ($token != 0) { $query = $query . ","; } $token = 1; $insertions = 1; $query = $query . "('" . $specific_data . "','" . md5($config) . "','" . $specific_instance . "','" . ($value == 'tt' ? 3 : ($value == 'tv' ? 2 : 1)) . "') "; } } if ($insertions > 0) { if ($dbml->query($query) === FALSE) { throw new \Exception('Error when saving into DB'); } } fclose($handle); } } // Store file model to DB $filemodel = getcwd() . '/cache/query/' . md5($config) . '-object.rds'; $fp = fopen($filemodel, 'r'); $content = fread($fp, filesize($filemodel)); $content = addslashes($content); fclose($fp); $query = "INSERT INTO model_storage (id_hash,type,file) VALUES ('" . md5($config) . "','learner','" . $content . "');"; if ($dbml->query($query) === FALSE) { throw new \Exception('Error when saving file model into DB'); } // Remove temporal files $output = shell_exec('rm -f ' . getcwd() . '/cache/query/' . md5($config) . '*.csv'); $output = shell_exec('rm -f ' . getcwd() . '/cache/query/' . md5($config) . '*.fin'); $output = shell_exec('rm -f ' . getcwd() . '/cache/query/' . md5($config) . '*.dat'); $output = shell_exec('rm -f ' . getcwd() . '/cache/query/' . md5($config) . '*.rds'); } $must_wait = "NO"; $count = 0; $max_x = $max_y = 0; $error_stats = ''; $query = "SELECT exe_time, pred_time, instance FROM predictions WHERE id_learner='" . md5($config) . "' AND exe_time > 100 LIMIT 5000"; // FIXME - CLUMPSY PATCH FOR BYPASS THE BUG FROM HIGHCHARTS... REMEMBER TO ERASE THIS LIMIT WHEN THE BUG IS SOLVED $result = $dbml->query($query); foreach ($result as $row) { $jsonExecs[$count]['y'] = (int) $row['exe_time']; $jsonExecs[$count]['x'] = (int) $row['pred_time']; $jsonExecs[$count]['mydata'] = $row['instance']; if ((int) $row['exe_time'] > $max_y) { $max_y = (int) $row['exe_time']; } if ((int) $row['pred_time'] > $max_x) { $max_x = (int) $row['pred_time']; } $count++; } $query = "SELECT AVG(ABS(exe_time - pred_time)) AS MAE, AVG(ABS(exe_time - pred_time)/exe_time) AS RAE, predict_code FROM predictions WHERE id_learner='" . md5($config) . "' AND predict_code > 0 AND exe_time > 100 GROUP BY predict_code"; $result = $dbml->query($query); foreach ($result as $row) { $error_stats = $error_stats . 'Dataset: ' . ($row['predict_code'] == 1 ? 'tr' : ($row['predict_code'] == 2 ? 'tv' : 'tt')) . ' => MAE: ' . $row['MAE'] . ' RAE: ' . $row['RAE'] . '<br/>'; } if (isset($_GET['dump'])) { $data = json_encode($jsonExecs); echo "Observed, Predicted, Execution\n"; echo str_replace(array('},{"y":', '"x":', '"mydata":', '[{"y":', '"}]'), array("\n", '', '', '', ''), $data); exit(0); } if (isset($_GET['pass'])) { $data = json_encode($jsonExecs); $retval = "Observed, Predicted, Execution\n"; $retval = $retval . str_replace(array('},{"y":', '"x":', '"mydata":', '[{"y":', '"}]'), array("\n", '', '', '', ''), $data); return $retval; } } $dbml = null; } catch (\Exception $e) { $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n"); $jsonExecs = '[]'; $max_x = $max_y = 0; $must_wait = 'NO'; $dbml = null; } echo $this->container->getTwig()->render('mltemplate/mlprediction.html.twig', array('selected' => 'mlprediction', 'jsonExecs' => json_encode($jsonExecs), 'max_p' => min(array($max_x, $max_y)), 'benchs' => $params['benchs'], 'nets' => $params['nets'], 'disks' => $params['disks'], 'blk_sizes' => $params['blk_sizes'], 'comps' => $params['comps'], 'id_clusters' => $params['id_clusters'], 'mapss' => $params['mapss'], 'replications' => $params['replications'], 'iosfs' => $params['iosfs'], 'iofilebufs' => $params['iofilebufs'], 'datanodess' => $params['datanodess'], 'bench_types' => $params['bench_types'], 'vm_sizes' => $params['vm_sizes'], 'vm_coress' => $params['vm_coress'], 'vm_RAMs' => $params['vm_RAMs'], 'types' => $params['types'], 'unrestricted' => $unrestricted, 'learn' => $learn_param, 'must_wait' => $must_wait, 'instance' => $instance, 'model_info' => $model_info, 'id_learner' => md5($config), 'error_stats' => $error_stats, 'preset' => $preset, 'selPreset' => $selPreset, 'options' => Utils::getFilterOptions($db))); }