Beispiel #1
0
 public function __construct(\alojaweb\inc\DBUtils $dbConnection)
 {
     $this->dbConnection = $dbConnection;
     /* In this array there are the filter names with its default options
      * that will be overwritten by the given custom defaults and options if given
      * Array with filter => filter specific settings
      *
      * Specific settings is an array with
      * types: inputText, inputNumber[{le,ge}], inputDate[{le,ge}], selectOne, selectMultiple, checkbox[Negated]
      * default: null (any), array(values)
      * table: associated DB table name
      * parseFunction: function to parse special filter, for filters that need a lot of customization
      *
      * Very custom filters such as advanced filters not in this array
      *
      */
     $this->filters = array('bench' => array('table' => 'execs', 'default' => array('terasort', 'wordcount'), 'type' => 'selectMultiple', 'label' => 'Benchmarks:'), 'datasize' => array('database' => 'aloja2', 'table' => 'execs', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Datasize: ', 'beautifier' => function ($value) {
         return Utils::beautifyDatasize($value);
     }, 'parseFunction' => 'parseDatasize'), 'scale_factor' => array('table' => 'execs', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Scale factor: '), 'bench_type' => array('table' => 'execs', 'default' => array('HiBench'), 'type' => 'selectOne', 'label' => 'Bench suite:'), 'net' => array('table' => 'execs', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Network:', 'beautifier' => function ($value) {
         return Utils::getNetworkName($value);
     }), 'disk' => array('table' => 'execs', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Disk:'), 'blk_size' => array('table' => 'execs', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Block size (b):', 'beautifier' => function ($value) {
         return $value . ' MB';
     }), 'comp' => array('table' => 'execs', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Compression (c):', 'beautifier' => function ($value) {
         return Utils::getCompressionName($value);
     }), 'id_cluster' => array('table' => 'execs', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Clusters (CL):', 'beautifier' => function ($value) {
         return $this->filters['id_cluster']['namesClusters'][$value];
     }, 'generateChoices' => function () {
         $choices = $this->dbConnection->get_rows("select distinct id_cluster,CONCAT_WS('/',LPAD(id_cluster,2,0),c.vm_size,CONCAT(c.datanodes,'Dn')) as name  from aloja2.execs e join aloja2.clusters c using (id_cluster) WHERE 1 " . DBUtils::getFilterExecs(' ') . " ORDER BY c.name ASC");
         $returnChoices = array();
         foreach ($choices as $choice) {
             $returnChoices[] = $choice['id_cluster'];
             //Not nice, but saves multiple queries to DB in the beautifier
             $this->filters['id_cluster']['namesClusters'][$choice['id_cluster']] = $choice['name'];
         }
         return $returnChoices;
     }), 'maps' => array('table' => 'execs', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Maps:', 'beautifier' => function ($value) {
         if ($value == 0) {
             return 'N/A';
         } else {
             return $value;
         }
     }), 'replication' => array('table' => 'execs', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Replication (r):'), 'iosf' => array('table' => 'execs', 'default' => null, 'type' => 'selectMultiple', 'label' => 'I/O sort factor (I):'), 'iofilebuf' => array('table' => 'execs', 'default' => null, 'type' => 'selectMultiple', 'label' => 'I/O file buffer:', 'beautifier' => function ($value) {
         $suffix = ' KB';
         if ($value >= 1024) {
             $value /= 1024;
             $suffix = ' MB';
         }
         return $value . $suffix;
     }), 'provider' => array('table' => 'clusters', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Provider:'), 'vm_OS' => array('table' => 'clusters', 'default' => null, 'type' => 'selectMultiple', 'label' => 'VM OS:'), 'datanodes' => array('table' => 'clusters', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Cluster datanodes:'), 'vm_size' => array('table' => 'clusters', 'default' => null, 'type' => 'selectMultiple', 'label' => 'VM Size:'), 'vm_cores' => array('table' => 'clusters', 'default' => null, 'type' => 'selectMultiple', 'label' => 'VM cores:'), 'vm_RAM' => array('table' => 'clusters', 'default' => null, 'type' => 'selectMultiple', 'label' => 'VM RAM:', 'beautifier' => function ($value) {
         if ($value * 10 % 10 != 0) {
             return number_format($value, 1) . ' GB';
         } else {
             return number_format($value, 0) . ' GB';
         }
     }), 'type' => array('table' => 'clusters', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Cluster type:'), 'hadoop_version' => array('table' => 'execs', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Hadoop version:'), 'minexetime' => array('table' => 'execs', 'field' => 'exe_time', 'default' => Utils::in_dev() ? 1 : 50, 'type' => 'inputNumberge', 'label' => 'Min exec time:'), 'maxexetime' => array('table' => 'execs', 'field' => 'exe_time', 'default' => null, 'type' => 'inputNumberle', 'label' => 'Max exec time:'), 'datefrom' => array('table' => 'execs', 'field' => 'start_time', 'default' => null, 'type' => 'inputDatege', 'label' => 'Date from:'), 'dateto' => array('table' => 'execs', 'field' => 'end_time', 'default' => null, 'type' => 'inputDatele', 'label' => 'Date to:'), 'money' => array('table' => 'mixed', 'field' => '(clustersAlias.cost_hour/3600)*execsAlias.exe_time', 'default' => null, 'type' => 'inputNumberle', 'label' => 'Max cost (US$):'), 'valid' => array('table' => 'execs', 'field' => 'valid', 'type' => 'checkbox', 'default' => 1, 'label' => 'Only valid execs'), 'filter' => array('table' => 'execs', 'field' => 'filter', 'type' => 'checkbox', 'default' => 1, 'label' => 'Filter', 'parseFunction' => function () {
         $whereClause = "";
         if (isset($_GET['filter'])) {
             $values = 1;
         } else {
             if (!$this->formisSubmitted()) {
                 $values = $this->filters['filter']['default'];
             } else {
                 $values = 0;
             }
         }
         if ($values) {
             $whereClause = " AND execsAlias.filter = 0 ";
         }
         return array('currentChoice' => $values, 'whereClause' => $whereClause);
     }), 'prepares' => array('table' => 'execs', 'type' => 'checkbox', 'default' => Utils::in_dev() ? 1 : 0, 'label' => 'Include prepares', 'parseFunction' => function () {
         $whereClause = "";
         $values = 0;
         if (isset($_GET['prepares'])) {
             $values = 1;
         } else {
             $values = $this->filters['prepares']['default'];
             if (!$values) {
                 $whereClause = " AND execsAlias.bench NOT LIKE 'prep_%' ";
             }
         }
         return array('currentChoice' => $values, 'whereClause' => $whereClause);
     }), 'perf_details' => array('table' => 'execs', 'type' => 'checkbox', 'default' => 0, 'label' => 'Only execs with perf details'), 'prediction_model' => array('type' => 'selectOne', 'default' => null, 'label' => 'Reference Model: ', 'generateChoices' => function () {
         $query = "SELECT DISTINCT id_learner FROM aloja_ml.predictions";
         $retval = $this->dbConnection->get_rows($query);
         return array_column($retval, "id_learner");
     }, 'parseFunction' => function () {
         $choice = isset($_GET['prediction_model']) ? Utils::get_GET_stringArray('prediction_model') : array("");
         if ($choice = array("")) {
             $query = "SELECT DISTINCT id_learner FROM aloja_ml.predictions LIMIT 1";
             $choice = $this->dbConnection->get_rows($query)[0]['id_learner'];
         }
         return array('whereClause' => '', 'currentChoice' => $choice);
     }, 'filterGroup' => 'MLearning'), 'upred' => array('type' => 'checkbox', 'default' => 0, 'label' => 'Use predictions', 'parseFunction' => function () {
         $choice = !isset($_GET['upred']) ? 0 : 1;
         return array('whereClause' => '', 'currentChoice' => $choice);
     }, 'filterGroup' => 'MLearning'), 'uobsr' => array('type' => 'checkbox', 'default' => 1, 'label' => 'Use observations', 'parseFunction' => function () {
         $choice = !isset($_GET['uobsr']) && $this->formIssubmitted() ? 0 : 1;
         return array('whereClause' => '', 'currentChoice' => $choice);
     }, 'filterGroup' => 'MLearning'), 'warning' => array('field' => 'outlier', 'table' => 'ml_predictions', 'type' => 'checkbox', 'default' => 0, 'label' => 'Show warnings', 'parseFunction' => function () {
         $learner = $this->filters['prediction_model']['currentChoice'];
         $whereClause = "";
         $values = isset($_GET['warning']) ? 1 : 0;
         if ($values && !empty($learner)) {
             $whereClause = " AND (ml_predictionsAlias.outlier <= {$values} OR ml_predictionsAlias.outlier IS NULL) " . "AND (ml_predictionsAlias.id_learner = '{$learner[0]}' OR ml_predictionsAlias.id_learner IS NULL)";
         }
         return array('currentChoice' => $values, 'whereClause' => $whereClause);
     }, 'filterGroup' => 'MLearning'), 'outlier' => array('table' => 'ml_predictions', 'type' => 'checkbox', 'default' => 0, 'label' => 'Show outliers', 'parseFunction' => function () {
         $learner = $this->filters['prediction_model']['currentChoice'];
         $whereClause = "";
         $values = isset($_GET['outlier']) ? 2 : 0;
         if ($values && !empty($learner)) {
             $whereClause = " AND (ml_predictionsAlias.outlier <= 2 OR ml_predictionsAlias.outlier IS NULL) " . "AND (ml_predictionsAlias.id_learner = '{$learner}' OR ml_predictionsAlias.id_learner IS NULL)";
             $values = 1;
         } else {
             if (!empty($learner) && !isset($_GET['warning'])) {
                 $whereClause = " AND (ml_predictionsAlias.outlier = 0 OR ml_predictionsAlias.outlier IS NULL) " . "AND (ml_predictionsAlias.id_learner = '{$learner}' OR ml_predictionsAlias.id_learner IS NULL)";
             }
         }
         return array('currentChoice' => $values, 'whereClause' => $whereClause);
     }, 'filterGroup' => 'MLearning'));
     $this->aliasesTables = array('execs' => 'e', 'clusters' => 'c', 'ml_predictions' => 'p');
     //To render groups on template. Rows are of 2 columns each. emptySpace puts an empty element on the rendered row
     $this->filterGroups = array('basic' => array('label' => 'Basic filters', 'filters' => array('money', 'bench', 'bench_type', 'datasize', 'scale_factor', 'id_cluster', 'net', 'disk'), 'tabOpenDefault' => true), 'hardware' => array('label' => 'Hardware', 'filters' => array('datanodes', 'vm_size', 'vm_cores', 'vm_RAM', 'type', 'provider', 'vm_OS'), 'tabOpenDefault' => false), 'hadoop' => array('label' => 'Hadoop', 'filters' => array('maps', 'comp', 'replication', 'blk_size', 'iosf', 'iofilebuf', 'hadoop_version'), 'tabOpenDefault' => false), 'advanced' => array('label' => 'Advanced filters', 'filters' => array('valid', 'filter', 'prepares', 'perf_details', 'datefrom', 'dateto', 'minexetime', 'maxexetime'), 'tabOpenDefault' => false), 'MLearning' => array('label' => 'Machine Learning', 'filters' => array('prediction_model', 'upred', 'uobsr', 'warning', 'outlier'), 'tabOpenDefault' => true));
 }
 private function nodesEvalCore($scalabilityType, $dbUtils)
 {
     $categories = array();
     $series = array();
     $datanodes = array();
     try {
         $filter_execs = DBUtils::getFilterExecs();
         $innerQueryWhere = str_replace("e.", "e2.", $this->whereClause);
         $innerQueryWhere = str_replace("c.", "c2.", $innerQueryWhere);
         $innerQueryWhere = str_replace("p.", "p2.", $innerQueryWhere);
         $execs = "SELECT c.datanodes as 'category',e.exec_type,c.vm_OS,c.vm_size,(e.exe_time * (c.cost_hour/3600)) as cost,e.exe_time,c.*\n\t\t\t\t\tFROM aloja2.execs e JOIN aloja2.clusters c USING (id_cluster)\n\t\t\t\t\tLEFT JOIN aloja_ml.predictions p USING (id_exec)\n\t\t\t\t\tINNER JOIN ( SELECT c2.datanodes as 'category',e2.exec_type,c2.vm_OS,c2.vm_size as vmsize,MIN(e2.exe_time) as minexe\n\t\t\t\t\t\t\t\tfrom execs e2 JOIN aloja2.clusters c2 USING (id_cluster)\n\t\t\t\t\t\t\t\tLEFT JOIN aloja_ml.predictions p2 USING (id_exec)\n\t\t\t\t\t\t\t\tWHERE 1 {$innerQueryWhere} GROUP BY c2.datanodes,e2.exec_type,c2.vm_OS,c2.vm_size ) t ON t.minexe = e.exe_time\n\t\t\t\t\tAND t.category = category AND t.vmsize = c.vm_size\n\t\t\t\t\tWHERE 1 {$filter_execs}  GROUP BY c.datanodes,e.exec_type,c.vm_OS,c.vm_size\n\t\t\t\t\tORDER BY c.datanodes ASC,c.vm_OS,c.vm_size DESC";
         $predExecs = "SELECT c.datanodes as 'category','predicted' as 'exec_type',c.vm_OS,c.vm_size,(e.exe_time * (c.cost_hour/3600)) as cost,e.exe_time,c.*\n\t\t\t\t\tFROM aloja_ml.predictions e JOIN aloja2.clusters c USING (id_cluster)\n\t\t\t\t\tINNER JOIN ( SELECT c2.datanodes as 'category','default' as 'exec_type',c2.vm_OS,c2.vm_size as vmsize,MIN(p2.exe_time) as minexe\n\t\t\t\t\t\t\t\tfrom aloja_ml.predictions p2 JOIN aloja2.clusters c2 USING (id_cluster)\n\t\t\t\t\t\t\t\tWHERE 1 " . str_replace("e2.", "p2.", $innerQueryWhere) . " GROUP BY c2.datanodes,exec_type,c2.vm_OS,c2.vm_size ) t ON t.minexe = e.exe_time\n\t\t\t\t\tAND t.category = category AND t.vmsize = c.vm_size\n\t\t\t\t\tWHERE 1 {$filter_execs}  GROUP BY c.datanodes,exec_type,c.vm_OS,c.vm_size\n\t\t\t\t\tORDER BY c.datanodes ASC,c.vm_OS,c.vm_size DESC";
         if ($scalabilityType == 'Datasize') {
             $execs = str_replace("c2.datanodes", "e2.datasize", $execs);
             $execs = str_replace("c.datanodes", "e.datasize", $execs);
             $predExecs = str_replace("c2.datanodes", "p2.datasize", $predExecs);
             $predExecs = str_replace("c.datanodes", "e.datasize", $predExecs);
         }
         $params = $this->filters->getFiltersSelectedChoices(array('upred', 'uobsr'));
         if ($params['uobsr'] == 1 && $params['upred'] == 1) {
             $execs = "({$execs}) UNION ({$predExecs})";
         } else {
             if ($params['uobsr'] == 0 && $params['upred'] == 1) {
                 $execs = $predExecs;
             }
         }
         $execs = $dbUtils->get_rows($execs);
         $vmSizes = array();
         $dataNodes = array();
         $vmOS = array();
         $execTypes = array();
         foreach ($execs as &$exec) {
             if (!isset($dataNodes[$exec['category']])) {
                 $dataNodes[$exec['category']] = 1;
                 $categories[] = $exec['category'];
             }
             if (!isset($vmOS[$exec['vm_OS']])) {
                 $vmOS[$exec['vm_OS']] = 1;
             }
             if (!isset($execTypes[$exec['exec_type']])) {
                 $execTypes[$exec['exec_type']] = 1;
             }
             $vmSizes[$exec['vm_size']][$exec['exec_type']][$exec['vm_OS']][$exec['category']] = array(round($exec['exe_time'], 2), round($exec['cost'], 2));
         }
         $i = 0;
         $seriesColors = array('#7cb5ec', '#434348', '#90ed7d', '#f7a35c', '#8085e9', '#f15c80', '#e4d354', '#2b908f', '#f45b5b', '#91e8e1');
         foreach ($vmSizes as $vmSize => $value) {
             foreach ($execTypes as $execType => $typevalue) {
                 foreach ($vmOS as $OS => $osvalue) {
                     if (isset($vmSizes[$vmSize][$execType][$OS])) {
                         if ($i == sizeof($seriesColors)) {
                             $i = 0;
                         }
                         $costSeries = array('name' => "{$vmSize} {$execType} {$OS} Run cost", 'type' => 'spline', 'dashStyle' => 'longdash', 'yAxis' => 0, 'data' => array(), 'tooltip' => array('valueSuffix' => ' US$'), 'color' => $seriesColors[$i]);
                         $timeSeries = array('name' => "{$vmSize} {$execType} {$OS} Run execution time", 'type' => 'spline', 'yAxis' => 1, 'data' => array(), 'tooltip' => array('valueSuffix' => ' s'), 'color' => $seriesColors[$i++]);
                         foreach ($dataNodes as $datanode => $dvalue) {
                             $datanodes[] = $datanode;
                             if (!isset($value[$execType][$OS][$datanode])) {
                                 $costSeries['data'][] = "null";
                                 $timeSeries['data'][] = "null";
                             } else {
                                 $costSeries['data'][] = $value[$execType][$OS][$datanode][1];
                                 $timeSeries['data'][] = $value[$execType][$OS][$datanode][0];
                             }
                         }
                         $series[] = $timeSeries;
                         $series[] = $costSeries;
                     }
                 }
             }
         }
     } catch (\Exception $e) {
         $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n");
     }
     if ($scalabilityType == 'Datasize') {
         foreach ($categories as &$category) {
             $category = Utils::beautifyDatasize($category);
         }
     }
     return array('categories' => json_encode($categories), 'seriesData' => str_replace('"null"', 'null', json_encode($series)), 'datanodess' => $datanodes);
 }