public function __construct(\alojaweb\inc\DBUtils $dbConnection)
{
$this->dbConnection = $dbConnection;
/* In this array there are the filter names with its default options
* that will be overwritten by the given custom defaults and options if given
* Array with filter => filter specific settings
*
* Specific settings is an array with
* types: inputText, inputNumber[{le,ge}], inputDate[{le,ge}], selectOne, selectMultiple, checkbox[Negated]
* default: null (any), array(values)
* table: associated DB table name
* parseFunction: function to parse special filter, for filters that need a lot of customization
*
* Very custom filters such as advanced filters not in this array
*
*/
$this->filters = array('bench' => array('table' => 'execs', 'default' => array('terasort', 'wordcount'), 'type' => 'selectMultiple', 'label' => 'Benchmarks:'), 'datasize' => array('database' => 'aloja2', 'table' => 'execs', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Datasize: ', 'beautifier' => function ($value) {
return Utils::beautifyDatasize($value);
}, 'parseFunction' => 'parseDatasize'), 'scale_factor' => array('table' => 'execs', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Scale factor: '), 'bench_type' => array('table' => 'execs', 'default' => array('HiBench'), 'type' => 'selectOne', 'label' => 'Bench suite:'), 'net' => array('table' => 'execs', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Network:', 'beautifier' => function ($value) {
return Utils::getNetworkName($value);
}), 'disk' => array('table' => 'execs', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Disk:'), 'blk_size' => array('table' => 'execs', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Block size (b):', 'beautifier' => function ($value) {
return $value . ' MB';
}), 'comp' => array('table' => 'execs', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Compression (c):', 'beautifier' => function ($value) {
return Utils::getCompressionName($value);
}), 'id_cluster' => array('table' => 'execs', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Clusters (CL):', 'beautifier' => function ($value) {
return $this->filters['id_cluster']['namesClusters'][$value];
}, 'generateChoices' => function () {
$choices = $this->dbConnection->get_rows("select distinct id_cluster,CONCAT_WS('/',LPAD(id_cluster,2,0),c.vm_size,CONCAT(c.datanodes,'Dn')) as name from aloja2.execs e join aloja2.clusters c using (id_cluster) WHERE 1 " . DBUtils::getFilterExecs(' ') . " ORDER BY c.name ASC");
$returnChoices = array();
foreach ($choices as $choice) {
$returnChoices[] = $choice['id_cluster'];
//Not nice, but saves multiple queries to DB in the beautifier
$this->filters['id_cluster']['namesClusters'][$choice['id_cluster']] = $choice['name'];
}
return $returnChoices;
}), 'maps' => array('table' => 'execs', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Maps:', 'beautifier' => function ($value) {
if ($value == 0) {
return 'N/A';
} else {
return $value;
}
}), 'replication' => array('table' => 'execs', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Replication (r):'), 'iosf' => array('table' => 'execs', 'default' => null, 'type' => 'selectMultiple', 'label' => 'I/O sort factor (I):'), 'iofilebuf' => array('table' => 'execs', 'default' => null, 'type' => 'selectMultiple', 'label' => 'I/O file buffer:', 'beautifier' => function ($value) {
$suffix = ' KB';
if ($value >= 1024) {
$value /= 1024;
$suffix = ' MB';
}
return $value . $suffix;
}), 'provider' => array('table' => 'clusters', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Provider:'), 'vm_OS' => array('table' => 'clusters', 'default' => null, 'type' => 'selectMultiple', 'label' => 'VM OS:'), 'datanodes' => array('table' => 'clusters', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Cluster datanodes:'), 'vm_size' => array('table' => 'clusters', 'default' => null, 'type' => 'selectMultiple', 'label' => 'VM Size:'), 'vm_cores' => array('table' => 'clusters', 'default' => null, 'type' => 'selectMultiple', 'label' => 'VM cores:'), 'vm_RAM' => array('table' => 'clusters', 'default' => null, 'type' => 'selectMultiple', 'label' => 'VM RAM:', 'beautifier' => function ($value) {
if ($value * 10 % 10 != 0) {
return number_format($value, 1) . ' GB';
} else {
return number_format($value, 0) . ' GB';
}
}), 'type' => array('table' => 'clusters', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Cluster type:'), 'hadoop_version' => array('table' => 'execs', 'default' => null, 'type' => 'selectMultiple', 'label' => 'Hadoop version:'), 'minexetime' => array('table' => 'execs', 'field' => 'exe_time', 'default' => Utils::in_dev() ? 1 : 50, 'type' => 'inputNumberge', 'label' => 'Min exec time:'), 'maxexetime' => array('table' => 'execs', 'field' => 'exe_time', 'default' => null, 'type' => 'inputNumberle', 'label' => 'Max exec time:'), 'datefrom' => array('table' => 'execs', 'field' => 'start_time', 'default' => null, 'type' => 'inputDatege', 'label' => 'Date from:'), 'dateto' => array('table' => 'execs', 'field' => 'end_time', 'default' => null, 'type' => 'inputDatele', 'label' => 'Date to:'), 'money' => array('table' => 'mixed', 'field' => '(clustersAlias.cost_hour/3600)*execsAlias.exe_time', 'default' => null, 'type' => 'inputNumberle', 'label' => 'Max cost (US$):'), 'valid' => array('table' => 'execs', 'field' => 'valid', 'type' => 'checkbox', 'default' => 1, 'label' => 'Only valid execs'), 'filter' => array('table' => 'execs', 'field' => 'filter', 'type' => 'checkbox', 'default' => 1, 'label' => 'Filter', 'parseFunction' => function () {
$whereClause = "";
if (isset($_GET['filter'])) {
$values = 1;
} else {
if (!$this->formisSubmitted()) {
$values = $this->filters['filter']['default'];
} else {
$values = 0;
}
}
if ($values) {
$whereClause = " AND execsAlias.filter = 0 ";
}
return array('currentChoice' => $values, 'whereClause' => $whereClause);
}), 'prepares' => array('table' => 'execs', 'type' => 'checkbox', 'default' => Utils::in_dev() ? 1 : 0, 'label' => 'Include prepares', 'parseFunction' => function () {
$whereClause = "";
$values = 0;
if (isset($_GET['prepares'])) {
$values = 1;
} else {
$values = $this->filters['prepares']['default'];
if (!$values) {
$whereClause = " AND execsAlias.bench NOT LIKE 'prep_%' ";
}
}
return array('currentChoice' => $values, 'whereClause' => $whereClause);
}), 'perf_details' => array('table' => 'execs', 'type' => 'checkbox', 'default' => 0, 'label' => 'Only execs with perf details'), 'prediction_model' => array('type' => 'selectOne', 'default' => null, 'label' => 'Reference Model: ', 'generateChoices' => function () {
$query = "SELECT DISTINCT id_learner FROM aloja_ml.predictions";
$retval = $this->dbConnection->get_rows($query);
return array_column($retval, "id_learner");
}, 'parseFunction' => function () {
$choice = isset($_GET['prediction_model']) ? Utils::get_GET_stringArray('prediction_model') : array("");
if ($choice = array("")) {
$query = "SELECT DISTINCT id_learner FROM aloja_ml.predictions LIMIT 1";
$choice = $this->dbConnection->get_rows($query)[0]['id_learner'];
}
return array('whereClause' => '', 'currentChoice' => $choice);
}, 'filterGroup' => 'MLearning'), 'upred' => array('type' => 'checkbox', 'default' => 0, 'label' => 'Use predictions', 'parseFunction' => function () {
$choice = !isset($_GET['upred']) ? 0 : 1;
return array('whereClause' => '', 'currentChoice' => $choice);
}, 'filterGroup' => 'MLearning'), 'uobsr' => array('type' => 'checkbox', 'default' => 1, 'label' => 'Use observations', 'parseFunction' => function () {
$choice = !isset($_GET['uobsr']) && $this->formIssubmitted() ? 0 : 1;
return array('whereClause' => '', 'currentChoice' => $choice);
}, 'filterGroup' => 'MLearning'), 'warning' => array('field' => 'outlier', 'table' => 'ml_predictions', 'type' => 'checkbox', 'default' => 0, 'label' => 'Show warnings', 'parseFunction' => function () {
$learner = $this->filters['prediction_model']['currentChoice'];
$whereClause = "";
$values = isset($_GET['warning']) ? 1 : 0;
if ($values && !empty($learner)) {
$whereClause = " AND (ml_predictionsAlias.outlier <= {$values} OR ml_predictionsAlias.outlier IS NULL) " . "AND (ml_predictionsAlias.id_learner = '{$learner[0]}' OR ml_predictionsAlias.id_learner IS NULL)";
}
return array('currentChoice' => $values, 'whereClause' => $whereClause);
}, 'filterGroup' => 'MLearning'), 'outlier' => array('table' => 'ml_predictions', 'type' => 'checkbox', 'default' => 0, 'label' => 'Show outliers', 'parseFunction' => function () {
$learner = $this->filters['prediction_model']['currentChoice'];
$whereClause = "";
$values = isset($_GET['outlier']) ? 2 : 0;
if ($values && !empty($learner)) {
$whereClause = " AND (ml_predictionsAlias.outlier <= 2 OR ml_predictionsAlias.outlier IS NULL) " . "AND (ml_predictionsAlias.id_learner = '{$learner}' OR ml_predictionsAlias.id_learner IS NULL)";
$values = 1;
} else {
if (!empty($learner) && !isset($_GET['warning'])) {
$whereClause = " AND (ml_predictionsAlias.outlier = 0 OR ml_predictionsAlias.outlier IS NULL) " . "AND (ml_predictionsAlias.id_learner = '{$learner}' OR ml_predictionsAlias.id_learner IS NULL)";
}
}
return array('currentChoice' => $values, 'whereClause' => $whereClause);
}, 'filterGroup' => 'MLearning'));
$this->aliasesTables = array('execs' => 'e', 'clusters' => 'c', 'ml_predictions' => 'p');
//To render groups on template. Rows are of 2 columns each. emptySpace puts an empty element on the rendered row
$this->filterGroups = array('basic' => array('label' => 'Basic filters', 'filters' => array('money', 'bench', 'bench_type', 'datasize', 'scale_factor', 'id_cluster', 'net', 'disk'), 'tabOpenDefault' => true), 'hardware' => array('label' => 'Hardware', 'filters' => array('datanodes', 'vm_size', 'vm_cores', 'vm_RAM', 'type', 'provider', 'vm_OS'), 'tabOpenDefault' => false), 'hadoop' => array('label' => 'Hadoop', 'filters' => array('maps', 'comp', 'replication', 'blk_size', 'iosf', 'iofilebuf', 'hadoop_version'), 'tabOpenDefault' => false), 'advanced' => array('label' => 'Advanced filters', 'filters' => array('valid', 'filter', 'prepares', 'perf_details', 'datefrom', 'dateto', 'minexetime', 'maxexetime'), 'tabOpenDefault' => false), 'MLearning' => array('label' => 'Machine Learning', 'filters' => array('prediction_model', 'upred', 'uobsr', 'warning', 'outlier'), 'tabOpenDefault' => true));
}
private function nodesEvalCore($scalabilityType, $dbUtils)
{
$categories = array();
$series = array();
$datanodes = array();
try {
$filter_execs = DBUtils::getFilterExecs();
$innerQueryWhere = str_replace("e.", "e2.", $this->whereClause);
$innerQueryWhere = str_replace("c.", "c2.", $innerQueryWhere);
$innerQueryWhere = str_replace("p.", "p2.", $innerQueryWhere);
$execs = "SELECT c.datanodes as 'category',e.exec_type,c.vm_OS,c.vm_size,(e.exe_time * (c.cost_hour/3600)) as cost,e.exe_time,c.*\n\t\t\t\t\tFROM aloja2.execs e JOIN aloja2.clusters c USING (id_cluster)\n\t\t\t\t\tLEFT JOIN aloja_ml.predictions p USING (id_exec)\n\t\t\t\t\tINNER JOIN ( SELECT c2.datanodes as 'category',e2.exec_type,c2.vm_OS,c2.vm_size as vmsize,MIN(e2.exe_time) as minexe\n\t\t\t\t\t\t\t\tfrom execs e2 JOIN aloja2.clusters c2 USING (id_cluster)\n\t\t\t\t\t\t\t\tLEFT JOIN aloja_ml.predictions p2 USING (id_exec)\n\t\t\t\t\t\t\t\tWHERE 1 {$innerQueryWhere} GROUP BY c2.datanodes,e2.exec_type,c2.vm_OS,c2.vm_size ) t ON t.minexe = e.exe_time\n\t\t\t\t\tAND t.category = category AND t.vmsize = c.vm_size\n\t\t\t\t\tWHERE 1 {$filter_execs} GROUP BY c.datanodes,e.exec_type,c.vm_OS,c.vm_size\n\t\t\t\t\tORDER BY c.datanodes ASC,c.vm_OS,c.vm_size DESC";
$predExecs = "SELECT c.datanodes as 'category','predicted' as 'exec_type',c.vm_OS,c.vm_size,(e.exe_time * (c.cost_hour/3600)) as cost,e.exe_time,c.*\n\t\t\t\t\tFROM aloja_ml.predictions e JOIN aloja2.clusters c USING (id_cluster)\n\t\t\t\t\tINNER JOIN ( SELECT c2.datanodes as 'category','default' as 'exec_type',c2.vm_OS,c2.vm_size as vmsize,MIN(p2.exe_time) as minexe\n\t\t\t\t\t\t\t\tfrom aloja_ml.predictions p2 JOIN aloja2.clusters c2 USING (id_cluster)\n\t\t\t\t\t\t\t\tWHERE 1 " . str_replace("e2.", "p2.", $innerQueryWhere) . " GROUP BY c2.datanodes,exec_type,c2.vm_OS,c2.vm_size ) t ON t.minexe = e.exe_time\n\t\t\t\t\tAND t.category = category AND t.vmsize = c.vm_size\n\t\t\t\t\tWHERE 1 {$filter_execs} GROUP BY c.datanodes,exec_type,c.vm_OS,c.vm_size\n\t\t\t\t\tORDER BY c.datanodes ASC,c.vm_OS,c.vm_size DESC";
if ($scalabilityType == 'Datasize') {
$execs = str_replace("c2.datanodes", "e2.datasize", $execs);
$execs = str_replace("c.datanodes", "e.datasize", $execs);
$predExecs = str_replace("c2.datanodes", "p2.datasize", $predExecs);
$predExecs = str_replace("c.datanodes", "e.datasize", $predExecs);
}
$params = $this->filters->getFiltersSelectedChoices(array('upred', 'uobsr'));
if ($params['uobsr'] == 1 && $params['upred'] == 1) {
$execs = "({$execs}) UNION ({$predExecs})";
} else {
if ($params['uobsr'] == 0 && $params['upred'] == 1) {
$execs = $predExecs;
}
}
$execs = $dbUtils->get_rows($execs);
$vmSizes = array();
$dataNodes = array();
$vmOS = array();
$execTypes = array();
foreach ($execs as &$exec) {
if (!isset($dataNodes[$exec['category']])) {
$dataNodes[$exec['category']] = 1;
$categories[] = $exec['category'];
}
if (!isset($vmOS[$exec['vm_OS']])) {
$vmOS[$exec['vm_OS']] = 1;
}
if (!isset($execTypes[$exec['exec_type']])) {
$execTypes[$exec['exec_type']] = 1;
}
$vmSizes[$exec['vm_size']][$exec['exec_type']][$exec['vm_OS']][$exec['category']] = array(round($exec['exe_time'], 2), round($exec['cost'], 2));
}
$i = 0;
$seriesColors = array('#7cb5ec', '#434348', '#90ed7d', '#f7a35c', '#8085e9', '#f15c80', '#e4d354', '#2b908f', '#f45b5b', '#91e8e1');
foreach ($vmSizes as $vmSize => $value) {
foreach ($execTypes as $execType => $typevalue) {
foreach ($vmOS as $OS => $osvalue) {
if (isset($vmSizes[$vmSize][$execType][$OS])) {
if ($i == sizeof($seriesColors)) {
$i = 0;
}
$costSeries = array('name' => "{$vmSize} {$execType} {$OS} Run cost", 'type' => 'spline', 'dashStyle' => 'longdash', 'yAxis' => 0, 'data' => array(), 'tooltip' => array('valueSuffix' => ' US$'), 'color' => $seriesColors[$i]);
$timeSeries = array('name' => "{$vmSize} {$execType} {$OS} Run execution time", 'type' => 'spline', 'yAxis' => 1, 'data' => array(), 'tooltip' => array('valueSuffix' => ' s'), 'color' => $seriesColors[$i++]);
foreach ($dataNodes as $datanode => $dvalue) {
$datanodes[] = $datanode;
if (!isset($value[$execType][$OS][$datanode])) {
$costSeries['data'][] = "null";
$timeSeries['data'][] = "null";
} else {
$costSeries['data'][] = $value[$execType][$OS][$datanode][1];
$timeSeries['data'][] = $value[$execType][$OS][$datanode][0];
}
}
$series[] = $timeSeries;
$series[] = $costSeries;
}
}
}
}
} catch (\Exception $e) {
$this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n");
}
if ($scalabilityType == 'Datasize') {
foreach ($categories as &$category) {
$category = Utils::beautifyDatasize($category);
}
}
return array('categories' => json_encode($categories), 'seriesData' => str_replace('"null"', 'null', json_encode($series)), 'datanodess' => $datanodes);
}