public function mlnewconfigsAction()
{
$jsonData = $jsonHeader = $configs = $jsonNewconfs = $jsonNewconfsHeader = '[]';
$message = $instance = $config = $model_info = $slice_info = '';
$max_x = $max_y = 0;
$must_wait = 'NO';
try {
$dbml = new \PDO($this->container->get('config')['db_conn_chain'], $this->container->get('config')['mysql_user'], $this->container->get('config')['mysql_pwd']);
$dbml->setAttribute(\PDO::ATTR_ERRMODE, \PDO::ERRMODE_EXCEPTION);
$dbml->setAttribute(\PDO::ATTR_EMULATE_PREPARES, false);
$db = $this->container->getDBUtils();
$where_configs = '';
// FIXME - This must be counted BEFORE building filters, as filters inject rubbish in GET when there are no parameters...
$instructions = count($_GET) <= 1;
// Where_Configs and Manual Presets
if (count($_GET) <= 1 || count($_GET) == 2 && array_key_exists('learn', $_GET)) {
$_GET['id_cluster'] = $params['id_cluster'] = array('3', '5', '8');
$where_configs .= ' AND id_cluster IN (3,5,8)';
//$_GET['bench'] = $params['bench'] = array('terasort'); $where_configs .= ' AND bench IN ("terasort")';
//$_GET['disk'] = $params['disk'] = array('HDD','SSD'); $where_configs .= ' AND disk IN ("HDD","SSD")';
$_GET['blk_size'] = $params['blk_size'] = array('64', '128', '256');
$where_configs .= ' AND blk_size IN ("64","128","256")';
$_GET['iofilebuf'] = $params['iofilebuf'] = array('32768', '65536', '131072');
$where_configs .= ' AND iofilebuf IN ("32768","65536","131072")';
$_GET['comp'] = $params['comp'] = array('0');
$where_configs .= ' AND comp IN ("0")';
$_GET['replication'] = $params['replication'] = array('1');
$where_configs .= ' AND replication IN ("1")';
//$_GET['hadoop_version'] = $params['hadoop_version'] = array('1','1.03','2'); $where_configs .= ' AND hadoop_version IN ("1","1.03","2")';
//$_GET['bench_type'] = $params['bench_type'] = array('HiBench'); $where_configs .= ' AND bench_type IN ("HiBench")';
$_GET['datanodes'] = $params['datanodes'] = array('3');
// $where_configs .= ' AND datanodes = 3';
$_GET['vm_OS'] = $params['vm_OS'] = array('linux');
// $where_configs .= ' AND vm_OS = "linux"';
$_GET['vm_size'] = $params['vm_size'] = array('SYS-6027R-72RF');
// $where_configs .= ' AND vm_size = "SYS-6027R-72RF"';
$_GET['vm_cores'] = $params['vm_cores'] = array('12');
// $where_configs .= ' AND vm_cores = 12';
$_GET['vm_RAM'] = $params['vm_RAM'] = array('128');
// $where_configs .= ' AND vm_RAM = 128';
$_GET['type'] = $params['type'] = array('On-premise');
// $where_configs .= ' AND type = "On-premise"';
$_GET['provider'] = $params['provider'] = array('on-premise');
// $where_configs .= ' AND provider = "on-premise"';
$_GET['datefrom'] = $params['datefrom'] = '';
$_GET['dateto'] = $params['dateto'] = '';
$_GET['maxexetime'] = $params['maxexetime'] = 20000;
$_GET['minexetime'] = $params['minexetime'] = 1;
} else {
$param_names_whereconfig = array('bench', 'net', 'disk', 'maps', 'iosf', 'replication', 'iofilebuf', 'comp', 'blk_size', 'id_cluster', 'bench_type', 'hadoop_version', 'datasize', 'scale_factor');
foreach ($param_names_whereconfig as $p) {
MLNewconfigsController::add_where_configs($p, $where_configs);
}
$where_configs .= (isset($_GET['datefrom']) && $_GET['datefrom'] != '' ? ' AND start_time >= ' . $_GET['datefrom'] : '') . (isset($_GET['dateto']) && $_GET['dateto'] != '' ? ' AND end_time <= ' . $_GET['dateto'] : '') . (isset($_GET['minexetime']) && $_GET['minexetime'] != '' ? ' AND exe_time >= ' . $_GET['minexetime'] : '') . (isset($_GET['maxexetime']) && $_GET['maxexetime'] != '' ? ' AND exe_time <= ' . $_GET['maxexetime'] : '') . (isset($_GET['valid']) ? ' AND valid = ' . $_GET['valid'] : '') . (isset($_GET['filter']) ? ' AND filter = ' . $_GET['filter'] : '');
}
// Real fetching of parameters
$params = array();
$param_names = array('bench', 'net', 'disk', 'maps', 'iosf', 'replication', 'iofilebuf', 'comp', 'blk_size', 'id_cluster', 'datanodes', 'vm_OS', 'vm_cores', 'vm_RAM', 'provider', 'vm_size', 'type', 'bench_type', 'hadoop_version', 'datasize', 'scale_factor');
// Order is important
foreach ($param_names as $p) {
$params[$p] = MLNewconfigsController::read_params($p);
sort($params[$p]);
}
$params_additional = array();
$param_names_additional = array('datefrom', 'dateto', 'minexetime', 'maxexetime', 'valid', 'filter');
// Order is important
foreach ($param_names_additional as $p) {
$params_additional[$p] = MLNewconfigsController::read_params($p);
}
$learn_param = array_key_exists('learn', $_GET) ? $_GET['learn'] : 'regtree';
$param_id_cluster = $params['id_cluster'];
unset($params['id_cluster']);
// Exclude the param from now on
$where_configs = str_replace("AND .", "AND ", $where_configs);
// Semi-Dummy Filters (For ModelInfo and SimpleInstance)
$this->buildFilters(array('learn' => array('type' => 'selectOne', 'default' => array('regtree'), 'label' => 'Learning method: ', 'generateChoices' => function () {
return array('regtree', 'nneighbours', 'nnet', 'polyreg');
}, 'beautifier' => function ($value) {
$labels = array('regtree' => 'Regression Tree', 'nneighbours' => 'k-NN', 'nnet' => 'NNets', 'polyreg' => 'PolyReg-3');
return $labels[$value];
}, 'parseFunction' => function () {
$choice = isset($_GET['learn']) ? $_GET['learn'] : array('regtree');
return array('whereClause' => '', 'currentChoice' => $choice);
}, 'filterGroup' => 'MLearning'), 'minexetime' => array('default' => 1), 'maxexetime' => array('default' => 20000), 'datefrom' => array('default' => ''), 'dateto' => array('default' => ''), 'valid' => array('default' => 1), 'filter' => array('default' => 1), 'prepares' => array('default' => 0)));
$this->buildFilterGroups(array('MLearning' => array('label' => 'Machine Learning', 'tabOpenDefault' => true, 'filters' => array('learn'))));
if ($instructions) {
MLUtils::getIndexNewconfs($jsonNewconfs, $jsonNewconfsHeader, $dbml);
$params['id_cluster'] = $param_id_cluster;
$return_params = array('selected' => 'mlnewconfigs', 'instructions' => 'YES', 'jsonData' => $jsonData, 'jsonHeader' => $jsonHeader, 'configs' => $configs, 'newconfs' => $jsonNewconfs, 'header_newconfs' => $jsonNewconfsHeader, 'must_wait' => $must_wait, 'options' => MLNewconfigsController::getFilterOptions($db));
foreach ($param_names as $p) {
$return_params[$p] = $params[$p];
}
foreach ($param_names_additional as $p) {
$return_params[$p] = $params_additional[$p];
}
echo $this->container->getTwig()->render('mltemplate/mlnewconfigs.html.twig', $return_params);
return;
}
// compose instance
$model_info = MLUtils::generateModelInfo($this->filters, $param_names, $params, true, true);
$param_names_aux = array_diff($param_names, array('id_cluster'));
$instance = MLUtils::generateSimpleInstance($this->filters, $param_names_aux, $params, true, true);
$instances = MLUtils::completeInstances($this->filters, array($instance), $param_names, $params, $db);
$slice_info = MLUtils::generateDatasliceInfo($this->filters, $param_names_additional, $params_additional);
$config = $model_info . ' ' . $learn_param . ' ' . $slice_info . ' newminconfs';
if ($learn_param == 'regtree') {
$learn_method = 'aloja_regtree';
$learn_options = 'prange=0,20000';
} else {
if ($learn_param == 'nneighbours') {
$learn_method = 'aloja_nneighbors';
$learn_options = 'kparam=3';
} else {
if ($learn_param == 'nnet') {
$learn_method = 'aloja_nnet';
$learn_options = 'prange=0,20000';
} else {
if ($learn_param == 'polyreg') {
$learn_method = 'aloja_linreg';
$learn_options = 'ppoly=3:prange=0,20000';
}
}
}
}
$cache_ds = getcwd() . '/cache/ml/' . md5($config) . '-cache.csv';
$is_cached_mysql = $dbml->query("SELECT count(*) as num FROM aloja_ml.learners WHERE id_learner = '" . md5($config . "M") . "'");
$tmp_result = $is_cached_mysql->fetch();
$is_cached = $tmp_result['num'] > 0;
$is_cached_mysql = $dbml->query("SELECT count(*) as num FROM aloja_ml.minconfigs WHERE id_minconfigs = '" . md5($config . 'R') . "' AND id_learner = '" . md5($config . "M") . "'");
$tmp_result = $is_cached_mysql->fetch();
$is_cached = $is_cached && $tmp_result['num'] > 0;
$in_process = file_exists(getcwd() . '/cache/ml/' . md5($config) . '.lock');
$finished_process = file_exists(getcwd() . '/cache/ml/' . md5($config) . '.fin');
// Create Models and Predictions
if (!$is_cached && !$in_process && !$finished_process) {
// get headers for csv
$header_names = array('id_exec' => 'ID', 'bench' => 'Benchmark', 'exe_time' => 'Exe.Time', 'e.net' => 'Net', 'e.disk' => 'Disk', 'maps' => 'Maps', 'iosf' => 'IO.SFac', 'replication' => 'Rep', 'iofilebuf' => 'IO.FBuf', 'comp' => 'Comp', 'blk_size' => 'Blk.size', 'datanodes' => 'Datanodes', 'c.vm_OS' => 'VM.OS', 'c.vm_cores' => 'VM.Cores', 'c.vm_RAM' => 'VM.RAM', 'c.provider' => 'Provider', 'c.vm_size' => 'VM.Size', 'type' => 'Type', 'bench_type' => 'Bench.Type', 'hadoop_version' => 'Hadoop.Version', 'IFNULL(datasize,0)' => 'Datasize', 'scale_factor' => 'Scale.Factor');
$added_names = array('maxtxkbs' => 'Net.maxtxKB.s', 'maxrxkbs' => 'Net.maxrxKB.s', 'maxtxpcks' => 'Net.maxtxPck.s', 'maxrxpcks' => 'Net.maxrxPck.s', 'maxtxcmps' => 'Net.maxtxCmp.s', 'maxrxcmps' => 'Net.maxrxCmp.s', 'maxrxmscts' => 'Net.maxrxmsct.s', 'maxtps' => 'Disk.maxtps', 'maxsvctm' => 'Disk.maxsvctm', 'maxrds' => 'Disk.maxrd.s', 'maxwrs' => 'Disk.maxwr.s', 'maxrqsz' => 'Disk.maxrqsz', 'maxqusz' => 'Disk.maxqusz', 'maxawait' => 'Disk.maxawait', 'maxutil' => 'Disk.maxutil');
// dump the result to csv
$query = "SELECT " . implode(",", array_keys($header_names)) . ",\n\t\t\t\t\tn.maxtxkbs, n.maxrxkbs, n.maxtxpcks, n.maxrxpcks, n.maxtxcmps, n.maxrxcmps, n.maxrxmscts,\n\t\t\t\t\td.maxtps, d.maxsvctm, d.maxrds, d.maxwrs, d.maxrqsz, d.maxqusz, d.maxawait, d.maxutil\n\t\t\t\t\tFROM aloja2.execs AS e LEFT JOIN aloja2.clusters AS c ON e.id_cluster = c.id_cluster,\n\t\t\t\t\t(\n\t\t\t\t\t SELECT MAX(n1.`maxtxkB/s`) AS maxtxkbs, MAX(n1.`maxrxkB/s`) AS maxrxkbs,\n\t\t\t\t\t MAX(n1.`maxtxpck/s`) AS maxtxpcks, MAX(n1.`maxrxpck/s`) AS maxrxpcks,\n\t\t\t\t\t MAX(n1.`maxtxcmp/s`) AS maxtxcmps, MAX(n1.`maxrxcmp/s`) AS maxrxcmps,\n\t\t\t\t\t MAX(n1.`maxrxmcst/s`) AS maxrxmscts,\n\t\t\t\t\t e1.net AS net, c1.vm_cores, c1.vm_RAM, c1.vm_size, c1.vm_OS, c1.provider\n\t\t\t\t\t FROM aloja2.precal_network_metrics AS n1,\n\t\t\t\t\t aloja2.execs AS e1 LEFT JOIN aloja2.clusters AS c1 ON e1.id_cluster = c1.id_cluster\n\t\t\t\t\t WHERE e1.id_exec = n1.id_exec\n\t\t\t\t\t GROUP BY e1.net, c1.vm_cores, c1.vm_RAM, c1.vm_size, c1.vm_OS, c1.provider\n\t\t\t\t\t) AS n,\n\t\t\t\t\t(\n\t\t\t\t\t SELECT MAX(d1.maxtps) AS maxtps, MAX(d1.maxsvctm) as maxsvctm,\n\t\t\t\t\t MAX(d1.`maxrd_sec/s`) as maxrds, MAX(d1.`maxwr_sec/s`) as maxwrs,\n\t\t\t\t\t MAX(d1.maxrq_sz) as maxrqsz, MAX(d1.maxqu_sz) as maxqusz,\n\t\t\t\t\t MAX(d1.maxawait) as maxawait, MAX(d1.`max%util`) as maxutil,\n\t\t\t\t\t e2.disk AS disk, c1.vm_cores, c1.vm_RAM, c1.vm_size, c1.vm_OS, c1.provider\n\t\t\t\t\t FROM aloja2.precal_disk_metrics AS d1,\n\t\t\t\t\t aloja2.execs AS e2 LEFT JOIN aloja2.clusters AS c1 ON e2.id_cluster = c1.id_cluster\n\t\t\t\t\t WHERE e2.id_exec = d1.id_exec\n\t\t\t\t\t GROUP BY e2.disk, c1.vm_cores, c1.vm_RAM, c1.vm_size, c1.vm_OS, c1.provider\n\t\t\t\t\t) AS d\n\t\t\t\t\tWHERE e.net = n.net AND c.vm_cores = n.vm_cores AND c.vm_RAM = n.vm_RAM AND c.vm_size = n.vm_size\n\t\t\t\t\tAND c.vm_OS = n.vm_OS AND c.provider = n.provider AND e.disk = d.disk AND c.vm_cores = d.vm_cores\n\t\t\t\t\tAND c.vm_RAM = d.vm_RAM AND c.vm_size = d.vm_size AND c.vm_OS = d.vm_OS AND c.provider = d.provider\n\t\t\t\t\tAND hadoop_version IS NOT NULL" . $where_configs . ";";
$rows = $db->get_rows($query);
if (empty($rows)) {
throw new \Exception('No data matches with your critteria.');
}
$fp = fopen($cache_ds, 'w');
fputcsv($fp, array_values(array_merge($header_names, $added_names)), ',', '"');
foreach ($rows as $row) {
//$row['id_cluster'] = "Cl".$row['id_cluster']; // Cluster is numerically codified...
$row['comp'] = "Cmp" . $row['comp'];
// Compression is numerically codified...
fputcsv($fp, array_values($row), ',', '"');
}
// run the R processor
$vin = "Benchmark,Net,Disk,Maps,IO.SFac,Rep,IO.FBuf,Comp,Blk.size,Datanodes,VM.OS,VM.Cores,VM.RAM,Provider,VM.Size,Type,Bench.Type,Hadoop.Version,Datasize,Scale.Factor,Net.maxtxKB.s,Net.maxrxKB.s,Net.maxtxPck.s,Net.maxrxPck.s,Net.maxtxCmp.s,Net.maxrxCmp.s,Net.maxrxmsct.s,Disk.maxtps,Disk.maxsvctm,Disk.maxrd.s,Disk.maxwr.s,Disk.maxrqsz,Disk.maxqusz,Disk.maxawait,Disk.maxutil";
exec('cd ' . getcwd() . '/cache/ml; touch ' . md5($config) . '.lock');
$command = getcwd() . '/resources/queue -c "cd ' . getcwd() . '/cache/ml; ../../resources/aloja_cli.r -d ' . $cache_ds . ' -m ' . $learn_method . ' -p ' . $learn_options . ':saveall=' . md5($config . "F") . ':vin=\'' . $vin . '\' >debug1.txt 2>&1 && ';
$count = 1;
foreach ($instances as $inst) {
$command = $command . '../../resources/aloja_cli.r -m aloja_predict_instance -l ' . md5($config . "F") . ' -p inst_predict=\'' . $inst . '\':saveall=' . md5($config . "D") . '-' . $count++ . ':vin=\'' . $vin . '\' >>debug2.txt 2>&1 && ';
}
$command = $command . ' head -1 ' . md5($config . "D") . '-1-dataset.data >' . md5($config . "D") . '-dataset.data 2>>debug2-1.txt && ';
$command = $command . ' cat ' . md5($config . "D") . '*-dataset.data >' . md5($config . "D") . '-aux.data 2>>debug2-1.txt && ';
$command = $command . ' grep -v "ID" ' . md5($config . "D") . '*-aux.data >>' . md5($config . "D") . '-dataset.data 2>>debug2-1.txt && ';
$command = $command . '../../resources/aloja_cli.r -d ' . md5($config . "D") . '-dataset.data -m ' . $learn_method . ' -p ' . $learn_options . ':saveall=' . md5($config . "M") . ':vin=\'' . $vin . '\' >debug3.txt 2>&1 && ';
$command = $command . '../../resources/aloja_cli.r -m aloja_minimal_instances -l ' . md5($config . "M") . ' -p saveall=' . md5($config . 'R') . ':kmax=200 >debug4.txt 2>&1; rm -f ' . md5($config) . '.lock; touch ' . md5($config) . '.fin" >debug4.tmp 2>&1 &';
exec($command);
sleep(2);
}
$in_process = file_exists(getcwd() . '/cache/ml/' . md5($config) . '.lock');
if ($in_process) {
$must_wait = "YES";
throw new \Exception('WAIT');
}
$learners = array();
$learners[] = md5($config . "F");
$learners[] = md5($config . "M");
foreach ($learners as $learner_1) {
// Save learning model to DB, with predictions
$is_cached_mysql = $dbml->query("SELECT id_learner FROM aloja_ml.learners WHERE id_learner = '" . $learner_1 . "'");
$tmp_result = $is_cached_mysql->fetch();
if ($tmp_result['id_learner'] != $learner_1) {
// register model to DB
$query = "INSERT IGNORE INTO aloja_ml.learners (id_learner,instance,model,algorithm,dataslice)";
$query = $query . " VALUES ('" . $learner_1 . "','" . $instance . "','" . substr($model_info, 1) . "','" . $learn_param . "','" . $slice_info . "');";
if ($dbml->query($query) === FALSE) {
throw new \Exception('Error when saving model into DB');
}
// read results of the CSV and dump to DB
foreach (array("tt", "tv", "tr") as $value) {
if (($handle = fopen(getcwd() . '/cache/ml/' . $learner_1 . '-' . $value . '.csv', 'r')) !== FALSE) {
$header = fgetcsv($handle, 1000, ",");
$token = 0;
$query = "INSERT IGNORE INTO aloja_ml.predictions (\n\t\t\t\t\t\t\t\tid_exec,exe_time,bench,net,disk,maps,iosf,replication,iofilebuf,comp,blk_size,\n\t\t\t\t\t\t\t\tdatanodes,vm_OS,vm_cores,vm_RAM,provider,vm_size,type,bench_type,hadoop_version,\n\t\t\t\t\t\t\t\tdatasize,scale_factor,\n\t\t\t\t\t\t\t\tnet_maxtxkbs,net_maxrxkbs,net_maxtxpcks,net_maxrxpcks,net_maxtxcmps,net_maxrxcmps,net_maxrxmscts,\n\t\t\t\t\t\t\t\tdisk_maxtps,disk_maxsvctm,disk_maxrds,disk_maxwrs,disk_maxrqsz,disk_maxqusz,disk_maxawait, disk_maxutil,\n\t\t\t\t\t\t\t\tpred_time,id_learner,instance,predict_code) VALUES ";
while (($data = fgetcsv($handle, 1000, ",")) !== FALSE) {
$specific_instance = implode(",", array_slice($data, 2, 35));
$specific_data = implode(",", $data);
$specific_data = preg_replace('/,Cmp(\\d+),/', ',${1},', $specific_data);
$specific_data = preg_replace('/,Cl(\\d+),/', ',${1},', $specific_data);
$specific_data = str_replace(",", "','", $specific_data);
$query_var = "SELECT count(*) as num FROM aloja_ml.predictions WHERE instance = '" . $specific_instance . "' AND id_learner = '" . $learner_1 . "'";
$result = $dbml->query($query_var);
$row = $result->fetch();
// Insert instance values
if ($row['num'] == 0) {
if ($token != 0) {
$query = $query . ",";
}
$token = 1;
$query = $query . "('" . $specific_data . "','" . $learner_1 . "','" . $specific_instance . "','" . ($value == 'tt' ? 3 : ($value == 'tv' ? 2 : 1)) . "') ";
}
}
if ($dbml->query($query) === FALSE) {
throw new \Exception('Error when saving into DB');
}
fclose($handle);
}
}
// Remove temporal files
$output = shell_exec('rm -f ' . getcwd() . '/cache/ml/' . $learner_1 . '*.{dat,csv}');
}
}
// Save minconfigs to DB, with props and centers
$is_cached_mysql = $dbml->query("SELECT id_minconfigs FROM aloja_ml.minconfigs WHERE id_minconfigs = '" . md5($config . 'R') . "'");
$tmp_result = $is_cached_mysql->fetch();
if ($tmp_result['id_minconfigs'] != md5($config . 'R')) {
// register minconfigs to DB
$query = "INSERT IGNORE INTO aloja_ml.minconfigs (id_minconfigs,id_learner,instance,model,is_new)";
$query = $query . " VALUES ('" . md5($config . 'R') . "','" . md5($config . 'M') . "','" . $instance . "','" . substr($model_info, 1) . "','1');";
if ($dbml->query($query) === FALSE) {
throw new \Exception('Error when saving minconfis into DB');
}
$clusters = array();
// Save results of the CSV - MAE or RAE
if (file_exists(getcwd() . '/cache/ml/' . md5($config . 'R') . '-raes.csv')) {
$error_file = 'raes.csv';
} else {
$error_file = 'maes.csv';
}
$handle = fopen(getcwd() . '/cache/ml/' . md5($config . 'R') . '-' . $error_file, 'r');
while (($data = fgetcsv($handle, 1000, ",")) !== FALSE) {
$cluster = (int) $data[0];
if ($error_file == 'raes.csv') {
$error_mae = 'NULL';
$error_rae = (double) $data[1];
}
if ($error_file == 'maes.csv') {
$error_mae = (double) $data[1];
$error_rae = 'NULL';
}
// register minconfigs_props to DB
$query = "INSERT INTO aloja_ml.minconfigs_props (id_minconfigs,cluster,MAE,RAE)";
$query = $query . " VALUES ('" . md5($config . 'R') . "','" . $cluster . "','" . $error_mae . "','" . $error_rae . "');";
if ($dbml->query($query) === FALSE) {
throw new \Exception('Error when saving minconfis into DB');
}
$clusters[] = $cluster;
}
fclose($handle);
// Save results of the CSV - Configs
$handle_sizes = fopen(getcwd() . '/cache/ml/' . md5($config . 'R') . '-sizes.csv', 'r');
foreach ($clusters as $cluster) {
// Get supports from sizes
$sizes = fgetcsv($handle_sizes, 1000, ",");
// Get clusters
$handle = fopen(getcwd() . '/cache/ml/' . md5($config . 'R') . '-dsk' . $cluster . '.csv', 'r');
$header = fgetcsv($handle, 1000, ",");
$i = 0;
while (($data = fgetcsv($handle, 1000, ",")) !== FALSE) {
$subdata1 = array_slice($data, 0, 11);
$subdata2 = array_slice($data, 18, 4);
$specific_data = implode(',', array_merge($subdata1, $subdata2));
$specific_data = preg_replace('/,Cmp(\\d+),/', ',${1},', $specific_data);
$specific_data = preg_replace('/,Cl(\\d+),/', ',${1},', $specific_data);
$specific_data = preg_replace('/,Cl(\\d+)/', ',${1}', $specific_data);
$specific_data = str_replace(",", "','", $specific_data);
// register minconfigs_props to DB
$query = "INSERT INTO aloja_ml.minconfigs_centers (id_minconfigs,cluster,id_exec,exe_time,bench,net,disk,maps,iosf,replication,iofilebuf,comp,blk_size,bench_type,hadoop_version,datasize,scale_factor,support)";
$query = $query . " VALUES ('" . md5($config . 'R') . "','" . $cluster . "','" . $specific_data . "','" . $sizes[$i++] . "');";
if ($dbml->query($query) === FALSE) {
throw new \Exception('Error when saving centers into DB');
}
}
fclose($handle);
}
fclose($handle_sizes);
// Store file model to DB
$filemodel = getcwd() . '/cache/ml/' . md5($config . 'F') . '-object.rds';
$fp = fopen($filemodel, 'r');
$content = fread($fp, filesize($filemodel));
$content = addslashes($content);
fclose($fp);
$query = "INSERT INTO aloja_ml.model_storage (id_hash,type,file) VALUES ('" . md5($config . 'F') . "','learner','" . $content . "');";
if ($dbml->query($query) === FALSE) {
throw new \Exception('Error when saving file model into DB');
}
$filemodel = getcwd() . '/cache/ml/' . md5($config . 'M') . '-object.rds';
$fp = fopen($filemodel, 'r');
$content = fread($fp, filesize($filemodel));
$content = addslashes($content);
fclose($fp);
$query = "INSERT INTO aloja_ml.model_storage (id_hash,type,file) VALUES ('" . md5($config . 'M') . "','learner','" . $content . "');";
if ($dbml->query($query) === FALSE) {
throw new \Exception('Error when saving file model into DB');
}
$filemodel = getcwd() . '/cache/ml/' . md5($config . 'R') . '-object.rds';
$fp = fopen($filemodel, 'r');
$content = fread($fp, filesize($filemodel));
$content = addslashes($content);
fclose($fp);
$query = "INSERT INTO aloja_ml.model_storage (id_hash,type,file) VALUES ('" . md5($config . 'R') . "','minconf','" . $content . "');";
if ($dbml->query($query) === FALSE) {
throw new \Exception('Error when saving file minconf into DB');
}
// Remove temporal files
exec('rm -f ' . getcwd() . '/cache/ml/' . md5($config . 'R') . '*.rds');
exec('rm -f ' . getcwd() . '/cache/ml/' . md5($config . 'R') . '*.dat');
exec('rm -f ' . getcwd() . '/cache/ml/' . md5($config . 'R') . '*.csv');
exec('rm -f ' . getcwd() . '/cache/ml/' . md5($config . 'D') . '*.csv');
exec('rm -f ' . getcwd() . '/cache/ml/' . md5($config . 'D') . '*.dat');
exec('rm -f ' . getcwd() . '/cache/ml/' . md5($config . 'D') . '*.data');
exec('rm -f ' . getcwd() . '/cache/ml/' . md5($config . 'F') . '*.rds');
exec('rm -f ' . getcwd() . '/cache/ml/' . md5($config . 'F') . '*.csv');
exec('rm -f ' . getcwd() . '/cache/ml/' . md5($config . 'F') . '*.dat');
exec('rm -f ' . getcwd() . '/cache/ml/' . md5($config . 'M') . '*.rds');
exec('rm -f ' . getcwd() . '/cache/ml/' . md5($config . 'M') . '*.csv');
exec('rm -f ' . getcwd() . '/cache/ml/' . md5($config . 'M') . '*.dat');
exec('rm -f ' . getcwd() . '/cache/ml/' . md5($config) . '*.csv');
exec('rm -f ' . getcwd() . '/cache/ml/' . md5($config) . '*.dat');
exec('rm -f ' . getcwd() . '/cache/ml/' . md5($config) . '*.fin');
}
// Retrieve minconfig progression results from DB
$header = "id_exec,exe_time,bench,net,disk,maps,iosf,replication,iofilebuf,comp,blk_size,bench_type,hadoop_version,datasize,scale_factor,support";
$header_array = explode(",", $header);
$last_y = 9000000000000000.0;
$configs = '[';
$jsonData = array();
$query = "SELECT cluster, MAE, RAE FROM aloja_ml.minconfigs_props WHERE id_minconfigs='" . md5($config . 'R') . "'";
$result = $dbml->query($query);
foreach ($result as $row) {
// Retrieve minconfig progression results from DB
if ((double) $row['MAE'] > 0) {
$error = (double) $row['MAE'];
} else {
$error = (double) $row['RAE'];
}
$cluster = (int) $row['cluster'];
$new_val = array();
$new_val['x'] = $cluster;
if ($error > $last_y) {
$new_val['y'] = $last_y;
} else {
$last_y = $new_val['y'] = $error;
}
$jsonData[] = $new_val;
// Retrieve minconfig centers from DB
$query_2 = "SELECT " . $header . " FROM aloja_ml.minconfigs_centers WHERE id_minconfigs='" . md5($config . 'R') . "' AND cluster='" . $cluster . "'";
$result_2 = $dbml->query($query_2);
$jsonConfig = '[';
foreach ($result_2 as $row_2) {
$values = '';
foreach ($header_array as $ha) {
$values = $values . ($values != '' ? ',' : '') . '\'' . $row_2[$ha] . '\'';
}
$jsonConfig = $jsonConfig . ($jsonConfig != '[' ? ',' : '') . '[' . $values . ']';
}
$jsonConfig = $jsonConfig . ']';
$configs = $configs . ($configs != '[' ? ',' : '') . $jsonConfig;
}
$configs = $configs . ']';
$jsonData = json_encode($jsonData);
$jsonHeader = '[{title:""},{title:"Est.Time"},{title:"Benchmark"},{title:"Network"},{title:"Disk"},{title:"Maps"},{title:"IO.SF"},{title:"Replicas"},{title:"IO.FBuf"},{title:"Compression"},{title:"Blk.Size"},{title:"Bench.Type"},{title:"Hadoop.Ver"},{title:"Data.Size"},{title:"Scale.Factor"},{title:"Support"}]';
$query = "SELECT MAX(cluster) as mcluster, MAX(MAE) as mmae, MAX(RAE) as mrae FROM aloja_ml.minconfigs_props WHERE id_minconfigs='" . md5($config . 'R') . "'";
$is_cached_mysql = $dbml->query($query);
$tmp_result = $is_cached_mysql->fetch();
$max_x = (double) $tmp_result['mmae'] > 0 ? (double) $tmp_result['mmae'] : (double) $tmp_result['mrae'];
$max_y = (double) $tmp_result['mcluster'];
} catch (\Exception $e) {
if ($e->getMessage() != "WAIT") {
$this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n");
}
$jsonData = $jsonHeader = $configs = '[]';
}
$dbml = null;
$params['id_cluster'] = $param_id_cluster;
$return_params = array('selected' => 'mlnewconfigs', 'jsonData' => $jsonData, 'jsonHeader' => $jsonHeader, 'configs' => $configs, 'newconfs' => $jsonNewconfs, 'header_newconfs' => $jsonNewconfsHeader, 'max_p' => min(array($max_x, $max_y)), 'instance' => $instance, 'id_newconf' => md5($config), 'id_newconf_first' => md5($config . 'F'), 'id_newconf_dataset' => md5($config . 'D'), 'id_newconf_model' => md5($config . 'M'), 'id_newconf_result' => md5($config . 'R'), 'model_info' => $model_info, 'slice_info' => $slice_info, 'learn' => $learn_param, 'must_wait' => $must_wait, 'options' => MLNewconfigsController::getFilterOptions($db));
foreach ($param_names as $p) {
$return_params[$p] = $params[$p];
}
foreach ($param_names_additional as $p) {
$return_params[$p] = $params_additional[$p];
}
echo $this->container->getTwig()->render('mltemplate/mlnewconfigs.html.twig', $return_params);
}
