public function hdp2CountersAction() { try { $db = $this->container->getDBUtils(); $this->buildFilters(array('bench' => array('default' => null))); $whereClause = $this->filters->getWhereClause(); $benchOptions = $db->get_rows("SELECT DISTINCT bench FROM aloja2.execs e JOIN aloja2.HDI_JOB_details USING (id_exec) WHERE valid = 1"); $discreteOptions = array(); $discreteOptions['bench'][] = 'All'; foreach ($benchOptions as $option) { $discreteOptions['bench'][] = array_shift($option); } $dbUtil = $this->container->getDBUtils(); $message = null; //check the URL $execs = Utils::get_GET_intArray('execs'); if (Utils::get_GET_string('pageTab')) { $type = Utils::get_GET_string('pageTab'); } else { $type = 'SUMMARY'; } $join = "JOIN aloja2.execs e using (id_exec) JOIN aloja2.clusters USING (id_cluster) LEFT JOIN aloja_ml.predictions p USING (id_exec) WHERE job_name NOT IN\n ('TeraGen', 'random-text-writer', 'mahout-examples-0.7-job.jar', 'Create pagerank nodes', 'Create pagerank links') {$whereClause}" . ($execs ? ' AND id_exec IN (' . join(',', $execs) . ') ' : ''); if (isset($_GET['jobid'])) { $join .= " AND JOB_ID = '" . Utils::get_GET_string('jobid') . "' "; } $join .= " LIMIT 10000"; $query = ""; if ($type == 'SUMMARY') { $query = "SELECT e.bench, e.exe_time, c.id_exec, c.JOB_ID, c.job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME, c.TOTAL_MAPS, c.FAILED_MAPS, c.FINISHED_MAPS, c.TOTAL_REDUCES, c.FAILED_REDUCES, c.job_name as CHARTS\n \t\t\tFROM aloja2.HDI_JOB_details c {$join}"; } else { if ($type == "MAP") { $query = "SELECT e.bench, e.exe_time, c.id_exec, JOB_ID, job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME, c.TOTAL_MAPS, c.FAILED_MAPS, c.FINISHED_MAPS, `TOTAL_LAUNCHED_MAPS`,\n \t\t\t`RACK_LOCAL_MAPS`,\n \t\t\t`SPILLED_RECORDS`,\n \t\t\t`MAP_INPUT_RECORDS`,\n \t\t\t`MAP_OUTPUT_RECORDS`,\n \t\t\t`MAP_OUTPUT_BYTES`,\n \t\t\t`MAP_OUTPUT_MATERIALIZED_BYTES`\n \t\t\tFROM aloja2.HDI_JOB_details c {$join}"; } else { if ($type == 'REDUCE') { $query = "SELECT e.bench, e.exe_time, c.id_exec, c.JOB_ID, c.job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME, c.TOTAL_REDUCES, c.FAILED_REDUCES,\n \t\t\t`TOTAL_LAUNCHED_REDUCES`,\n \t\t\t`REDUCE_INPUT_GROUPS`,\n \t\t\t`REDUCE_INPUT_RECORDS`,\n \t\t\t`REDUCE_OUTPUT_RECORDS`,\n \t\t\t`REDUCE_SHUFFLE_BYTES`,\n \t\t\t`COMBINE_INPUT_RECORDS`,\n \t\t\t`COMBINE_OUTPUT_RECORDS`\n \t\t\tFROM aloja2.HDI_JOB_details c {$join}"; } else { if ($type == 'FILE-IO') { $query = "SELECT e.bench, e.exe_time, c.id_exec, c.JOB_ID, c.job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME,\n \t\t\t`SLOTS_MILLIS_MAPS`,\n \t\t\t`SLOTS_MILLIS_REDUCES`,\n \t\t\t`SPLIT_RAW_BYTES`,\n \t\t\t`FILE_BYTES_WRITTEN`,\n \t\t\t`FILE_BYTES_READ`,\n \t\t\t`WASB_BYTES_WRITTEN`,\n \t\t\t`WASB_BYTES_READ`,\n \t\t\t`BYTES_READ`,\n \t\t\t`BYTES_WRITTEN`\n \t\t\tFROM aloja2.HDI_JOB_details c {$join}"; } else { if ($type == 'DETAIL') { $query = "SELECT e.bench, e.exe_time, c.* FROM aloja_logs.aloja2 c {$join}"; } else { if ($type == "TASKS") { $query = "SELECT e.bench, e.exe_time, j.job_name, c.* FROM aloja_logs.HDI_JOB_tasks c\n \t\t\tJOIN aloja2.HDI_JOB_details j USING(id_exec,JOB_ID) {$join} "; // $taskStatusOptions = $db->get_rows("SELECT DISTINCT TASK_STATUS FROM HDI_JOB_tasks JOIN aloja2.execs USING (id_exec) WHERE valid = 1"); // $typeOptions = $db->get_rows("SELECT DISTINCT TASK_TYPE FROM HDI_JOB_tasks JOIN aloja2.execs USING (id_exec) WHERE valid = 1"); // // $discreteOptions['TASK_STATUS'][] = 'All'; // $discreteOptions['TASK_TYPE'][] = 'All'; // foreach($taskStatusOptions as $option) { // $discreteOptions['TASK_STATUS'][] = array_shift($option); // } // foreach($typeOptions as $option) { // $discreteOptions['TASK_TYPE'][] = array_shift($option); // } $discreteOptions['TASK_STATUS'] = array('All', 'SUCCEEDED'); $discreteOptions['TASK_TYPE'] = array('All', 'MAP', 'REDUCE'); } else { throw new \Exception('Unknown type!'); } } } } } } $exec_rows = $dbUtil->get_rows($query); if (count($exec_rows) > 0) { $show_in_result_counters = array('id_exec' => 'ID', 'JOB_ID' => 'JOBID', 'bench' => 'Bench', 'job_name' => 'JOBNAME'); $show_in_result_counters = Utils::generate_show($show_in_result_counters, $exec_rows, 4); } } catch (\Exception $e) { $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n"); } return $this->render('repositoryViews/hdp2counters.html.twig', array('theaders' => isset($show_in_result_counters) ? $show_in_result_counters : array(), 'message' => $message, 'title' => 'Hadoop Jobs and Tasks Execution Counters', 'type' => $type, 'execs' => $execs, 'execsParam' => isset($_GET['execs']) ? Utils::get_GET_intArray('execs') : '', 'discreteOptions' => $discreteOptions, 'hdp2' => true)); }
public function hdp2CountersDataAction() { $db = $this->container->getDBUtils(); $this->buildFilters(array('bench' => array('table' => 'execs', 'default' => null, 'type' => 'selectMultiple'))); $whereClause = $this->filters->getWhereClause(); try { //check the URL $execs = Utils::get_GET_intArray('execs'); if (!($type = Utils::get_GET_string('pageTab'))) { $type = 'SUMMARY'; } $join = "JOIN aloja2.execs e using (id_exec) LEFT JOIN aloja_ml.predictions p USING (id_exec) WHERE e.valid = 1 AND job_name NOT IN\n ('TeraGen', 'random-text-writer', 'mahout-examples-0.7-job.jar', 'Create pagerank nodes', 'Create pagerank links') {$whereClause}" . ($execs ? ' AND id_exec IN (' . join(',', $execs) . ') ' : '') . " LIMIT 10000"; if ($type == 'SUMMARY') { $query = "SELECT e.bench, e.exe_time, c.id_exec, c.JOB_ID, c.job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME, c.TOTAL_MAPS, c.FAILED_MAPS, c.FINISHED_MAPS, c.TOTAL_REDUCES, c.FAILED_REDUCES, c.job_name as CHARTS,\n \t\t\te.perf_details\n \t\t\tFROM aloja2.HDI_JOB_details c {$join}"; } elseif ($type == 'MAP') { $query = "SELECT e.bench, e.exe_time, c.id_exec, JOB_ID, job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME, c.TOTAL_MAPS, c.FAILED_MAPS, c.FINISHED_MAPS, `TOTAL_LAUNCHED_MAPS`,\n \t\t\t`RACK_LOCAL_MAPS`,\n \t\t\t`SPILLED_RECORDS`,\n \t\t\t`MAP_INPUT_RECORDS`,\n \t\t\t`MAP_OUTPUT_RECORDS`,\n \t\t\t`MAP_OUTPUT_BYTES`,\n \t\t\t`MAP_OUTPUT_MATERIALIZED_BYTES`,\n \t\t\te.perf_details\n \t\t\tFROM aloja2.HDI_JOB_details c {$join}"; } elseif ($type == 'REDUCE') { $query = "SELECT e.bench, e.exe_time, c.id_exec, c.JOB_ID, c.job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME, c.TOTAL_REDUCES, c.FAILED_REDUCES,\n \t\t\t`TOTAL_LAUNCHED_REDUCES`,\n \t\t\t`REDUCE_INPUT_GROUPS`,\n \t\t\t`REDUCE_INPUT_RECORDS`,\n \t\t\t`REDUCE_OUTPUT_RECORDS`,\n \t\t\t`REDUCE_SHUFFLE_BYTES`,\n \t\t\t`COMBINE_INPUT_RECORDS`,\n \t\t\t`COMBINE_OUTPUT_RECORDS`,\n \t\t\te.perf_details\n \t\t\tFROM aloja2.HDI_JOB_details c {$join}"; } elseif ($type == 'FILE-IO') { $query = "SELECT e.bench, e.exe_time, c.id_exec, c.JOB_ID, c.job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME,\n \t\t\t`SLOTS_MILLIS_MAPS`,\n \t\t\t`SLOTS_MILLIS_REDUCES`,\n \t\t\t`SPLIT_RAW_BYTES`,\n \t\t\t`FILE_BYTES_WRITTEN`,\n \t\t\t`FILE_BYTES_READ`,\n \t\t\t`WASB_BYTES_WRITTEN`,\n \t\t\t`WASB_BYTES_READ`,\n \t\t\t`BYTES_READ`,\n \t\t\t`BYTES_WRITTEN`,\n \t\t\te.perf_details\n \t\t\tFROM aloja2.HDI_JOB_details c {$join}"; } elseif ($type == 'DETAIL') { $query = "SELECT e.bench, e.exe_time, c.*,e.perf_details FROM aloja2.HDI_JOB_details c {$join}"; } elseif ($type == 'TASKS') { $query = "SELECT e.bench, e.exe_time, j.job_name, c.*,e.perf_details FROM aloja_logs.HDI_JOB_tasks c\n \t\t\tJOIN aloja2.HDI_JOB_details j USING(id_exec,JOB_ID) {$join} "; } else { throw new \Exception('Unknown type!'); } $exec_rows = $db->get_rows($query); if (count($exec_rows) > 0) { $show_in_result_counters = array('id_exec' => 'ID', 'JOB_ID' => 'JOBID', 'bench' => 'Bench', 'job_name' => 'JOBNAME'); $show_in_result_counters = Utils::generate_show($show_in_result_counters, $exec_rows, 4); $jsonData = Utils::generateJSONTable($exec_rows, $show_in_result_counters, 0, 'COUNTER'); header('Content-Type: application/json'); echo json_encode(array('aaData' => $jsonData)); // if (count($exec_rows) > 10000) { // $message .= 'WARNING, large resulset, please limit the query! Rows: '.count($exec_rows); // } } else { echo 'No data available'; } } catch (\Exception $e) { exit($e->getMessage()); echo 'No data available'; /*$noData = array(); for($i = 0; $i<=sizeof($show_in_result); ++$i) $noData[] = 'error'; echo json_encode(array('aaData' => array($noData)));*/ } }