public function hdp2CountersDataAction() { $db = $this->container->getDBUtils(); try { //check the URL $execs = Utils::get_GET_execs(); if (!($type = Utils::get_GET_string('type'))) { $type = 'SUMMARY'; } $join = "JOIN execs e using (id_exec) WHERE e.valid = 1 AND job_name NOT IN\n ('TeraGen', 'random-text-writer', 'mahout-examples-0.7-job.jar', 'Create pagerank nodes', 'Create pagerank links')" . ($execs ? ' AND id_exec IN (' . join(',', $execs) . ') ' : '') . " LIMIT 10000"; if ($type == 'SUMMARY') { $query = "SELECT e.bench, exe_time, c.id_exec, c.JOB_ID, c.job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME, c.TOTAL_MAPS, c.FAILED_MAPS, c.FINISHED_MAPS, c.TOTAL_REDUCES, c.FAILED_REDUCES, c.job_name as CHARTS\n \t\t\tFROM HDI_JOB_details c {$join}"; } elseif ($type == 'MAP') { $query = "SELECT e.bench, exe_time, c.id_exec, JOB_ID, job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME, c.TOTAL_MAPS, c.FAILED_MAPS, c.FINISHED_MAPS, `TOTAL_LAUNCHED_MAPS`,\n \t\t\t`RACK_LOCAL_MAPS`,\n \t\t\t`SPILLED_RECORDS`,\n \t\t\t`MAP_INPUT_RECORDS`,\n \t\t\t`MAP_OUTPUT_RECORDS`,\n \t\t\t`MAP_OUTPUT_BYTES`,\n \t\t\t`MAP_OUTPUT_MATERIALIZED_BYTES`\n \t\t\tFROM HDI_JOB_details c {$join}"; } elseif ($type == 'REDUCE') { $query = "SELECT e.bench, exe_time, c.id_exec, c.JOB_ID, c.job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME, c.TOTAL_REDUCES, c.FAILED_REDUCES,\n \t\t\t`TOTAL_LAUNCHED_REDUCES`,\n \t\t\t`REDUCE_INPUT_GROUPS`,\n \t\t\t`REDUCE_INPUT_RECORDS`,\n \t\t\t`REDUCE_OUTPUT_RECORDS`,\n \t\t\t`REDUCE_SHUFFLE_BYTES`,\n \t\t\t`COMBINE_INPUT_RECORDS`,\n \t\t\t`COMBINE_OUTPUT_RECORDS`\n \t\t\tFROM HDI_JOB_details c {$join}"; } elseif ($type == 'FILE-IO') { $query = "SELECT e.bench, exe_time, c.id_exec, c.JOB_ID, c.job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME,\n \t\t\t`SLOTS_MILLIS_MAPS`,\n \t\t\t`SLOTS_MILLIS_REDUCES`,\n \t\t\t`SPLIT_RAW_BYTES`,\n \t\t\t`FILE_BYTES_WRITTEN`,\n \t\t\t`FILE_BYTES_READ`,\n \t\t\t`WASB_BYTES_WRITTEN`,\n \t\t\t`WASB_BYTES_READ`,\n \t\t\t`BYTES_READ`,\n \t\t\t`BYTES_WRITTEN`\n \t\t\tFROM HDI_JOB_details c {$join}"; } elseif ($type == 'DETAIL') { $query = "SELECT e.bench, exe_time, c.* FROM HDI_JOB_details c {$join}"; } elseif ($type == 'TASKS') { $query = "SELECT e.bench, exe_time, j.job_name, c.* FROM HDI_JOB_tasks c\n \t\t\tJOIN HDI_JOB_details j USING(id_exec,JOB_ID) {$join} "; } else { throw new \Exception('Unknown type!'); } $exec_rows = $db->get_rows($query); if (count($exec_rows) > 0) { $show_in_result_counters = array('id_exec' => 'ID', 'JOB_ID' => 'JOBID', 'bench' => 'Bench', 'job_name' => 'JOBNAME'); $show_in_result_counters = Utils::generate_show($show_in_result_counters, $exec_rows, 4); $jsonData = Utils::generateJSONTable($exec_rows, $show_in_result_counters, 0, 'COUNTER'); header('Content-Type: application/json'); echo $jsonData; // if (count($exec_rows) > 10000) { // $message .= 'WARNING, large resulset, please limit the query! Rows: '.count($exec_rows); // } } else { $noData = array(); for ($i = 0; $i < 18; ++$i) { $noData[] = 'No Data found'; } ob_start('ob_gzhandler'); echo json_encode(array('aaData' => $noData)); } } catch (Exception $e) { $noData = array(); for ($i = 0; $i <= sizeof($show_in_result); ++$i) { $noData[] = 'error'; } echo json_encode(array('aaData' => array($noData))); } }
public function hdp2CountersAction() { try { $db = $this->container->getDBUtils(); $benchOptions = $db->get_rows("SELECT DISTINCT bench FROM execs e JOIN HDI_JOB_details USING (id_exec) WHERE valid = 1"); $discreteOptions = array(); $discreteOptions['bench'][] = 'All'; foreach ($benchOptions as $option) { $discreteOptions['bench'][] = array_shift($option); } $dbUtil = $this->container->getDBUtils(); $message = null; //check the URL $execs = Utils::get_GET_execs(); if (Utils::get_GET_string('type')) { $type = Utils::get_GET_string('type'); } else { $type = 'SUMMARY'; } $join = "JOIN execs e using (id_exec) WHERE job_name NOT IN\n ('TeraGen', 'random-text-writer', 'mahout-examples-0.7-job.jar', 'Create pagerank nodes', 'Create pagerank links')" . ($execs ? ' AND id_exec IN (' . join(',', $execs) . ') ' : ''); if (isset($_GET['jobid'])) { $join .= " AND JOB_ID = '{$_GET['jobid']}' "; } $join .= " LIMIT 10000"; $query = ""; if ($type == 'SUMMARY') { $query = "SELECT e.bench, exe_time, c.id_exec, c.JOB_ID, c.job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME, c.TOTAL_MAPS, c.FAILED_MAPS, c.FINISHED_MAPS, c.TOTAL_REDUCES, c.FAILED_REDUCES, c.job_name as CHARTS\n \t\t\tFROM HDI_JOB_details c {$join}"; } else { if ($type == "MAP") { $query = "SELECT e.bench, exe_time, c.id_exec, JOB_ID, job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME, c.TOTAL_MAPS, c.FAILED_MAPS, c.FINISHED_MAPS, `TOTAL_LAUNCHED_MAPS`,\n \t\t\t`RACK_LOCAL_MAPS`,\n \t\t\t`SPILLED_RECORDS`,\n \t\t\t`MAP_INPUT_RECORDS`,\n \t\t\t`MAP_OUTPUT_RECORDS`,\n \t\t\t`MAP_OUTPUT_BYTES`,\n \t\t\t`MAP_OUTPUT_MATERIALIZED_BYTES`\n \t\t\tFROM HDI_JOB_details c {$join}"; } else { if ($type == 'REDUCE') { $query = "SELECT e.bench, exe_time, c.id_exec, c.JOB_ID, c.job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME, c.TOTAL_REDUCES, c.FAILED_REDUCES,\n \t\t\t`TOTAL_LAUNCHED_REDUCES`,\n \t\t\t`REDUCE_INPUT_GROUPS`,\n \t\t\t`REDUCE_INPUT_RECORDS`,\n \t\t\t`REDUCE_OUTPUT_RECORDS`,\n \t\t\t`REDUCE_SHUFFLE_BYTES`,\n \t\t\t`COMBINE_INPUT_RECORDS`,\n \t\t\t`COMBINE_OUTPUT_RECORDS`\n \t\t\tFROM HDI_JOB_details c {$join}"; } else { if ($type == 'FILE-IO') { $query = "SELECT e.bench, exe_time, c.id_exec, c.JOB_ID, c.job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME,\n \t\t\t`SLOTS_MILLIS_MAPS`,\n \t\t\t`SLOTS_MILLIS_REDUCES`,\n \t\t\t`SPLIT_RAW_BYTES`,\n \t\t\t`FILE_BYTES_WRITTEN`,\n \t\t\t`FILE_BYTES_READ`,\n \t\t\t`WASB_BYTES_WRITTEN`,\n \t\t\t`WASB_BYTES_READ`,\n \t\t\t`BYTES_READ`,\n \t\t\t`BYTES_WRITTEN`\n \t\t\tFROM HDI_JOB_details c {$join}"; } else { if ($type == 'DETAIL') { $query = "SELECT e.bench, exe_time, c.* FROM HDI_JOB_details c {$join}"; } else { if ($type == "TASKS") { $query = "SELECT e.bench, exe_time, j.job_name, c.* FROM HDI_JOB_tasks c\n \t\t\tJOIN HDI_JOB_details j USING(id_exec,JOB_ID) {$join} "; // $taskStatusOptions = $db->get_rows("SELECT DISTINCT TASK_STATUS FROM HDI_JOB_tasks JOIN execs USING (id_exec) WHERE valid = 1"); // $typeOptions = $db->get_rows("SELECT DISTINCT TASK_TYPE FROM HDI_JOB_tasks JOIN execs USING (id_exec) WHERE valid = 1"); // // $discreteOptions['TASK_STATUS'][] = 'All'; // $discreteOptions['TASK_TYPE'][] = 'All'; // foreach($taskStatusOptions as $option) { // $discreteOptions['TASK_STATUS'][] = array_shift($option); // } // foreach($typeOptions as $option) { // $discreteOptions['TASK_TYPE'][] = array_shift($option); // } $discreteOptions['TASK_STATUS'] = array('All', 'SUCCEEDED'); $discreteOptions['TASK_TYPE'] = array('All', 'MAP', 'REDUCE'); } else { throw new \Exception('Unknown type!'); } } } } } } $exec_rows = $dbUtil->get_rows($query); if (count($exec_rows) > 0) { $show_in_result_counters = array('id_exec' => 'ID', 'JOB_ID' => 'JOBID', 'bench' => 'Bench', 'job_name' => 'JOBNAME'); $show_in_result_counters = Utils::generate_show($show_in_result_counters, $exec_rows, 4); } } catch (\Exception $e) { $this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n"); } echo $this->container->getTwig()->render('counters/hdp2counters.html.twig', array('selected' => 'Hadoop 2 Job Counters', 'theaders' => $show_in_result_counters, 'message' => $message, 'title' => 'Hadoop Jobs and Tasks Execution Counters', 'type' => $type, 'execs' => $execs, 'execsParam' => isset($_GET['execs']) ? $_GET['execs'] : '', 'discreteOptions' => $discreteOptions, 'hdp2' => true)); }