public function hdp2CountersDataAction()
{
$db = $this->container->getDBUtils();
try {
//check the URL
$execs = Utils::get_GET_execs();
if (!($type = Utils::get_GET_string('type'))) {
$type = 'SUMMARY';
}
$join = "JOIN execs e using (id_exec) WHERE e.valid = 1 AND job_name NOT IN\n ('TeraGen', 'random-text-writer', 'mahout-examples-0.7-job.jar', 'Create pagerank nodes', 'Create pagerank links')" . ($execs ? ' AND id_exec IN (' . join(',', $execs) . ') ' : '') . " LIMIT 10000";
if ($type == 'SUMMARY') {
$query = "SELECT e.bench, exe_time, c.id_exec, c.JOB_ID, c.job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME, c.TOTAL_MAPS, c.FAILED_MAPS, c.FINISHED_MAPS, c.TOTAL_REDUCES, c.FAILED_REDUCES, c.job_name as CHARTS\n \t\t\tFROM HDI_JOB_details c {$join}";
} elseif ($type == 'MAP') {
$query = "SELECT e.bench, exe_time, c.id_exec, JOB_ID, job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME, c.TOTAL_MAPS, c.FAILED_MAPS, c.FINISHED_MAPS, `TOTAL_LAUNCHED_MAPS`,\n \t\t\t`RACK_LOCAL_MAPS`,\n \t\t\t`SPILLED_RECORDS`,\n \t\t\t`MAP_INPUT_RECORDS`,\n \t\t\t`MAP_OUTPUT_RECORDS`,\n \t\t\t`MAP_OUTPUT_BYTES`,\n \t\t\t`MAP_OUTPUT_MATERIALIZED_BYTES`\n \t\t\tFROM HDI_JOB_details c {$join}";
} elseif ($type == 'REDUCE') {
$query = "SELECT e.bench, exe_time, c.id_exec, c.JOB_ID, c.job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME, c.TOTAL_REDUCES, c.FAILED_REDUCES,\n \t\t\t`TOTAL_LAUNCHED_REDUCES`,\n \t\t\t`REDUCE_INPUT_GROUPS`,\n \t\t\t`REDUCE_INPUT_RECORDS`,\n \t\t\t`REDUCE_OUTPUT_RECORDS`,\n \t\t\t`REDUCE_SHUFFLE_BYTES`,\n \t\t\t`COMBINE_INPUT_RECORDS`,\n \t\t\t`COMBINE_OUTPUT_RECORDS`\n \t\t\tFROM HDI_JOB_details c {$join}";
} elseif ($type == 'FILE-IO') {
$query = "SELECT e.bench, exe_time, c.id_exec, c.JOB_ID, c.job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME,\n \t\t\t`SLOTS_MILLIS_MAPS`,\n \t\t\t`SLOTS_MILLIS_REDUCES`,\n \t\t\t`SPLIT_RAW_BYTES`,\n \t\t\t`FILE_BYTES_WRITTEN`,\n \t\t\t`FILE_BYTES_READ`,\n \t\t\t`WASB_BYTES_WRITTEN`,\n \t\t\t`WASB_BYTES_READ`,\n \t\t\t`BYTES_READ`,\n \t\t\t`BYTES_WRITTEN`\n \t\t\tFROM HDI_JOB_details c {$join}";
} elseif ($type == 'DETAIL') {
$query = "SELECT e.bench, exe_time, c.* FROM HDI_JOB_details c {$join}";
} elseif ($type == 'TASKS') {
$query = "SELECT e.bench, exe_time, j.job_name, c.* FROM HDI_JOB_tasks c\n \t\t\tJOIN HDI_JOB_details j USING(id_exec,JOB_ID) {$join} ";
} else {
throw new \Exception('Unknown type!');
}
$exec_rows = $db->get_rows($query);
if (count($exec_rows) > 0) {
$show_in_result_counters = array('id_exec' => 'ID', 'JOB_ID' => 'JOBID', 'bench' => 'Bench', 'job_name' => 'JOBNAME');
$show_in_result_counters = Utils::generate_show($show_in_result_counters, $exec_rows, 4);
$jsonData = Utils::generateJSONTable($exec_rows, $show_in_result_counters, 0, 'COUNTER');
header('Content-Type: application/json');
echo $jsonData;
// if (count($exec_rows) > 10000) {
// $message .= 'WARNING, large resulset, please limit the query! Rows: '.count($exec_rows);
// }
} else {
$noData = array();
for ($i = 0; $i < 18; ++$i) {
$noData[] = 'No Data found';
}
ob_start('ob_gzhandler');
echo json_encode(array('aaData' => $noData));
}
} catch (Exception $e) {
$noData = array();
for ($i = 0; $i <= sizeof($show_in_result); ++$i) {
$noData[] = 'error';
}
echo json_encode(array('aaData' => array($noData)));
}
}
public function hdp2CountersAction()
{
try {
$db = $this->container->getDBUtils();
$benchOptions = $db->get_rows("SELECT DISTINCT bench FROM execs e JOIN HDI_JOB_details USING (id_exec) WHERE valid = 1");
$discreteOptions = array();
$discreteOptions['bench'][] = 'All';
foreach ($benchOptions as $option) {
$discreteOptions['bench'][] = array_shift($option);
}
$dbUtil = $this->container->getDBUtils();
$message = null;
//check the URL
$execs = Utils::get_GET_execs();
if (Utils::get_GET_string('type')) {
$type = Utils::get_GET_string('type');
} else {
$type = 'SUMMARY';
}
$join = "JOIN execs e using (id_exec) WHERE job_name NOT IN\n ('TeraGen', 'random-text-writer', 'mahout-examples-0.7-job.jar', 'Create pagerank nodes', 'Create pagerank links')" . ($execs ? ' AND id_exec IN (' . join(',', $execs) . ') ' : '');
if (isset($_GET['jobid'])) {
$join .= " AND JOB_ID = '{$_GET['jobid']}' ";
}
$join .= " LIMIT 10000";
$query = "";
if ($type == 'SUMMARY') {
$query = "SELECT e.bench, exe_time, c.id_exec, c.JOB_ID, c.job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME, c.TOTAL_MAPS, c.FAILED_MAPS, c.FINISHED_MAPS, c.TOTAL_REDUCES, c.FAILED_REDUCES, c.job_name as CHARTS\n \t\t\tFROM HDI_JOB_details c {$join}";
} else {
if ($type == "MAP") {
$query = "SELECT e.bench, exe_time, c.id_exec, JOB_ID, job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME, c.TOTAL_MAPS, c.FAILED_MAPS, c.FINISHED_MAPS, `TOTAL_LAUNCHED_MAPS`,\n \t\t\t`RACK_LOCAL_MAPS`,\n \t\t\t`SPILLED_RECORDS`,\n \t\t\t`MAP_INPUT_RECORDS`,\n \t\t\t`MAP_OUTPUT_RECORDS`,\n \t\t\t`MAP_OUTPUT_BYTES`,\n \t\t\t`MAP_OUTPUT_MATERIALIZED_BYTES`\n \t\t\tFROM HDI_JOB_details c {$join}";
} else {
if ($type == 'REDUCE') {
$query = "SELECT e.bench, exe_time, c.id_exec, c.JOB_ID, c.job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME, c.TOTAL_REDUCES, c.FAILED_REDUCES,\n \t\t\t`TOTAL_LAUNCHED_REDUCES`,\n \t\t\t`REDUCE_INPUT_GROUPS`,\n \t\t\t`REDUCE_INPUT_RECORDS`,\n \t\t\t`REDUCE_OUTPUT_RECORDS`,\n \t\t\t`REDUCE_SHUFFLE_BYTES`,\n \t\t\t`COMBINE_INPUT_RECORDS`,\n \t\t\t`COMBINE_OUTPUT_RECORDS`\n \t\t\tFROM HDI_JOB_details c {$join}";
} else {
if ($type == 'FILE-IO') {
$query = "SELECT e.bench, exe_time, c.id_exec, c.JOB_ID, c.job_name, c.SUBMIT_TIME, c.LAUNCH_TIME,\n \t\t\tc.FINISH_TIME,\n \t\t\t`SLOTS_MILLIS_MAPS`,\n \t\t\t`SLOTS_MILLIS_REDUCES`,\n \t\t\t`SPLIT_RAW_BYTES`,\n \t\t\t`FILE_BYTES_WRITTEN`,\n \t\t\t`FILE_BYTES_READ`,\n \t\t\t`WASB_BYTES_WRITTEN`,\n \t\t\t`WASB_BYTES_READ`,\n \t\t\t`BYTES_READ`,\n \t\t\t`BYTES_WRITTEN`\n \t\t\tFROM HDI_JOB_details c {$join}";
} else {
if ($type == 'DETAIL') {
$query = "SELECT e.bench, exe_time, c.* FROM HDI_JOB_details c {$join}";
} else {
if ($type == "TASKS") {
$query = "SELECT e.bench, exe_time, j.job_name, c.* FROM HDI_JOB_tasks c\n \t\t\tJOIN HDI_JOB_details j USING(id_exec,JOB_ID) {$join} ";
// $taskStatusOptions = $db->get_rows("SELECT DISTINCT TASK_STATUS FROM HDI_JOB_tasks JOIN execs USING (id_exec) WHERE valid = 1");
// $typeOptions = $db->get_rows("SELECT DISTINCT TASK_TYPE FROM HDI_JOB_tasks JOIN execs USING (id_exec) WHERE valid = 1");
//
// $discreteOptions['TASK_STATUS'][] = 'All';
// $discreteOptions['TASK_TYPE'][] = 'All';
// foreach($taskStatusOptions as $option) {
// $discreteOptions['TASK_STATUS'][] = array_shift($option);
// }
// foreach($typeOptions as $option) {
// $discreteOptions['TASK_TYPE'][] = array_shift($option);
// }
$discreteOptions['TASK_STATUS'] = array('All', 'SUCCEEDED');
$discreteOptions['TASK_TYPE'] = array('All', 'MAP', 'REDUCE');
} else {
throw new \Exception('Unknown type!');
}
}
}
}
}
}
$exec_rows = $dbUtil->get_rows($query);
if (count($exec_rows) > 0) {
$show_in_result_counters = array('id_exec' => 'ID', 'JOB_ID' => 'JOBID', 'bench' => 'Bench', 'job_name' => 'JOBNAME');
$show_in_result_counters = Utils::generate_show($show_in_result_counters, $exec_rows, 4);
}
} catch (\Exception $e) {
$this->container->getTwig()->addGlobal('message', $e->getMessage() . "\n");
}
echo $this->container->getTwig()->render('counters/hdp2counters.html.twig', array('selected' => 'Hadoop 2 Job Counters', 'theaders' => $show_in_result_counters, 'message' => $message, 'title' => 'Hadoop Jobs and Tasks Execution Counters', 'type' => $type, 'execs' => $execs, 'execsParam' => isset($_GET['execs']) ? $_GET['execs'] : '', 'discreteOptions' => $discreteOptions, 'hdp2' => true));
}
