save_job_info($file, $job, $jobinfo);
}
if (!isset($jobinfo["usedspace"])) {
// calculate space used by temp dir + results
$usedspace = get_job_disk_usage(OC_User::getUser(), $file, $job);
if (!$usedspace) {
$usedspace = "--";
}
$jobinfo['usedspace'] = $usedspace;
// get_job_status returned FINISHED or KILLED, we save that info in the job JSON
save_job_info($file, $job, $jobinfo);
insert_job_log($file, $job, "{$status}. Used disk space: {$usedspace}");
}
} elseif ($status === JobStatus::$UNKNOWN) {
$jobinfo['status'] = $status;
save_job_info($file, $job, $jobinfo);
} else {
/* Job is RUNNING */
$queueinfo = get_queue_info($qsub_jobname);
$sgequeued = $queueinfo[0];
$sgerunning = $queueinfo[1];
}
$showoutputjs = "javascript:show_output('" . $outputid . "', '" . $output_file . "')";
$killjs = OC_Helper::linkTo("neurocloud", "ajax/kill_job.php", array("study" => $file, "jobid" => $job, "redirect" => 1));
$refreshstatusjs = "javascript:refresh_job_status('" . $rowid . "','" . $file . "','" . $job . "','" . $outputid . "')";
$showjobinfojs = "javascript:show_output('" . $outputid . "','" . get_job_info_file($file, $job) . "', true)";
#$queueinfojs = "javascript:show_queue_info('" . $outputid . "','". $jobinfo['qsub_jobname'] . "', true)";
$deletejs = "javascript:delete_results('" . $rowid . "','" . $file . "','" . $job . "')";
$script = isset($jobinfo["script"]) ? basename($jobinfo["script"]) : false;
$params = array("study" => $file, "script" => $script, "jobid" => $job, "mode" => "remote");
$rerun_link = OC_Helper::linkTo("neurocloud", "execute.php", $params);
$jobinfo["exec_type"] = "local";
// default execution type is local execution
}
$pid = $jobinfo["pid"];
if ($pid > 0 && $jobinfo["exec_type"] === "local") {
$ret = -1;
$output = "";
// uses the script killtree.sh
exec($killtree_path . " " . $pid . " KILL", $output, $ret);
if ($ret == 0) {
$jobinfo["status"] = JobStatus::$KILLED;
save_job_info($study, $jobid, $jobinfo);
do_exit($jobid, $pid, JobStatus::$KILLED);
} else {
do_exit($jobid, $pid, JobStatus::$FINISHED);
}
} else {
if ($pid > 0 && $jobinfo["exec_type"] === "remote") {
$ret = _kill_job_remote($jobinfo);
if ($ret == 0) {
$jobinfo["status"] = JobStatus::$KILLED;
save_job_info($study, $jobid, $jobinfo);
do_exit($jobid, $pid, JobStatus::$KILLED);
} else {
do_exit($jobid, $pid, JobStatus::$FINISHED);
}
} else {
do_exit($jobid, $pid, "ERROR: Wrong pid " . $pid);
}
}
exit;
/**
* Executes a script in a remote server via SSH
*
* @param type $study the case study
* @param type $jobid the job id
* @param type $workdir the working directory
* @param type $rerun if the execution is a re-run of an existing job
* @param type $script_path the script file, with path relative to the study_dir
*/
function execute_script_remote($study, $jobid, $workdir, $rerun, $cmd, $args = "")
{
include 'config.inc.php';
$jobid_qsub = 'j' . str_replace(".", "_", $jobid);
$orig_cmd = $cmd;
# check if the command is a python script
if (is_python_script($cmd) !== false) {
// prepend the python script with our command
$exec_dir_local = get_job_exec_dir($jobid);
if (!$rerun) {
// if the job is NOT a re-run, prepend the python script with the python-prepend.py file
$script_content = file_get_contents($exec_dir_local . "/" . $cmd);
$script_prepend = file_get_contents("lib/pipeline_prepend.py", true);
$script_content = $script_prepend . $script_content;
//error_log("Prepending python script at " . $exec_dir_local . "/" . $cmd);
file_put_contents($exec_dir_local . "/" . $cmd, $script_content);
// copying qsub_template.sh to workdir
$template = file_get_contents("lib/qsub_template.sh", true);
file_put_contents($exec_dir_local . "/qsub_template.sh", $template);
}
$cmd = $NC_CONFIG["python-bin"] . " " . $cmd;
} elseif (strpos($cmd, ".sh") !== false) {
#$cmd = "qsub -cwd -l mf=1.4G -N $jobid_qsub -sync y -o results/nc_stdout.log -e results/nc_stdout.log " . $cmd;
$cmd = "/bin/bash " . $cmd;
$args = $jobid_qsub;
}
// http://unix.stackexchange.com/a/29495
// The jobs submitted by nipype using qsub are set with the name
// node_name.workflow_name.user_name
// the user_name is passed to nipype through the environment variable $LOGNAME
// we cannot customize easily this behavior, so we customize the variable $LOGNAME to
// job_id.user_name
// in this way, the final job name will be
// node_name.workflow_name.user_name.job_id
// and it will be easier for the Neurocloud webapp to know if there are still running/queued SGE jobs for this job id
$cmdline = create_ssh_command() . " 'cd {$workdir} && chmod +x {$orig_cmd} && { LOGNAME={$jobid_qsub}.\$LOGNAME nohup {$cmd} {$args} >>results/nc_stdout.log 2>&1 & } && echo \$!' ";
//error_log($cmdline);
$pid = exec($cmdline);
$jobinfo = array("study" => $study, "jobid" => $jobid, "qsub_jobname" => $jobid_qsub, "pid" => $pid, "start_date" => date("Y-m-d H:i:s"), "script" => $orig_cmd, "cmdline" => $cmdline, "exec_dir" => $workdir, "exec_type" => "remote");
save_job_info($study, $jobid, $jobinfo);
return $pid;
}
