/**
* Executes a script in a remote server via SSH
*
* @param type $study the case study
* @param type $jobid the job id
* @param type $workdir the working directory
* @param type $rerun if the execution is a re-run of an existing job
* @param type $script_path the script file, with path relative to the study_dir
*/
function execute_script_remote($study, $jobid, $workdir, $rerun, $cmd, $args = "")
{
include 'config.inc.php';
$jobid_qsub = 'j' . str_replace(".", "_", $jobid);
$orig_cmd = $cmd;
# check if the command is a python script
if (is_python_script($cmd) !== false) {
// prepend the python script with our command
$exec_dir_local = get_job_exec_dir($jobid);
if (!$rerun) {
// if the job is NOT a re-run, prepend the python script with the python-prepend.py file
$script_content = file_get_contents($exec_dir_local . "/" . $cmd);
$script_prepend = file_get_contents("lib/pipeline_prepend.py", true);
$script_content = $script_prepend . $script_content;
//error_log("Prepending python script at " . $exec_dir_local . "/" . $cmd);
file_put_contents($exec_dir_local . "/" . $cmd, $script_content);
// copying qsub_template.sh to workdir
$template = file_get_contents("lib/qsub_template.sh", true);
file_put_contents($exec_dir_local . "/qsub_template.sh", $template);
}
$cmd = $NC_CONFIG["python-bin"] . " " . $cmd;
} elseif (strpos($cmd, ".sh") !== false) {
#$cmd = "qsub -cwd -l mf=1.4G -N $jobid_qsub -sync y -o results/nc_stdout.log -e results/nc_stdout.log " . $cmd;
$cmd = "/bin/bash " . $cmd;
$args = $jobid_qsub;
}
// http://unix.stackexchange.com/a/29495
// The jobs submitted by nipype using qsub are set with the name
// node_name.workflow_name.user_name
// the user_name is passed to nipype through the environment variable $LOGNAME
// we cannot customize easily this behavior, so we customize the variable $LOGNAME to
// job_id.user_name
// in this way, the final job name will be
// node_name.workflow_name.user_name.job_id
// and it will be easier for the Neurocloud webapp to know if there are still running/queued SGE jobs for this job id
$cmdline = create_ssh_command() . " 'cd {$workdir} && chmod +x {$orig_cmd} && { LOGNAME={$jobid_qsub}.\$LOGNAME nohup {$cmd} {$args} >>results/nc_stdout.log 2>&1 & } && echo \$!' ";
//error_log($cmdline);
$pid = exec($cmdline);
$jobinfo = array("study" => $study, "jobid" => $jobid, "qsub_jobname" => $jobid_qsub, "pid" => $pid, "start_date" => date("Y-m-d H:i:s"), "script" => $orig_cmd, "cmdline" => $cmdline, "exec_dir" => $workdir, "exec_type" => "remote");
save_job_info($study, $jobid, $jobinfo);
return $pid;
}
/**
*
* @param type $qsub_jobname
* @return array an array containing 2 values: the first is the number of queued jobs, the second is the number of running jobs
*/
function get_queue_info($qsub_jobname)
{
include "neurocloud/config.inc.php";
$output = "";
$return_var = 0;
$user = OC_User::getUser();
$host = $NC_CONFIG["remote_host"];
$key = get_private_key_file($user);
$return_string = array(0, 0);
// the process is not running, but there could be still pending SGE jobs in the cluster
// so, we will check with qstat if there are still jobs
$cmdline_qstat = create_ssh_command() . "qstat -u {$user} -xml 2>/dev/null ";
exec($cmdline_qstat, $output, $return_var);
if ($return_var == 0) {
$xml_output = join("", $output);
// parse the XML given in output
$xml = new SimpleXMLElement($xml_output);
/* the returned XML has this structure:
<job_info><queue_info>[<job_list>*]<job_info>[<job_list>*]
*
* each job_list contains the data for a single SGE job (name, queue, state, etc)
*/
if ($xml->queue_info->job_list->count() == 0 && $xml->job_info->job_list->count() == 0) {
#$return_string = "No running SGE jobs in queue for job_id " . $qsub_jobname;
return $return_string;
} else {
#$return_string = "Running SGE jobs:\n";
foreach ($xml->queue_info->job_list as $job) {
// check if the SGE job name contains the generated job id for this neurocloud job
if (strpos($job->JB_name, $qsub_jobname)) {
#$return_string = $return_string . $job->JB_name . "\n";
$return_string[1] = $return_string[1] + 1;
}
}
#$return_string = $return_string . "\nQueued SGE jobs:\n";
foreach ($xml->job_info->job_list as $job) {
// check if the SGE job name contains the generated job id for this neurocloud job
if (strpos($job->JB_name, $qsub_jobname)) {
#$return_string = $return_string . $job->JB_name . "\n";
$return_string[0] = $return_string[0] + 1;
}
}
}
return $return_string;
} else {
//return "No SGE jobs in queue for job_id " . $qsub_jobname;
return array(0, 0);
}
}
