Exemplo n.º 1
0
function curl($opts, $pipe, $cmd = __FUNCTION__, $default_protocol = null)
{
    # set up internal (which by default comes from there being a default protocol)
    if ($default_protocol) {
        $internal = true;
    } else {
        $internal = false;
    }
    # merge the opts
    $opts = merge_opts($opts, $pipe);
    # get prefix opt
    $prefix = get_opt('curl', $opts, 'prefix', 'curl');
    if (!check_opt_set_type($cmd, $prefix, 'prefix', 'string')) {
        return false;
    }
    # get execute opt
    $execute = get_opt_config_value($prefix, $opts, 'execute', true);
    if (!check_opt_set_type($cmd, $execute, 'execute', 'boolean')) {
        return false;
    }
    # check if we should execute or not
    if (!$execute) {
        debug_echo($cmd, "not executing {$prefix} request");
        return;
    }
    # get debug opt
    $debug = get_opt_config_value($prefix, $opts, 'debug', false);
    if (!check_opt_set_type($cmd, $debug, 'debug', 'boolean')) {
        return false;
    }
    ## saved files ##
    # get save_to_file opt
    $save_to_file = get_opt_config_value($prefix, $opts, 'save_to_file');
    if (!check_opt_if_set_type($cmd, $save_to_file, 'save_to_file', 'string')) {
        return false;
    }
    # get use saved file opt
    $use_saved_file = get_opt_config_value($prefix, $opts, 'use_saved_file', false);
    if (!check_opt_set_type($cmd, $use_saved_file, 'use_saved_file', 'boolean')) {
        return false;
    }
    # get backup old saved files
    $backup_old_saved_files = get_opt_config_value($prefix, $opts, 'backup_old_saved_files', false);
    if (!check_opt_set_type($cmd, $backup_old_saved_files, 'backup_old_saved_files', 'boolean')) {
        return false;
    }
    # check if file exists and back up if necessary
    if (file_exists($save_to_file) && $use_saved_file) {
        debug_echo($cmd, "using saved file instead of executing request : {$save_to_file}");
        return array('file' => $save_to_file, 'response_file' => $save_to_file);
    }
    # get curlopts opt
    $curlopts = get_opt($prefix, $opts, 'curlopts');
    if (!check_opt_if_set_type($cmd, $curlopts, 'curlopts', 'array')) {
        return false;
    }
    if (is_array($curlopts)) {
        $curlopts =& $opts['curlopts'];
    } else {
        $curlopts = array();
        $opts['curlopts'] =& $curlopts;
    }
    # get finalize_curlopts opt
    $finalize_curlopts = get_opt($prefix, $opts, 'finalize_curlopts');
    if (!check_opt_if_set_type($cmd, $finalize_curlopts, 'finalize_curlopts', 'array')) {
        return false;
    }
    # set up response
    $res = array();
    # call pre functions if internal and set
    if ($internal) {
        # get pre functions opt
        $init_functions = get_opt($prefix, $opts, 'init_functions');
        if (!check_opt_if_set_type($cmd, $init_functions, 'init_functions', 'array_of_strings')) {
            return false;
        }
        # get pre exec functions opt
        $pre_exec_functions = get_opt($prefix, $opts, 'pre_exec_functions');
        if (!check_opt_if_set_type($cmd, $pre_exec_functions, 'pre_exec_functions', 'array_of_strings')) {
            return false;
        }
        # get post exec functions opt
        $post_exec_functions = get_opt($prefix, $opts, 'post_exec_functions');
        if (!check_opt_if_set_type($cmd, $post_exec_functions, 'post_exec_functions', 'array_of_strings')) {
            return false;
        }
        # get finalize functions opt
        $finalize_functions = get_opt($prefix, $opts, 'finalize_functions');
        if (!check_opt_if_set_type($cmd, $finalize_functions, 'finalize_functions', 'array_of_strings')) {
            return false;
        }
        # set array of arrays to pass to hooked functions
        $func_args = array('curlopts' => &$curlopts, 'opts' => &$opts, 'res' => &$res);
        # call the pre functions
        if ($init_functions) {
            foreach ($init_functions as $function) {
                $r = call_user_func($function, $prefix, $func_args, $cmd, $debug);
                if ($r === false) {
                    return false;
                }
            }
        }
    }
    # get download progress update opt
    $download_progress_update = get_opt_config_value($prefix, $opts, 'download_progress_update', true);
    if (!check_opt_set_type($cmd, $download_progress_update, 'download_progress_update', 'boolean')) {
        return false;
    }
    # add download process update to curlopts if necessary
    if ($download_progress_update) {
        $curlopts['noprogress'] = false;
        if ($curlopts['progressfunction'] = 'curl_progress_function') {
        }
    } else {
        $curlopts['noprogress'] = true;
    }
    # get url opt
    do {
        # get url opt (from curlopts)
        $url = get_opt($prefix, $curlopts, 'url');
        if (!check_opt_if_set_type($cmd, $url, 'url', 'string')) {
            return false;
        }
        if ($url) {
            break;
        }
        # get url opt (from opts)
        $url = get_opt($prefix, $opts, 'url');
        if (!check_opt_if_set_type($cmd, $url, 'url', 'string')) {
            return false;
        }
        if ($url) {
            break;
        }
        # get server opt
        $tries = array('server', 'svr');
        foreach ($tries as $try) {
            $server = get_opt($prefix, $opts, $try);
            if (!check_opt_if_set_type($cmd, $server, $try, 'string')) {
                return false;
            }
            if ($server) {
                break;
            }
        }
        # get path opt
        $path = get_opt($prefix, $opts, 'path');
        if (!check_opt_if_set_type($cmd, $path, 'path', 'string')) {
            return false;
        }
        # build path from dir / file if necessary
        if (!$path) {
            # get dir opt
            $dir = get_opt($prefix, $opts, 'dir');
            if (!check_opt_if_set_type($cmd, $dir, 'dir', 'string')) {
                return false;
            }
            # get file opt
            $file = get_opt($prefix, $opts, 'file');
            if (!check_opt_if_set_type($cmd, $file, 'file', 'string')) {
                return false;
            }
            # build path from dir/file
            $path = '';
            if ($dir) {
                if (substr($dir, -1) != '/') {
                    $dir .= '/';
                }
                $path .= $dir;
            }
            if ($file) {
                $path .= $file;
            }
            if (!$path) {
                $path = '/';
            } elseif ($path[0] != '/') {
                $path = "/{$path}";
            }
        }
        # build url from server and path
        if ($server) {
            if (substr($server, -1) == '/') {
                $server = substr($server, 0, -1);
            }
            $url = "{$server}{$path}";
            break;
        }
        return error($cmd, "no 'url' option set, and cannot be constructed from a 'server' and an optional 'path' option");
    } while (false);
    # get protocol opt
    $protocol = get_opt($prefix, $opts, 'protocol');
    if (!check_opt_if_set_type($cmd, $protocol, 'protocol', 'enum:ftp|ftps|http|https')) {
        return false;
    }
    # check the protocol is valid / accepted
    if (!$protocol) {
        $substr_6 = substr($url, 0, 6);
        $substr_7 = substr($url, 0, 7);
        $substr_8 = substr($url, 0, 8);
        if ($substr_6 == 'ftp://') {
            $protocol = 'ftp';
        } elseif ($substr_7 == 'ftps://') {
            $protocol = 'ftps';
        } elseif ($substr_7 == 'http://') {
            $protocol = 'http';
        } elseif ($substr_8 == 'https://') {
            $protocol = 'https';
        } elseif ($default_protocol) {
            $protocol = $default_protocol;
        } else {
            return error($cmd, "protocol '{$protocol}' is not supported");
        }
        $opts['protocol'] = $protocol;
    }
    # check if the url has the protocol included
    if (substr($url, 0, strlen($protocol) + 3) != "{$protocol}://") {
        $url = "{$protocol}://{$url}";
    }
    # check if the path is a directory, and add a slash if so
    if (substr($url, -1) != '/') {
        # get path is dir opt
        $path_is_dir = get_opt($prefix, $opts, 'path_is_dir', false);
        if (!check_opt_set_type($cmd, $path_is_dir, 'path_is_dir', 'boolean')) {
            return false;
        }
        # add slash to end if necessary
        if ($path_is_dir) {
            $url .= '/';
        }
    }
    # set the url
    $curlopts['url'] = $url;
    # get port opt
    $port = get_opt($prefix, $opts, 'port');
    if (!check_opt_if_set_type($cmd, $port, 'port', 'integer')) {
        return false;
    }
    # set the port curlopt
    if (!$port) {
        $port = $GLOBALS['curl_default_ports'][$protocol];
    }
    $curlopts['port'] = $port;
    # get user opt
    $tries = array('user', 'usr');
    foreach ($tries as $try) {
        $user = get_opt($prefix, $opts, $try);
        if (!check_opt_if_set_type($cmd, $user, $try, 'string')) {
            return false;
        }
        if ($user) {
            break;
        }
    }
    # get password opt
    $tries = array('password', 'pwd');
    foreach ($tries as $try) {
        $password = get_opt($prefix, $opts, $try);
        if (!check_opt_if_set_type($cmd, $password, $try, 'string')) {
            return false;
        }
        if ($password) {
            break;
        }
    }
    # set the authentication
    if ($user && $password) {
        $curlopts['userpwd'] = "{$user}:{$password}";
    }
    # get curlinfo opt
    $curlinfo = get_opt($prefix, $opts, 'curlinfo');
    if (!check_opt_if_set_type($cmd, $curlinfo, 'curlinfo', 'array_of_strings')) {
        return false;
    }
    # get request retries opt
    $request_retries = get_opt_config_value($prefix, $opts, 'request_retries', 3);
    if (!check_opt_set_type($cmd, $request_retries, 'request_retries', 'integer')) {
        # change to having max 10
        return false;
    }
    ## messages ##
    # get exec msg opt
    $exec_msg = get_opt($prefix, $opts, 'exec_msg', "executing curl request to url : {$url}");
    if (!check_opt_set_type($cmd, $exec_msg, 'exec_msg', 'string,boolean')) {
        return false;
    }
    # get exec retry msg opt
    $exec_retry_msg = get_opt($prefix, $opts, 'exec_retry_msg', "previous request failed - trying again");
    if (!check_opt_set_type($cmd, $exec_retry_msg, 'exec_retry_msg', 'string,boolean')) {
        return false;
    }
    # get exec retry msg opt
    $success_msg = get_opt($prefix, $opts, 'success_msg', "request successful");
    if (!check_opt_set_type($cmd, $success_msg, 'success_msg', 'string,boolean')) {
        return false;
    }
    # get exec retry msg opt
    $fail_msg = get_opt($prefix, $opts, 'fail_msg', "request failed");
    if (!check_opt_set_type($cmd, $fail_msg, 'fail_msg', 'string,boolean')) {
        return false;
    }
    ## save to locations ##
    # get save request head to file opt
    $request_head_file = get_opt($prefix, $opts, 'save_request_head_to_file');
    if (!check_opt_if_set_type($cmd, $request_head_file, 'save_request_head_to_file', 'string')) {
        return false;
    }
    # get save request body to file opt
    $request_body_file = get_opt($prefix, $opts, 'save_request_body_to_file');
    if (!check_opt_if_set_type($cmd, $request_body_file, 'save_request_body_to_file', 'string')) {
        return false;
    }
    # get save head to file opt
    $response_head_file = get_opt($prefix, $opts, 'save_head_to_file');
    if (!check_opt_if_set_type($cmd, $response_head_file, 'save_head_to_file', 'string')) {
        return false;
    }
    # get save body to file opt
    $response_body_file = get_opt($prefix, $opts, 'save_body_to_file');
    if (!check_opt_if_set_type($cmd, $response_body_file, 'save_body_to_file', 'string')) {
        return false;
    }
    # get max saved body size opt
    $max_saved_body_size = get_opt_config_value($prefix, $opts, 'max_saved_body_size', 512 * 1024 * 1024);
    if (!check_opt_if_set_type($cmd, $max_saved_body_size, 'max_saved_body_size', 'integer')) {
        return false;
    }
    # get stderr opt
    $stderr = get_opt_config_value($prefix, $opts, 'stderr');
    if (!check_opt_if_set_type($cmd, $stderr, 'stderr', 'string')) {
        return false;
    }
    # set stderr curlopt if necessary
    if ($stderr) {
        if (!make_dir_for_saving_file($stderr, null, $cmd)) {
            return false;
        }
        $stderr_handle = fopen($stderr, 'w');
        $curlopts['stderr'] = $stderr_handle;
    }
    # get the connection
    #$conn = curl_get_conn ($prefix, $opts, $cmd);
    #if (!$conn)
    #  return  false;
    #$opts['conn'] = $conn;
    #$c = $conn['handle'];
    /*
    $c = curl_init ();
    
    if (!$c)
      return  error ($cmd, "could not create curl resource");
    */
    # execute pre-exec functions
    if ($internal && $pre_exec_functions) {
        foreach ($pre_exec_functions as $function) {
            $r = call_user_func($function, $prefix, $func_args, $cmd, $debug);
            if ($r === false) {
                return false;
            }
        }
    }
    # set up the options
    if (!curl_set_curlopts($prefix, $protocol, $curlopts, $cmd, $c, $internal, $debug)) {
        return false;
    }
    if ($save_to_file) {
        # backup old response file if set and exists
        if (is_file($save_to_file) && $backup_old_saved_files && !backup_file($save_to_file, null, $cmd)) {
            return error($cmd, "could not backup previously saved file : {$save_to_file}");
        }
        # create parent directory of response file
        if (!make_dir(dirname($save_to_file), $opts, $cmd)) {
            return false;
        }
        /*
        # create file handler for response file
          
        $save_to_file_handle = @fopen ($save_to_file, "w");
          
        if (!$save_to_file_handle)
          return  error ($cmd, "could not open response file : $save_to_file");
        */
        # add response file handler to request
        curl_setopt($c, CURLOPT_FILE, $save_to_file_handle);
        # display message and add to $res
        debug_echo($cmd, "saving response to file : {$save_to_file}");
        $res['response_file'] = $save_to_file;
        $res['file'] = $save_to_file;
    }
    # display exec message
    if ($exec_msg) {
        debug_echo($cmd, "{$exec_msg} ...");
    }
    # execute request
    $success = false;
    for ($i = 0; $i < $request_retries; $i++) {
        $r = curl_exec($c);
        if ($r) {
            $success = true;
            break;
        } elseif ($exec_retry_msg) {
            debug_echo($cmd, "{$exec_retry_msg} ...");
        }
    }
    # close file handles
    if (@$save_to_file_handle) {
        fclose($save_to_file_handle);
    }
    if (@$stderr_handle) {
        fclose($stderr_handle);
    }
    # display message about result (we add space to cover up any download monitor)
    if ($success) {
        if ($success_msg) {
            debug_echo($cmd, "{$success_msg}                     ");
        }
    } elseif ($fail_msg) {
        $curl_error = strtolower(curl_error($c));
        $fail_msg .= ", {$curl_error}";
        return error($cmd, "{$fail_msg}                        ");
    }
    # execute post-exec functions
    if ($internal && $post_exec_functions) {
        foreach ($post_exec_functions as $function) {
            $r = call_user_func($function, $prefix, $func_args, $cmd, $debug);
            if ($r === false) {
                return false;
            }
        }
    }
    # split the response into header and body
    $request_head = curl_getinfo($c, CURLINFO_HEADER_OUT);
    $response_head_size = curl_getinfo($c, CURLINFO_HEADER_SIZE);
    if ($save_to_file) {
        # get response body size
        $handle = fopen($save_to_file, 'r');
        $response_head = $response_head_size > 0 ? trim(fread($handle, $response_head_size)) : '';
        $stat = fstat($handle);
        $response_body_size = $stat['size'] - $response_head_size;
        # check max size of response body file before putting in separate file
        if ($response_body_size > $max_saved_body_size) {
            warning($cmd, "response body size is greater than max, so is not set : {$response_body_size} > {$max_saved_body_size}");
            $response_body = "[{$save_to_file}]";
            $no_reset_response_body_size = true;
        } else {
            $response_body = $response_body_size > 0 ? trim(fread($handle, $response_body_size)) : '';
        }
        fclose($handle);
    } else {
        $response_head = trim(substr($r, 0, $response_head_size));
        $response_body = trim(substr($r, $response_head_size));
    }
    $response_head_size = strlen($response_head);
    if (!@$no_reset_response_body_size) {
        $response_body_size = strlen($response_body);
    }
    # add main request and response variables to $res
    $res['request_url'] = $url;
    $res['request_head_size'] = strlen($request_head);
    $res['response_head_size'] = $response_head_size;
    $res['response_body_size'] = $response_body_size;
    # save data to files if set and save in $res
    $save_to_files = array('request_head' => array($request_head_file, $request_head), 'response_head' => array($response_head_file, $response_head), 'response_body' => array($response_body_file, $response_body));
    # save data to files if set and save in $res
    foreach ($save_to_files as $name => $vars) {
        $file = $vars[0];
        $data = $vars[1];
        if ($file) {
            $report_name = str_replace('_', ' ', $name);
            if (is_file($file) && $backup_old_saved_files && !backup_file($file, null, $cmd)) {
                return error($cmd, "could not backup previously saved {$report_name} file : {$file}");
            }
            if (strlen($data) == 0) {
                warning($cmd, "{$report_name} is of zero size, so shall not be saved to file : {$file}");
            } else {
                if (!file_put_contents($file, $data)) {
                    return error($cmd, "could not save {$report_name} to file : {$file}");
                }
                debug_echo($cmd, "{$report_name} saved to file : {$file}");
                $res["{$name}_file"] = $file;
            }
        }
        $res[$name] = $data;
    }
    # finalize curlopts
    if ($finalize_curlopts) {
        if (!curl_set_curlopts($prefix, $protocol, $finalize_curlopts, $cmd, $c, $internal, $debug, true)) {
            return false;
        }
    }
    # execute finalize functions
    if ($internal && $finalize_functions) {
        foreach ($finalize_functions as $function) {
            $r = call_user_func($function, $prefix, $func_args, $cmd, $debug);
            if ($r === false) {
                return false;
            }
        }
    }
    # sort $res, display and return
    ksort($res);
    curl_close($c);
    return $res;
}
Exemplo n.º 2
0
function curl($opts, $pipe, $cmd = __FUNCTION__, $default_protocol = null)
{
    # set up internal (which by default comes from there being a default protocol)
    if ($default_protocol) {
        $internal = true;
    } else {
        $internal = false;
    }
    # merge the opts
    $opts = merge_opts($opts, $pipe);
    # set prefix opt
    $prefix = $default_protocol ? $default_protocol : 'curl';
    # get execute opt
    $execute = get_opt_config_value($prefix, $opts, 'execute', true);
    if (!check_opt_set_type($cmd, $execute, 'execute', 'boolean')) {
        return false;
    }
    # check if we should execute or not
    if (!$execute) {
        debug_echo($cmd, "not executing {$prefix} request");
        return;
    }
    # get debug opt
    $debug = get_opt_config_value($prefix, $opts, 'debug', false);
    if (!check_opt_set_type($cmd, $debug, 'debug', 'boolean')) {
        return false;
    }
    ## saved files ##
    # get save_to_file opt
    $save_to_file = get_opt_config_value($prefix, $opts, 'save_to_file');
    if (!check_opt_if_set_type($cmd, $save_to_file, 'save_to_file', 'string')) {
        return false;
    }
    # get use saved file opt
    $use_saved_file = get_opt_config_value($prefix, $opts, 'use_saved_file', false);
    if (!check_opt_set_type($cmd, $use_saved_file, 'use_saved_file', 'boolean')) {
        return false;
    }
    # get backup old saved files
    $backup_old_saved_files = get_opt_config_value($prefix, $opts, 'backup_old_saved_files', false);
    if (!check_opt_set_type($cmd, $backup_old_saved_files, 'backup_old_saved_files', 'boolean')) {
        return false;
    }
    # check if file exists and back up if necessary
    if (file_exists($save_to_file) && $use_saved_file) {
        debug_echo($cmd, "using saved file instead of executing request : {$save_to_file}");
        return array('file' => $save_to_file, 'response_file' => $save_to_file);
    }
    # get curlopts opt
    $curlopts = get_opt($prefix, $opts, 'curlopts');
    if (!check_opt_if_set_type($cmd, $curlopts, 'curlopts', 'array')) {
        return false;
    }
    if (is_array($curlopts)) {
        $curlopts =& $opts['curlopts'];
    } else {
        $curlopts = array();
        $opts['curlopts'] =& $curlopts;
    }
    # get finalize_curlopts opt
    $finalize_curlopts = get_opt($prefix, $opts, 'finalize_curlopts');
    if (!check_opt_if_set_type($cmd, $finalize_curlopts, 'finalize_curlopts', 'array')) {
        return false;
    }
    # set up response
    $res = array();
    # call pre functions if internal and set
    if ($internal) {
        # get pre functions opt
        $init_functions = get_opt($prefix, $opts, 'init_functions');
        if (!check_opt_if_set_type($cmd, $init_functions, 'init_functions', 'array_of_strings')) {
            return false;
        }
        # get pre exec functions opt
        $pre_exec_functions = get_opt($prefix, $opts, 'pre_exec_functions');
        if (!check_opt_if_set_type($cmd, $pre_exec_functions, 'pre_exec_functions', 'array_of_strings')) {
            return false;
        }
        # get post exec functions opt
        $post_exec_functions = get_opt($prefix, $opts, 'post_exec_functions');
        if (!check_opt_if_set_type($cmd, $post_exec_functions, 'post_exec_functions', 'array_of_strings')) {
            return false;
        }
        # get finalize functions opt
        $finalize_functions = get_opt($prefix, $opts, 'finalize_functions');
        if (!check_opt_if_set_type($cmd, $finalize_functions, 'finalize_functions', 'array_of_strings')) {
            return false;
        }
        # set array of arrays to pass to hooked functions
        $func_args = array('curlopts' => &$curlopts, 'opts' => &$opts, 'res' => &$res);
        # call the pre functions
        if ($init_functions) {
            foreach ($init_functions as $function) {
                $r = call_user_func($function, $prefix, $func_args, $cmd, $debug);
                if ($r === false) {
                    return false;
                }
            }
        }
    }
    # get url opt
    do {
        # get url opt (from curlopts)
        $url = get_opt($prefix, $curlopts, 'url');
        if (!check_opt_if_set_type($cmd, $url, 'url', 'string')) {
            return false;
        }
        if ($url) {
            break;
        }
        # get url opt (from opts)
        $url = get_opt($prefix, $opts, 'url');
        if (!check_opt_if_set_type($cmd, $url, 'url', 'string')) {
            return false;
        }
        if ($url) {
            break;
        }
        # get server opt
        $tries = array('server', 'svr');
        foreach ($tries as $try) {
            $server = get_opt($prefix, $opts, $try);
            if (!check_opt_if_set_type($cmd, $server, $try, 'string')) {
                return false;
            }
            if ($server) {
                break;
            }
        }
        # get path opt
        $path = get_opt($prefix, $opts, 'path');
        if (!check_opt_if_set_type($cmd, $path, 'path', 'string')) {
            return false;
        }
        # build path from dir / file if necessary
        if (!$path) {
            # get dir opt
            $dir = get_opt($prefix, $opts, 'dir');
            if (!check_opt_if_set_type($cmd, $dir, 'dir', 'string')) {
                return false;
            }
            # get file opt
            $file = get_opt($prefix, $opts, 'file');
            if (!check_opt_if_set_type($cmd, $file, 'file', 'string')) {
                return false;
            }
            # build path from dir/file
            $path = '';
            if ($dir) {
                if (substr($dir, -1) != '/') {
                    $dir .= '/';
                }
                $path .= $dir;
            }
            if ($file) {
                $path .= $file;
            }
            if (!$path) {
                $path = '/';
            } elseif ($path[0] != '/') {
                $path = "/{$path}";
            }
        }
        # build url from server and path
        if ($server) {
            if (substr($server, -1) == '/') {
                $server = substr($server, 0, -1);
            }
            $url = "{$server}{$path}";
            break;
        }
        return error($cmd, "no 'url' option set, and cannot be constructed from a 'server' and an optional 'path' option");
    } while (false);
    # get protocol opt
    $protocol = get_opt($prefix, $opts, 'protocol');
    if (!check_opt_if_set_type($cmd, $protocol, 'protocol', 'enum:ftp|ftps|http|https|imap|imaps|smtp|smtps')) {
        return false;
    }
    # check the protocol is valid / accepted
    if (!$protocol) {
        $substr_6 = substr($url, 0, 6);
        $substr_7 = substr($url, 0, 7);
        $substr_8 = substr($url, 0, 8);
        if ($substr_6 == 'ftp://') {
            $protocol = 'ftp';
        } elseif ($substr_7 == 'ftps://') {
            $protocol = 'ftps';
        } elseif ($substr_7 == 'http://') {
            $protocol = 'http';
        } elseif ($substr_8 == 'https://') {
            $protocol = 'https';
        } elseif ($substr_7 == 'imap://') {
            $protocol = 'imap';
        } elseif ($substr_8 == 'imaps://') {
            $protocol = 'imaps';
        } elseif ($substr_7 == 'smtp://') {
            $protocol = 'smtp';
        } elseif ($substr_8 == 'smtps://') {
            $protocol = 'smtps';
        } elseif ($default_protocol) {
            $protocol = $default_protocol;
        } else {
            return error($cmd, "protocol '{$protocol}' is not supported");
        }
        $opts['protocol'] = $protocol;
    }
    # check if the url has the protocol included
    if (substr($url, 0, strlen($protocol) + 3) != "{$protocol}://") {
        $url = "{$protocol}://{$url}";
    }
    # get port opt
    $port = get_opt($prefix, $opts, 'port');
    if (!check_opt_if_set_type($cmd, $port, 'port', 'integer')) {
        return false;
    }
    # set the port curlopt
    if ($port) {
        $url .= ":{$port}";
    }
    # check if the path is a directory, and add a slash if so
    if (substr($url, -1) != '/') {
        # get path is dir opt
        $path_is_dir = get_opt($prefix, $opts, 'path_is_dir', false);
        if (!check_opt_set_type($cmd, $path_is_dir, 'path_is_dir', 'boolean')) {
            return false;
        }
        # add slash to end if necessary
        if ($path_is_dir) {
            $url .= '/';
        }
    }
    # set the url
    $curlopts['url'] = $url;
    # get user opt
    $tries = array('user', 'usr');
    foreach ($tries as $try) {
        $user = get_opt($prefix, $opts, $try);
        if (!check_opt_if_set_type($cmd, $user, $try, 'string')) {
            return false;
        }
        if ($user) {
            break;
        }
    }
    # get password opt
    $tries = array('password', 'pwd');
    foreach ($tries as $try) {
        $password = get_opt($prefix, $opts, $try);
        if (!check_opt_if_set_type($cmd, $password, $try, 'string')) {
            return false;
        }
        if ($password) {
            break;
        }
    }
    # set the authentication
    if ($user && $password) {
        $curlopts['user'] = "******";
    }
    # get curlinfo opt
    $curlinfo = get_opt($prefix, $opts, 'curlinfo');
    if (!check_opt_if_set_type($cmd, $curlinfo, 'curlinfo', 'array_of_strings')) {
        return false;
    }
    # get request retries opt
    $request_retries = get_opt_config_value($prefix, $opts, 'request_retries', 3);
    if (!check_opt_set_type($cmd, $request_retries, 'request_retries', 'integer')) {
        # change to having max 10
        return false;
    }
    $curlopts['retry'] = $request_retries;
    ## messages ##
    # get exec msg opt
    $exec_msg = get_opt($prefix, $opts, 'exec_msg', "executing curl request to url : {$url}");
    if (!check_opt_set_type($cmd, $exec_msg, 'exec_msg', 'string,boolean')) {
        return false;
    }
    # get exec retry msg opt
    $exec_retry_msg = get_opt($prefix, $opts, 'exec_retry_msg', "previous request failed - trying again");
    if (!check_opt_set_type($cmd, $exec_retry_msg, 'exec_retry_msg', 'string,boolean')) {
        return false;
    }
    # get exec retry msg opt
    $success_msg = get_opt($prefix, $opts, 'success_msg', "request successful");
    if (!check_opt_set_type($cmd, $success_msg, 'success_msg', 'string,boolean')) {
        return false;
    }
    # get exec retry msg opt
    $fail_msg = get_opt($prefix, $opts, 'fail_msg', "request failed");
    if (!check_opt_set_type($cmd, $fail_msg, 'fail_msg', 'string,boolean')) {
        return false;
    }
    ## save to locations ##
    # get save request head to file opt
    $request_head_file = get_opt($prefix, $opts, 'save_request_head_to_file');
    if (!check_opt_if_set_type($cmd, $request_head_file, 'save_request_head_to_file', 'string')) {
        return false;
    }
    # get save request body to file opt
    $request_body_file = get_opt($prefix, $opts, 'save_request_body_to_file');
    if (!check_opt_if_set_type($cmd, $request_body_file, 'save_request_body_to_file', 'string')) {
        return false;
    }
    # get save head to file opt
    $response_head_file = get_opt($prefix, $opts, 'save_head_to_file');
    if (!check_opt_if_set_type($cmd, $response_head_file, 'save_head_to_file', 'string')) {
        return false;
    }
    # get save body to file opt
    $response_body_file = get_opt($prefix, $opts, 'save_body_to_file');
    if (!check_opt_if_set_type($cmd, $response_body_file, 'save_body_to_file', 'string')) {
        return false;
    }
    # get max saved body size opt
    $max_saved_body_size = get_opt_config_value($prefix, $opts, 'max_saved_body_size', 512 * 1024 * 1024);
    if (!check_opt_if_set_type($cmd, $max_saved_body_size, 'max_saved_body_size', 'integer')) {
        return false;
    }
    # get stderr opt
    $stderr = get_opt_config_value($prefix, $opts, 'stderr');
    if (!check_opt_if_set_type($cmd, $stderr, 'stderr', 'string')) {
        return false;
    }
    # set stderr curlopt if necessary
    if ($stderr) {
        if (!make_dir_for_saving_file($stderr, null, $cmd)) {
            return false;
        }
        $stderr_handle = fopen($stderr, 'w');
        $curlopts['stderr'] = $stderr_handle;
    }
    # execute pre-exec functions
    if ($internal && $pre_exec_functions) {
        foreach ($pre_exec_functions as $function) {
            $r = call_user_func($function, $prefix, $func_args, $cmd, $debug);
            if ($r === false) {
                return false;
            }
        }
    }
    # save to file
    if ($save_to_file) {
        # backup old response file if set and exists
        if (is_file($save_to_file) && $backup_old_saved_files && !backup_file($save_to_file, null, $cmd)) {
            return error($cmd, "could not backup previously saved file : {$save_to_file}");
        }
        # create parent directory of response file
        if (!make_dir(dirname($save_to_file), $opts, $cmd)) {
            return false;
        }
        # add response file handler to request
        $curlopts['output'] = $save_to_file;
        # display message and add to $res
        debug_echo($cmd, "saving response to file : {$save_to_file}");
        $response_body_file = $save_to_file;
        $res['file'] = $save_to_file;
    } else {
        $response_body_file = null;
    }
    # add headers file
    $response_heads_file = tempnam('/tmp', "mawk-curl-response-heads-");
    $curlopts['dump-header'] = $response_heads_file;
    # set up the options
    $request_cmd = curl_build_request($prefix, $protocol, $curlopts, $cmd, $internal, $debug);
    if (!$request_cmd) {
        return false;
    }
    $res['request_cmd'] = $request_cmd;
    # display exec message
    if ($exec_msg) {
        debug_echo($cmd, "{$exec_msg} ...");
    }
    if ($debug) {
        debug_echo($cmd, "curl exec : {$request_cmd}");
    }
    # execute request
    $response_last_line = exec($request_cmd, $response_body, $rv);
    # merge the response body
    $response_body = implode('', $response_body);
    # display message about result (we add space to cover up any download monitor)
    if ($rv === 0) {
        if ($success_msg) {
            debug_echo($cmd, "{$success_msg}                     ");
        }
    } else {
        return error($cmd, "{$fail_msg}                        ");
    }
    # execute post-exec functions
    if ($internal && $post_exec_functions) {
        foreach ($post_exec_functions as $function) {
            $r = call_user_func($function, $prefix, $func_args, $cmd, $debug);
            if ($r === false) {
                return false;
            }
        }
    }
    # add main request and response variables to $res
    $res['request_url'] = $url;
    # get the response heads
    $response_heads = file_get_contents($response_heads_file);
    unlink($response_heads_file);
    switch ($protocol) {
        case 'http':
        case 'https':
            # get the last response heads
            $response_heads = explode("\r\n\r\n", $response_heads);
            foreach ($response_heads as $i => $headers) {
                if ($headers == '') {
                    unset($response_heads[$i]);
                }
            }
            $res['response_heads'] = $response_heads;
            # get response head
            $response_head = $response_heads[count($response_heads) - 1];
            break;
        default:
            $response_head = $response_heads;
            break;
    }
    # save data to files if set and save in $res
    $save_to_files = array('response_head' => array($response_head_file, $response_head, true), 'response_body' => array($response_body_file, $response_body));
    # save data to files if set and save in $res
    foreach ($save_to_files as $name => $vars) {
        $file = $vars[0];
        $data = $vars[1];
        $set = @$vars[2];
        if ($file) {
            if ($set) {
                $report_name = str_replace('_', ' ', $name);
                if (is_file($file) && $backup_old_saved_files && !backup_file($file, null, $cmd)) {
                    return error($cmd, "could not backup previously saved {$report_name} file : {$file}");
                }
                if (strlen($data) == 0) {
                    warning($cmd, "{$report_name} is of zero size, so shall not be saved to file : {$file}");
                } else {
                    if (!file_put_contents($file, $data)) {
                        return error($cmd, "could not save {$report_name} to file : {$file}");
                    }
                    debug_echo($cmd, "{$report_name} saved to file : {$file}");
                }
            } else {
                $data = file_get_contents($file);
            }
            $res["{$name}_file"] = $file;
        }
        $res[$name] = $data;
    }
    # execute finalize functions
    if ($internal && $finalize_functions) {
        foreach ($finalize_functions as $function) {
            $r = call_user_func($function, $prefix, $func_args, $cmd, $debug);
            if ($r === false) {
                return false;
            }
        }
    }
    # sort $res, display and return
    ksort($res);
    return $res;
}