function ftp_get_files_in_dir($opts, $pipe, $cmd = __FUNCTION__)
{
# set prefix
$prefix = $cmd;
# merge opts
$opts = merge_opts_for_output($pipe, $opts);
# get local dir opt
$local_dir = get_opt_config_value($prefix, $opts, 'local_dir');
if (!check_opt_if_set_type($cmd, $local_dir, 'local_dir', 'string')) {
return false;
}
# scan dir
$res = ftp_scandir($opts, null, $cmd);
if ($res === false) {
return false;
}
# set the local dir
$res['local_dir'] = $local_dir;
# get files
return ftp_get_files($res, $opts);
}
function set_as_key_index($opts, $pipe, $cmd = __FUNCTION__)
{
# set prefix
$prefix = 'result_set_as_key_index';
# merge opts
$opts = merge_opts($opts, null, 'key');
# get key opt
$key = get_opt($prefix, $opts, 'key');
if (!check_opt_set_type($cmd, $key, 'key', 'string')) {
return false;
}
# get index opt
$index = get_opt($prefix, $opts, 'index', 0);
if (!check_opt_set_type($cmd, $index, 'index', 'integer')) {
return false;
}
# build $res
$arr = merge_opts_for_output(array(), $pipe);
$res = array($key => array());
$res[$key][$index] = $arr;
return $res;
}
function imap_search_for_email($opts, $pipe, $cmd = __FUNCTION__)
{
# set prefix
$prefix = 'imap_search_for_email';
# unset opts that might affect the request
unset($pipe['dir']);
unset($pipe['file']);
unset($pipe['path']);
# merge opts
$merged_opts = merge_opts($opts, $pipe);
# get tests opt
$tests = get_opt($prefix, $merged_opts, 'tests');
if (!check_opt_set_type($cmd, $tests, 'tests', 'array')) {
return false;
}
# perform serach of mailbox
$r = imap_get_mailbox_email_count($opts, $pipe, $cmd);
if ($r === false) {
return false;
}
# get the count
$email_count = $r['email_count'];
# split the tests
$header_tests = @$tests['headers'];
$body_tests = @$tests['body'];
# loop over the emails to get the headers
$found = false;
for ($i = $email_count; $i >= 1; $i--) {
# set the email id
$email_id = $i;
$part_id = false;
$email_headers_res = null;
$email_body_res = null;
$opts['email_id'] = $i;
# get the eamil headers if required
if ($header_tests) {
$email_headers_res = imap_get_email_headers($opts, $pipe, $cmd);
if ($email_headers_res === false) {
return false;
}
# get the returned headers and set as lower-case
$email_headers = $email_headers_res['email_headers'];
foreach ($email_headers as $name => $value) {
$email_headers[strtolower($name)] = $value;
}
# test the headers
$found_headers = imap_test_values($header_tests, $email_headers);
if (!$found_headers) {
continue;
}
}
if ($body_tests) {
$email_body_res = imap_get_email_body($opts, $pipe, $cmd);
if ($email_body_res === false) {
return false;
}
# get the returned headers and set as lower-case
$email_body_parts = $email_body_res['email_body_parts'];
# run through the parts
foreach ($email_body_parts as $part_idx => $part) {
$found_part = imap_test_values($body_tests, $part);
if ($found_part) {
$part_id = $part_idx + 1;
break;
}
}
if (!$found_part) {
continue;
}
}
$found = true;
break;
}
# check if found or not
if (!$found) {
return error($cmd, "email not found with the search criteria");
}
# display success message
$msg = "email found with id {$email_id}";
if ($part_id) {
$msg .= " part {$part_id}";
}
debug_echo($cmd, $msg);
# create res
$res = array('email_id' => $email_id);
if ($part_id) {
$res['email_part_id'] = $part_id;
foreach ($part as $name => $value) {
$res["email_{$name}"] = $value;
}
}
# merge res with headers and body
$res = merge_opts_for_output($opts, $res);
if ($email_headers_res) {
$res = merge_opts_for_output($res, $email_headers_res);
}
if ($email_body_res) {
$res = merge_opts_for_output($res, $email_body_res);
}
ksort($res);
return $res;
}
function http_request($opts, $pipe = null, $cmd = __FUNCTION__)
{
# set prefix
$prefix = 'http';
# merge opts
$opts = merge_opts($opts, $pipe, 'url');
# get execute opt
$execute = get_opt_config_value($prefix, $opts, 'execute', true);
if (!check_opt_set_type($cmd, $execute, 'execute', 'boolean')) {
return false;
}
# check if we should execute or not
if (!$execute) {
return;
}
# get debug opt
$debug = get_opt_config_value($prefix, $opts, 'debug', false);
if (!check_opt_set_type($cmd, $debug, 'debug', 'boolean')) {
return false;
}
# get request method opt
$request_method = get_opt_config_value($prefix, $opts, 'request_method', 'get');
echo "{$request_method}\\[1]\n";
if (!check_opt_if_set_type($cmd, $request_method, 'request_method', 'http_request_method')) {
return false;
}
# get post args opt
$post_args = get_opt($prefix, $opts, 'post_args');
if (!check_opt_if_set_type($cmd, $post_args, 'post_args', 'array')) {
return false;
}
# get post form opt
$form_name = get_opt($prefix, $opts, 'form_name');
if (!check_opt_if_set_type($cmd, $form_name, 'form_name', 'string')) {
return false;
}
# get response body opt
$response_body = get_opt($prefix, $opts, 'response_body');
if (!check_opt_if_set_type($cmd, $response_body, 'response_body', 'string')) {
return false;
}
# check to see if we're posting
if ($post_args || $form_name) {
$request_method = 'post';
}
if ($request_method == 'post' && $response_body) {
debug_echo($cmd, "performing a post request and there was a previous request, so will search for action, method and inputs");
$opts = http_parse_form($opts, $form_name, $cmd);
if ($opts === false) {
return false;
}
$post_args = $opts['post_args'];
}
echo "{$request_method}\\[2\\]\n";
# get base url from url|response_location|request opt
$base_url_opts = array('url', 'response_location', 'form_action', 'response_form_action', 'request_url');
foreach ($base_url_opts as $opt) {
$base_url = get_opt($prefix, $opts, $opt);
if (!check_opt_if_set_type($cmd, $base_url, $opt, 'string')) {
return false;
}
if ($base_url) {
break;
}
}
if (!$base_url) {
return opt_not_set_msg($cmd, implode(',', $base_url_opts));
}
# get get args opt
$get_args = get_opt($prefix, $opts, 'get_args');
if (!check_opt_if_set_type($cmd, $get_args, 'get_args', 'array,string')) {
return false;
}
# get headers opt
$request_headers = get_opt($prefix, $opts, 'headers');
if (!check_opt_if_set_type($cmd, $request_headers, 'headers', 'array_of_strings')) {
return false;
}
# get auto redirect opt
$auto_redirect = get_opt($prefix, $opts, 'auto_redirect', true);
if (!check_opt_set_type($cmd, $auto_redirect, 'auto_redirect', 'boolean')) {
return false;
}
# get authorization opt
$authorization = get_opt($prefix, $opts, 'authorization');
if (!check_opt_if_set_type($cmd, $authorization, 'authorization', 'array')) {
return false;
}
# set authorization if necessary
if ($authorization) {
# get authorization:usr opt
$usr = get_opt($prefix, $authorization, 'usr');
if (!check_opt_set_type($cmd, $usr, 'authorization:usr', 'string,integer')) {
return false;
}
$usr = (string) $usr;
# get authorization:pwd opt
$pwd = get_opt($prefix, $authorization, 'pwd');
if (!check_opt_set_type($cmd, $pwd, 'authorization:pwd', 'string,integer')) {
return false;
}
$pwd = (string) $pwd;
# get authorization:type opt
$type = get_opt($prefix, $authorization, 'type', 'Basic');
if (!check_opt_set_type($cmd, $type, 'authorization:type', 'string')) {
return false;
}
# add the authorization header to the list of headers
$auth_header = "Authorization: {$type} " . base64_encode("{$usr}:{$pwd}");
if (!$request_headers) {
$request_headers = array();
}
$request_headers[] = $auth_header;
}
# get request retries opt
$request_retries = get_opt_config_value($prefix, $opts, 'request_retries', 3);
if (!check_opt_set_type($cmd, $request_retries, 'request_retries', 'integer')) {
return false;
}
# get download msg opt
$download_msg = get_opt($prefix, $opts, 'download_msg');
if (!check_opt_if_set_type($cmd, $download_msg, 'download_msg', 'string')) {
return false;
}
# get use downloaded file opt
$use_saved_file = get_opt_config_value($prefix, $opts, 'use_saved_file', true);
if (!check_opt_set_type($cmd, $use_saved_file, 'use_saved_file', 'boolean')) {
return false;
}
# get backup old downloaded files
$backup_old_saved_files = get_opt_config_value($prefix, $opts, 'backup_old_saved_files', true);
if (!check_opt_set_type($cmd, $backup_old_saved_files, 'backup_old_saved_files', 'boolean')) {
return false;
}
# get checked downloaded file opt
$checked_saved_file = get_config_value('http_checked_saved_file');
if (!check_opt_if_set_type($cmd, $checked_saved_file, 'http_checked_saved_file', 'string')) {
return false;
}
# get download to file opt
$save_to_file = get_opt($prefix, $opts, 'save_to_file');
if (!check_opt_if_set_type($cmd, $save_to_file, 'save_to_file', 'string')) {
return false;
}
# get download progress update opt
$download_progress_update = get_opt_config_value($prefix, $opts, 'download_progress_update', true);
if (!check_opt_set_type($cmd, $download_progress_update, 'download_progress_update', 'boolean')) {
return false;
}
# get max response body size opt
$max_response_body_size = get_opt_config_value($prefix, $opts, 'max_response_body_size', 32 * 1024 * 1024);
if (!check_opt_if_set_type($cmd, $max_response_body_size, 'max_response_body_size', 'integer')) {
return false;
}
# get save request head to file opt
$request_head_file = get_opt($prefix, $opts, 'save_request_head_to_file');
if (!check_opt_if_set_type($cmd, $request_head_file, 'save_request_head_to_file', 'string')) {
return false;
}
# get save request body to file opt
$request_body_file = get_opt($prefix, $opts, 'save_request_body_to_file');
if (!check_opt_if_set_type($cmd, $request_body_file, 'save_request_body_to_file', 'string')) {
return false;
}
# get save head to file opt
$response_head_file = get_opt($prefix, $opts, 'save_head_to_file');
if (!check_opt_if_set_type($cmd, $response_head_file, 'save_head_to_file', 'string')) {
return false;
}
# get save body to file opt
$response_body_file = get_opt($prefix, $opts, 'save_body_to_file');
if (!check_opt_if_set_type($cmd, $response_body_file, 'save_body_to_file', 'string')) {
return false;
}
# get parse response form opt
$parse_response_form = get_opt_config_value($prefix, $opts, 'parse_response_form');
if (!check_opt_if_set_type($cmd, $parse_response_form, 'parse_response_form', 'string')) {
return false;
}
# get parse response hidden inputs opt
$parse_response_form_args = get_opt_config_value($prefix, $opts, 'parse_response_form_args', false);
if (!check_opt_set_type($cmd, $parse_response_form_args, 'parse_response_form_args', 'boolean')) {
return false;
}
# check to see if we should use a downloaded file
if ($use_saved_file) {
# check if response body file exists
if (file_exists($response_body_file)) {
debug_echo($cmd, "using saved response body file instead of executing the HTTP request : {$response_body_file}");
$res['response_body_file'] = $response_body_file;
$res['response_file'] = $response_body_file;
$res['file'] = $response_body_file;
return $res;
}
# check if the save to file exists
if (file_exists($save_to_file)) {
debug_echo($cmd, "using saved file instead of executing the HTTP request : {$save_to_file}");
$res['response_file'] = $save_to_file;
$res['file'] = $save_to_file;
return $res;
}
# return immediately if there's a checked downloaded file
if ($checked_saved_file) {
$res['response_file'] = $checked_saved_file;
$res['file'] = $checked_saved_file;
return $res;
}
}
# initiate request and $res
$c = curl_init();
if (!$c) {
return error($cmd, "could not create HTTP request");
}
curl_setopt($c, CURLOPT_RETURNTRANSFER, true);
curl_setopt($c, CURLOPT_HEADER, true);
curl_setopt($c, CURLINFO_HEADER_OUT, true);
$res = array();
# set request url
if (is_array($get_args)) {
$get_args = http_build_query($get_args);
}
if (is_string($get_args)) {
$url = "{$base_url}?{$get_args}";
} else {
$url = $base_url;
}
curl_setopt($c, CURLOPT_URL, $url);
if ($debug) {
debug_echo($cmd, "initiating HTTP request : {$url}");
}
# add headers to request
if ($request_headers) {
curl_setopt($c, CURLOPT_HTTPHEADER, $request_headers);
if ($debug) {
debug_echo($cmd, "setting HTTP request headers :");
debug_dump_yaml($request_headers, true);
}
}
# add body to request
$request_body = '';
if ($post_args) {
foreach ($post_args as $key => $value) {
$request_body .= urlencode($key) . '=' . urlencode($value) . '&';
}
$request_body = rtrim($request_body, '&');
curl_setopt($c, CURLOPT_POST, strlen($request_body));
curl_setopt($c, CURLOPT_POSTFIELDS, $request_body);
$request_method = 'post';
if ($debug) {
debug_echo($cmd, "HTTP post args :");
debug_dump_yaml($post_args, true);
}
} else {
switch ($request_method) {
case 'post':
curl_setopt($c, CURLOPT_POST, 1);
}
}
# set up downloading to file if set
if ($save_to_file) {
# backup old response file if set and exists
if (is_file($save_to_file) && $backup_old_saved_files && !backup_file($save_to_file, null, $cmd)) {
return error($cmd, "could not backup previously saved file : {$save_to_file}");
}
# create parent directory of response file
if (!make_dir($opts, dirname($save_to_file), $cmd)) {
return false;
}
# create file handler for response file
$save_to_file_handle = @fopen($save_to_file, "w");
if (!$save_to_file_handle) {
return error($cmd, "could not open response file : {$save_to_file}");
}
# add response file handler to request
curl_setopt($c, CURLOPT_FILE, $save_to_file_handle);
# display message and add to $res
debug_echo($cmd, "saving response to file : {$save_to_file}");
$res['response_file'] = $save_to_file;
$res['file'] = $save_to_file;
}
# add download progress update function if set
if ($download_progress_update) {
curl_setopt($c, CURLOPT_PROGRESSFUNCTION, 'curl_transfer_progress');
curl_setopt($c, CURLOPT_NOPROGRESS, false);
}
# display download message
if (!$download_msg) {
$download_msg = "executing HTTP request : {$url}";
}
debug_echo($cmd, "{$download_msg} ...");
# execute request
for ($i = 0; $i < $request_retries; $i++) {
$r = curl_exec($c);
if ($r) {
break;
}
}
if (!$r) {
return error($cmd, "could not execute HTTP request : {$url}");
}
# split the response into header and body
$request_head = curl_getinfo($c, CURLINFO_HEADER_OUT);
$response_head_size = curl_getinfo($c, CURLINFO_HEADER_SIZE);
if ($save_to_file) {
$handle = fopen($save_to_file, 'r');
$response_head = trim(fread($handle, $response_head_size));
$stat = fstat($handle);
$response_body_size = $stat['size'] - $response_head_size;
if ($response_body_size > $max_response_body_size) {
warning($cmd, "response body size is greater than max, so is not set : {$response_body_size} > {$max_response_body_size}");
$response_body = "[{$save_to_file}]";
$no_reset_response_body_size = true;
} else {
$response_body = trim(fread($handle, $response_body_size));
}
fclose($handle);
} else {
$response_head = trim(substr($r, 0, $response_head_size));
$response_body = trim(substr($r, $response_head_size));
}
$response_head_size = strlen($response_head);
if (!@$no_reset_response_body_size) {
$response_body_size = strlen($response_body);
}
# setup response header parsing
$headers = explode("\n", $response_head);
$response_headers = array();
$response_cookies = array();
$pipe_headers = array();
# parse response headers
foreach ($headers as $header) {
# clean and add response header
$header = trim($header);
if ($header != '') {
$response_headers[] = $header;
}
# parse cookies
if (substr($header, 0, 11) == 'Set-Cookie:') {
# add cookie to pipe headers
$full_cookie = trim(substr($header, 11));
$pipe_headers[] = "Cookie: {$full_cookie}";
# add cookie to cookies
$cookie = strstr($full_cookie, ';', true);
$name = urldecode(strstr($cookie, '=', true));
$val = urldecode(substr(strstr($cookie, '='), 1));
$response_cookies[$name] = $val;
}
# parse location
if (substr($header, 0, 9) == 'Location:') {
$res['response_location'] = trim(substr($header, 9));
}
}
# add main request and response variables to $res
$res['headers'] = $pipe_headers;
$res['request_url'] = $base_url;
$res['request_get_args'] = $get_args;
$res['request_full_url'] = $url;
$res['request_method'] = $request_method;
$res['request_post_args'] = $post_args;
$res['request_headers'] = $request_headers;
$res['response_headers'] = $response_headers;
$res['response_cookies'] = $response_cookies;
$res['response_head_size'] = $response_head_size;
$res['response_body_size'] = $response_body_size;
# parse response body for JSON
$parsed_body = @json_decode($response_body, true);
if (!is_null($parsed_body)) {
$res['response_body_json_decoded'] = $parsed_body;
}
# save data to files if set and save in $res
$save_to_files = array('request_head' => array($request_head_file, $request_head), 'request_body' => array($request_body_file, $request_body), 'response_head' => array($response_head_file, $response_head), 'response_body' => array($response_body_file, $response_body));
foreach ($save_to_files as $name => $vars) {
$file = $vars[0];
$data = $vars[1];
if ($file) {
$report_name = str_replace('_', ' ', $name);
if (is_file($file) && $backup_old_saved_files && !backup_file($file, null, $cmd)) {
return error($cmd, "could not backup previously saved {$report_name} file : {$file}");
}
if (!file_put_contents($file, $data)) {
return error($cmd, "could not save {$report_name} to file : {$file}");
}
debug_echo($cmd, "{$report_name} saved to file : {$file}");
$res["{$name}_file"] = $file;
}
$res[$name] = $data;
}
# parse response hidden inputs if set
if ($parse_response_form_args) {
// we don't pass the $opts as it does not work properly, and $pipe needs to be second
debug_echo($cmd, "parsing response for post action url and hidden args");
$res = http_parse_form($parse_response_form, $res, $cmd, true);
}
# close handlers
curl_close($c);
if (@$save_to_file_handle) {
fclose($save_to_file_handle);
}
# sort $res, display and return
ksort($res);
if ($debug) {
debug_echo($cmd, "HTTP response:");
@debug_dump_yaml($res, true);
}
# check for automatic redirection
$response_location = @$res['response_location'];
if ($response_location && $auto_redirect) {
debug_echo($cmd, "automatically redirecting to url : {$response_location}");
$res = merge_opts_for_output($res, $opts);
# avoid infinite loops because the url opt is checked before the response_location opt
unset($res['form_name']);
unset($res['post_args']);
unset($res['request_method']);
unset($res['url']);
return http_request($res);
}
return $res;
}
function merge_opts_for_data_processing($res, $opts)
{
# keys shared by create_data() and search_data() functions
$keys = array('data', 'fields', 'search_indexes', 'search_results');
return merge_opts_for_output($res, $opts, $keys);
}
function http_request($opts, $pipe, $cmd = __FUNCTION__)
{
# set up functions for curl to parse
$req = array('init_functions' => array('http_init_parse_headers', 'http_init_parse_forms', 'http_init_set_url', 'http_init_set_authorization', 'http_init_set_headers', 'http_init_set_post_args', 'http_init_set_other_options'), 'finalize_functions' => array('http_finalize_set_response_body_json_decoded', 'http_finalize_parse_forms'), 'allowed_protocols' => array('http', 'https'));
# merge the opts and pass to curl for processing
$opts = merge_opts_for_output($opts, $req);
return curl($opts, $pipe, $cmd, 'http');
}
