function load_data_from_lengths_file($opts, $pipe, $cmd = __FUNCTION__)
{
# set prefix
$prefix = 'load_data_from_lengths_file';
# merge opts
$opts = merge_opts($opts, $pipe, 'file');
# get file opt
$file = get_opt($prefix, $opts, 'file');
if (!check_opt_set_type($cmd, $file, 'file', 'string')) {
return false;
}
# get defined fields opt
$defined_fields = get_opt($prefix, $opts, 'defined_fields');
if (!check_opt_if_set_type($cmd, $defined_fields, 'defined_fields', 'array_of_strings,yaml_array_of_strings')) {
return false;
}
if (is_string($defined_fields)) {
$defined_fields = yaml_decode($defined_fields);
}
# get save fields
$save_fields = get_opt($prefix, $opts, 'save_fields');
if (!check_opt_if_set_type($cmd, $save_fields, 'save_fields', 'array_of_strings,yaml_array_of_strings')) {
return false;
}
if (is_string($save_fields)) {
$save_fields = yaml_decode($save_fields);
}
# get splits opt
$splits = get_opt($prefix, $opts, 'splits');
if (!check_opt_if_set_type($cmd, $splits, 'splits', 'array_of_integers,yaml_array_of_integers')) {
return false;
}
if (is_string($splits)) {
$splits = yaml_decode($splits);
}
# get lengths opt
$lengths = get_opt($prefix, $opts, 'lengths');
if (!check_opt_if_set_type($cmd, $lengths, 'lengths', 'array_of_integers,yaml_array_of_integers')) {
return false;
}
if (is_string($lengths)) {
$lengths = yaml_decode($lengths);
}
# get fields lengths opt
$fields_lengths = get_opt($prefix, $opts, 'fields_lengths');
if (!check_opt_if_set_type($cmd, $fields_lengths, 'fields_lengths', 'array,yaml_array')) {
return false;
}
if (is_string($fields_lengths)) {
$fields_lengths = yaml_decode($fields_lengths);
}
# get fields splits opt
$fields_splits = get_opt($prefix, $opts, 'fields_splits');
if (!check_opt_if_set_type($cmd, $fields_splits, 'fields_splits', 'array,yaml_array')) {
return false;
}
if (is_string($fields_splits)) {
$fields_splits = yaml_decode($fields_splits);
}
# get trim opt (i.e. whether the values should have leading and trailing whitespace removed)
$trim = get_opt_config_value($prefix, $opts, 'trim', true);
if (!check_opt_set_type($cmd, $trim, 'trim', 'boolean')) {
return false;
}
# get limit opt
$limit = get_opt($prefix, $opts, 'limit', 0);
if (!check_opt_set_type($cmd, $limit, 'limit', 'positive_integer')) {
return false;
}
# get offset opt
$offset = get_opt($prefix, $opts, 'offset', 0);
if (!check_opt_set_type($cmd, $offset, 'offset', 'positive_integer')) {
return false;
}
# check file exists
if (!file_exists($file)) {
return error($cmd, "file does not exist : {$file}");
}
# open file
$file_handle = fopen($file, 'r');
if (!$file_handle) {
return error($cmd, "cannot read file : {$file}");
}
debug_echo($cmd, "creating data from column file : {$file}");
# check for splits/lengths fields
if ($fields_splits) {
$defined_fields = array_keys($fields_splits);
$splits = array_values($fields_splits);
} elseif ($fields_lengths) {
$defined_fields = array_keys($fields_lengths);
$lengths = array_values($fields_lengths);
}
# check for lengths or splits
if ($lengths) {
$total = 0;
$splits = array();
foreach ($lengths as $length) {
$total += $length;
$splits[] = $total;
}
} elseif (!$splits) {
return opt_not_set_msg($cmd, "lengths,splits,fields_lengths,fields_splits");
}
# set up splits
$split_count = count($splits);
# get the fields from list or file
if ($defined_fields) {
$defined_fields_source = '(from the config)';
$field_count = count($defined_fields);
if ($field_count != $split_count) {
return error($cmd, "the field count ({$field_count}) is not the same as the split count ({$split_count})");
}
} else {
$defined_fields = array();
$line = fgets($file_handle);
for ($i = 0; $i < $split_count; $i++) {
$min = $splits[$i];
if ($i == 0) {
$min = 0;
$len = $splits[0];
} else {
$min = $splits[$i - 1];
$len = $splits[$i] - $min;
}
$val = substr($line, $min, $len);
if ($trim) {
$val = trim($val);
}
$defined_fields[] = $val;
}
$defined_fields_source = '(from the source file)';
}
# set up response field indexes
if ($save_fields) {
$res_fields = $save_fields;
$res_field_count = count($res_fields);
$res_field_indexes = build_field_indexes($defined_fields, $res_fields, $cmd);
if ($res_field_indexes === false) {
return;
}
} else {
$res_fields = $defined_fields;
}
# display info
debug_echo($cmd, "splits used for dividing data : " . json_encode($splits));
debug_echo($cmd, "defined fields {$defined_fields_source} : ");
debug_dump_list($defined_fields, true);
if ($save_fields) {
debug_echo($cmd, "saved fields :");
debug_dump_list($save_fields, true);
}
# parse the offset
for ($off_i = 0; $off_i < $offset; $off_i++) {
if (feof($file_handle)) {
break;
}
fgets($file_handle);
}
# notice about offset reaching
if ($offset > 0 && !feof($file_handle)) {
notice($cmd, "offset ({$offset}) reached");
}
# loop over lines to split the file
$limit_i = 0;
$res_data = array();
if ($save_fields) {
while ($line = fgets($file_handle)) {
# parse all the fields
$line_data = array();
for ($i = 0; $i < $split_count; $i++) {
$min = $splits[$i];
if ($i == 0) {
$min = 0;
$len = $splits[0];
} else {
$min = $splits[$i - 1];
$len = $splits[$i] - $min;
}
$val = substr($line, $min, $len);
if ($trim) {
$val = trim($val);
}
$line_data[] = $val;
}
# save just the save fields
$new_line = array();
for ($i = 0; $i < $res_field_count; $i++) {
$new_line[] = $line_data[$res_field_indexes[$i]];
}
$res_data[] = $new_line;
# load_data_line ()
# check the limit
if ($limit != 0) {
$limit_i++;
if ($limit_i >= $limit) {
notice($cmd, "limit ({$limit}) reached");
break;
}
}
}
} else {
while ($line = fgets($file_handle)) {
# parse all the fields
$line_data = array();
for ($i = 0; $i < $split_count; $i++) {
$min = $splits[$i];
if ($i == 0) {
$min = 0;
$len = $splits[0];
} else {
$min = $splits[$i - 1];
$len = $splits[$i] - $min;
}
$val = substr($line, $min, $len);
if ($trim) {
$val = trim($val);
}
$line_data[] = $val;
}
$res_data[] = $line_data;
# load_data_line ()
# check the limit
if ($limit != 0) {
$limit_i++;
if ($limit_i >= $limit) {
notice($cmd, "limit ({$limit}) reached");
break;
}
}
}
}
# display info about reaching the end of file
if (feof($file_handle)) {
$total_lines = $off_i + $limit_i;
notice($cmd, "end of file reached after {$total_lines} lines of data (offset = {$offset}, limit = {$limit})");
}
fclose($file_handle);
# detail results
$line_count = count($res_data);
debug_echo($cmd, "creation of data from column file complete ({$line_count} lines processed)");
# create result
return build_result_data($res_fields, $res_data);
}
function http_request($opts, $pipe = null, $cmd = __FUNCTION__)
{
# set prefix
$prefix = 'http';
# merge opts
$opts = merge_opts($opts, $pipe, 'url');
# get execute opt
$execute = get_opt_config_value($prefix, $opts, 'execute', true);
if (!check_opt_set_type($cmd, $execute, 'execute', 'boolean')) {
return false;
}
# check if we should execute or not
if (!$execute) {
return;
}
# get debug opt
$debug = get_opt_config_value($prefix, $opts, 'debug', false);
if (!check_opt_set_type($cmd, $debug, 'debug', 'boolean')) {
return false;
}
# get request method opt
$request_method = get_opt_config_value($prefix, $opts, 'request_method', 'get');
echo "{$request_method}\\[1]\n";
if (!check_opt_if_set_type($cmd, $request_method, 'request_method', 'http_request_method')) {
return false;
}
# get post args opt
$post_args = get_opt($prefix, $opts, 'post_args');
if (!check_opt_if_set_type($cmd, $post_args, 'post_args', 'array')) {
return false;
}
# get post form opt
$form_name = get_opt($prefix, $opts, 'form_name');
if (!check_opt_if_set_type($cmd, $form_name, 'form_name', 'string')) {
return false;
}
# get response body opt
$response_body = get_opt($prefix, $opts, 'response_body');
if (!check_opt_if_set_type($cmd, $response_body, 'response_body', 'string')) {
return false;
}
# check to see if we're posting
if ($post_args || $form_name) {
$request_method = 'post';
}
if ($request_method == 'post' && $response_body) {
debug_echo($cmd, "performing a post request and there was a previous request, so will search for action, method and inputs");
$opts = http_parse_form($opts, $form_name, $cmd);
if ($opts === false) {
return false;
}
$post_args = $opts['post_args'];
}
echo "{$request_method}\\[2\\]\n";
# get base url from url|response_location|request opt
$base_url_opts = array('url', 'response_location', 'form_action', 'response_form_action', 'request_url');
foreach ($base_url_opts as $opt) {
$base_url = get_opt($prefix, $opts, $opt);
if (!check_opt_if_set_type($cmd, $base_url, $opt, 'string')) {
return false;
}
if ($base_url) {
break;
}
}
if (!$base_url) {
return opt_not_set_msg($cmd, implode(',', $base_url_opts));
}
# get get args opt
$get_args = get_opt($prefix, $opts, 'get_args');
if (!check_opt_if_set_type($cmd, $get_args, 'get_args', 'array,string')) {
return false;
}
# get headers opt
$request_headers = get_opt($prefix, $opts, 'headers');
if (!check_opt_if_set_type($cmd, $request_headers, 'headers', 'array_of_strings')) {
return false;
}
# get auto redirect opt
$auto_redirect = get_opt($prefix, $opts, 'auto_redirect', true);
if (!check_opt_set_type($cmd, $auto_redirect, 'auto_redirect', 'boolean')) {
return false;
}
# get authorization opt
$authorization = get_opt($prefix, $opts, 'authorization');
if (!check_opt_if_set_type($cmd, $authorization, 'authorization', 'array')) {
return false;
}
# set authorization if necessary
if ($authorization) {
# get authorization:usr opt
$usr = get_opt($prefix, $authorization, 'usr');
if (!check_opt_set_type($cmd, $usr, 'authorization:usr', 'string,integer')) {
return false;
}
$usr = (string) $usr;
# get authorization:pwd opt
$pwd = get_opt($prefix, $authorization, 'pwd');
if (!check_opt_set_type($cmd, $pwd, 'authorization:pwd', 'string,integer')) {
return false;
}
$pwd = (string) $pwd;
# get authorization:type opt
$type = get_opt($prefix, $authorization, 'type', 'Basic');
if (!check_opt_set_type($cmd, $type, 'authorization:type', 'string')) {
return false;
}
# add the authorization header to the list of headers
$auth_header = "Authorization: {$type} " . base64_encode("{$usr}:{$pwd}");
if (!$request_headers) {
$request_headers = array();
}
$request_headers[] = $auth_header;
}
# get request retries opt
$request_retries = get_opt_config_value($prefix, $opts, 'request_retries', 3);
if (!check_opt_set_type($cmd, $request_retries, 'request_retries', 'integer')) {
return false;
}
# get download msg opt
$download_msg = get_opt($prefix, $opts, 'download_msg');
if (!check_opt_if_set_type($cmd, $download_msg, 'download_msg', 'string')) {
return false;
}
# get use downloaded file opt
$use_saved_file = get_opt_config_value($prefix, $opts, 'use_saved_file', true);
if (!check_opt_set_type($cmd, $use_saved_file, 'use_saved_file', 'boolean')) {
return false;
}
# get backup old downloaded files
$backup_old_saved_files = get_opt_config_value($prefix, $opts, 'backup_old_saved_files', true);
if (!check_opt_set_type($cmd, $backup_old_saved_files, 'backup_old_saved_files', 'boolean')) {
return false;
}
# get checked downloaded file opt
$checked_saved_file = get_config_value('http_checked_saved_file');
if (!check_opt_if_set_type($cmd, $checked_saved_file, 'http_checked_saved_file', 'string')) {
return false;
}
# get download to file opt
$save_to_file = get_opt($prefix, $opts, 'save_to_file');
if (!check_opt_if_set_type($cmd, $save_to_file, 'save_to_file', 'string')) {
return false;
}
# get download progress update opt
$download_progress_update = get_opt_config_value($prefix, $opts, 'download_progress_update', true);
if (!check_opt_set_type($cmd, $download_progress_update, 'download_progress_update', 'boolean')) {
return false;
}
# get max response body size opt
$max_response_body_size = get_opt_config_value($prefix, $opts, 'max_response_body_size', 32 * 1024 * 1024);
if (!check_opt_if_set_type($cmd, $max_response_body_size, 'max_response_body_size', 'integer')) {
return false;
}
# get save request head to file opt
$request_head_file = get_opt($prefix, $opts, 'save_request_head_to_file');
if (!check_opt_if_set_type($cmd, $request_head_file, 'save_request_head_to_file', 'string')) {
return false;
}
# get save request body to file opt
$request_body_file = get_opt($prefix, $opts, 'save_request_body_to_file');
if (!check_opt_if_set_type($cmd, $request_body_file, 'save_request_body_to_file', 'string')) {
return false;
}
# get save head to file opt
$response_head_file = get_opt($prefix, $opts, 'save_head_to_file');
if (!check_opt_if_set_type($cmd, $response_head_file, 'save_head_to_file', 'string')) {
return false;
}
# get save body to file opt
$response_body_file = get_opt($prefix, $opts, 'save_body_to_file');
if (!check_opt_if_set_type($cmd, $response_body_file, 'save_body_to_file', 'string')) {
return false;
}
# get parse response form opt
$parse_response_form = get_opt_config_value($prefix, $opts, 'parse_response_form');
if (!check_opt_if_set_type($cmd, $parse_response_form, 'parse_response_form', 'string')) {
return false;
}
# get parse response hidden inputs opt
$parse_response_form_args = get_opt_config_value($prefix, $opts, 'parse_response_form_args', false);
if (!check_opt_set_type($cmd, $parse_response_form_args, 'parse_response_form_args', 'boolean')) {
return false;
}
# check to see if we should use a downloaded file
if ($use_saved_file) {
# check if response body file exists
if (file_exists($response_body_file)) {
debug_echo($cmd, "using saved response body file instead of executing the HTTP request : {$response_body_file}");
$res['response_body_file'] = $response_body_file;
$res['response_file'] = $response_body_file;
$res['file'] = $response_body_file;
return $res;
}
# check if the save to file exists
if (file_exists($save_to_file)) {
debug_echo($cmd, "using saved file instead of executing the HTTP request : {$save_to_file}");
$res['response_file'] = $save_to_file;
$res['file'] = $save_to_file;
return $res;
}
# return immediately if there's a checked downloaded file
if ($checked_saved_file) {
$res['response_file'] = $checked_saved_file;
$res['file'] = $checked_saved_file;
return $res;
}
}
# initiate request and $res
$c = curl_init();
if (!$c) {
return error($cmd, "could not create HTTP request");
}
curl_setopt($c, CURLOPT_RETURNTRANSFER, true);
curl_setopt($c, CURLOPT_HEADER, true);
curl_setopt($c, CURLINFO_HEADER_OUT, true);
$res = array();
# set request url
if (is_array($get_args)) {
$get_args = http_build_query($get_args);
}
if (is_string($get_args)) {
$url = "{$base_url}?{$get_args}";
} else {
$url = $base_url;
}
curl_setopt($c, CURLOPT_URL, $url);
if ($debug) {
debug_echo($cmd, "initiating HTTP request : {$url}");
}
# add headers to request
if ($request_headers) {
curl_setopt($c, CURLOPT_HTTPHEADER, $request_headers);
if ($debug) {
debug_echo($cmd, "setting HTTP request headers :");
debug_dump_yaml($request_headers, true);
}
}
# add body to request
$request_body = '';
if ($post_args) {
foreach ($post_args as $key => $value) {
$request_body .= urlencode($key) . '=' . urlencode($value) . '&';
}
$request_body = rtrim($request_body, '&');
curl_setopt($c, CURLOPT_POST, strlen($request_body));
curl_setopt($c, CURLOPT_POSTFIELDS, $request_body);
$request_method = 'post';
if ($debug) {
debug_echo($cmd, "HTTP post args :");
debug_dump_yaml($post_args, true);
}
} else {
switch ($request_method) {
case 'post':
curl_setopt($c, CURLOPT_POST, 1);
}
}
# set up downloading to file if set
if ($save_to_file) {
# backup old response file if set and exists
if (is_file($save_to_file) && $backup_old_saved_files && !backup_file($save_to_file, null, $cmd)) {
return error($cmd, "could not backup previously saved file : {$save_to_file}");
}
# create parent directory of response file
if (!make_dir($opts, dirname($save_to_file), $cmd)) {
return false;
}
# create file handler for response file
$save_to_file_handle = @fopen($save_to_file, "w");
if (!$save_to_file_handle) {
return error($cmd, "could not open response file : {$save_to_file}");
}
# add response file handler to request
curl_setopt($c, CURLOPT_FILE, $save_to_file_handle);
# display message and add to $res
debug_echo($cmd, "saving response to file : {$save_to_file}");
$res['response_file'] = $save_to_file;
$res['file'] = $save_to_file;
}
# add download progress update function if set
if ($download_progress_update) {
curl_setopt($c, CURLOPT_PROGRESSFUNCTION, 'curl_transfer_progress');
curl_setopt($c, CURLOPT_NOPROGRESS, false);
}
# display download message
if (!$download_msg) {
$download_msg = "executing HTTP request : {$url}";
}
debug_echo($cmd, "{$download_msg} ...");
# execute request
for ($i = 0; $i < $request_retries; $i++) {
$r = curl_exec($c);
if ($r) {
break;
}
}
if (!$r) {
return error($cmd, "could not execute HTTP request : {$url}");
}
# split the response into header and body
$request_head = curl_getinfo($c, CURLINFO_HEADER_OUT);
$response_head_size = curl_getinfo($c, CURLINFO_HEADER_SIZE);
if ($save_to_file) {
$handle = fopen($save_to_file, 'r');
$response_head = trim(fread($handle, $response_head_size));
$stat = fstat($handle);
$response_body_size = $stat['size'] - $response_head_size;
if ($response_body_size > $max_response_body_size) {
warning($cmd, "response body size is greater than max, so is not set : {$response_body_size} > {$max_response_body_size}");
$response_body = "[{$save_to_file}]";
$no_reset_response_body_size = true;
} else {
$response_body = trim(fread($handle, $response_body_size));
}
fclose($handle);
} else {
$response_head = trim(substr($r, 0, $response_head_size));
$response_body = trim(substr($r, $response_head_size));
}
$response_head_size = strlen($response_head);
if (!@$no_reset_response_body_size) {
$response_body_size = strlen($response_body);
}
# setup response header parsing
$headers = explode("\n", $response_head);
$response_headers = array();
$response_cookies = array();
$pipe_headers = array();
# parse response headers
foreach ($headers as $header) {
# clean and add response header
$header = trim($header);
if ($header != '') {
$response_headers[] = $header;
}
# parse cookies
if (substr($header, 0, 11) == 'Set-Cookie:') {
# add cookie to pipe headers
$full_cookie = trim(substr($header, 11));
$pipe_headers[] = "Cookie: {$full_cookie}";
# add cookie to cookies
$cookie = strstr($full_cookie, ';', true);
$name = urldecode(strstr($cookie, '=', true));
$val = urldecode(substr(strstr($cookie, '='), 1));
$response_cookies[$name] = $val;
}
# parse location
if (substr($header, 0, 9) == 'Location:') {
$res['response_location'] = trim(substr($header, 9));
}
}
# add main request and response variables to $res
$res['headers'] = $pipe_headers;
$res['request_url'] = $base_url;
$res['request_get_args'] = $get_args;
$res['request_full_url'] = $url;
$res['request_method'] = $request_method;
$res['request_post_args'] = $post_args;
$res['request_headers'] = $request_headers;
$res['response_headers'] = $response_headers;
$res['response_cookies'] = $response_cookies;
$res['response_head_size'] = $response_head_size;
$res['response_body_size'] = $response_body_size;
# parse response body for JSON
$parsed_body = @json_decode($response_body, true);
if (!is_null($parsed_body)) {
$res['response_body_json_decoded'] = $parsed_body;
}
# save data to files if set and save in $res
$save_to_files = array('request_head' => array($request_head_file, $request_head), 'request_body' => array($request_body_file, $request_body), 'response_head' => array($response_head_file, $response_head), 'response_body' => array($response_body_file, $response_body));
foreach ($save_to_files as $name => $vars) {
$file = $vars[0];
$data = $vars[1];
if ($file) {
$report_name = str_replace('_', ' ', $name);
if (is_file($file) && $backup_old_saved_files && !backup_file($file, null, $cmd)) {
return error($cmd, "could not backup previously saved {$report_name} file : {$file}");
}
if (!file_put_contents($file, $data)) {
return error($cmd, "could not save {$report_name} to file : {$file}");
}
debug_echo($cmd, "{$report_name} saved to file : {$file}");
$res["{$name}_file"] = $file;
}
$res[$name] = $data;
}
# parse response hidden inputs if set
if ($parse_response_form_args) {
// we don't pass the $opts as it does not work properly, and $pipe needs to be second
debug_echo($cmd, "parsing response for post action url and hidden args");
$res = http_parse_form($parse_response_form, $res, $cmd, true);
}
# close handlers
curl_close($c);
if (@$save_to_file_handle) {
fclose($save_to_file_handle);
}
# sort $res, display and return
ksort($res);
if ($debug) {
debug_echo($cmd, "HTTP response:");
@debug_dump_yaml($res, true);
}
# check for automatic redirection
$response_location = @$res['response_location'];
if ($response_location && $auto_redirect) {
debug_echo($cmd, "automatically redirecting to url : {$response_location}");
$res = merge_opts_for_output($res, $opts);
# avoid infinite loops because the url opt is checked before the response_location opt
unset($res['form_name']);
unset($res['post_args']);
unset($res['request_method']);
unset($res['url']);
return http_request($res);
}
return $res;
}
function check_opt_if_set_type($cmd, $opt, $name, $types, $msg_type_if_set = 'error', $msg_type_if_not_set = 'none')
{
if (is_null($opt)) {
if ($msg_type_if_not_set) {
opt_not_set_msg($cmd, $name, $msg_type_if_not_set);
if ($msg_type_if_not_set == 'error') {
return false;
}
}
return true;
}
return check_opt_type($cmd, $opt, $name, $types, $msg_type_if_set);
}
function merge_relational_data($opts, $pipe, $cmd = __FUNCTION__)
{
# set prefix
$prefix = 'merge_relational_data';
# merge opts
$opts = merge_opts($opts, $pipe);
# get data opt
$data = get_opt_config_value($prefix, $opts, 'data');
if (!check_opt_set_type($cmd, $data, 'data', 'array')) {
return false;
}
# get fields opt
$fields = get_opt_config_value($prefix, $opts, 'fields');
if (!check_opt_set_type($cmd, $fields, 'fields', 'array_of_strings')) {
return false;
}
# get primary key opt
$primary_key = get_opt_config_value($prefix, $opts, 'primary_key');
if (!check_opt_set_type($cmd, $primary_key, 'primary_key', 'string')) {
return false;
}
# get check field opt
$merge_rules = get_opt_config_value($prefix, $opts, 'merge_rules');
if (!check_opt_set_type($cmd, $merge_rules, 'merge_rules', 'array,yaml_array')) {
return false;
}
if (is_string($merge_rules)) {
$merge_rules = yaml_decode($merge_rules);
}
# set up primary key
if (!in_array($primary_key, $fields)) {
return error($cmd, "field '{$primary_key}' does not exist");
}
$fields_flipped = array_flip($fields);
$primary_key_index = $fields_flipped[$primary_key];
# set up new fields
$new_fields = array();
foreach ($fields as $field) {
$new_fields[] = $field;
}
# loop through all the rules
$rules = array();
foreach ($merge_rules as $i => $merge_rule) {
$rule = array();
# get check field opt
$check_field = get_opt_config_value($prefix, $merge_rule, 'check_field');
if (!check_opt_set_type($cmd, $check_field, 'check_field', 'string')) {
return false;
}
# check that the fields exist
if (!in_array($check_field, $fields)) {
return error($cmd, "field '{$check_field}' does not exist");
}
$check_field_index = $fields_flipped[$check_field];
$rule[0] = $check_field_index;
# get value field opt
$value_field = get_opt_config_value($prefix, $merge_rule, 'value_field');
if (!check_opt_if_set_type($cmd, $value_field, 'value_field', 'string')) {
return false;
}
# get value field map opt
$value_field_map = get_opt_config_value($prefix, $merge_rule, 'value_field_map');
if (!check_opt_if_set_type($cmd, $value_field_map, 'value_field_map', 'array,yaml_array')) {
return false;
}
if ($value_field_map && !is_array($value_field_map)) {
$value_field_map = yaml_decode($value_field_map);
}
if ($value_field === null && $value_field_map === null) {
return opt_not_set_msg($cmd, 'value_field,value_field_map');
}
# check the type of the rule
if ($value_field !== null) {
$rule[1] = 'f';
if (!in_array($value_field, $fields)) {
return error($cmd, "field '{$value_field}' does not exist");
}
$value_field_index = $fields_flipped[$value_field];
$rule[2] = $value_field_index;
} else {
$rule[1] = 'fm';
# set up value field map
foreach ($value_field_map as $value => $field) {
if (!in_array($field, $new_fields)) {
$new_fields[] = $field;
}
}
# create the value indexes
$new_fields_flipped = array_flip($new_fields);
$values_indexes = array();
foreach ($value_field_map as $value => $field) {
$values_indexes[$value] = $new_fields_flipped[$field];
}
$rule[2] = $values_indexes;
}
# get value opt
# TODO: value_map
if (array_key_exists('value', $merge_rule)) {
$rule[3] = 'v';
$rule[4] = $merge_rule['value'];
} elseif (array_key_exists('value_map', $merge_rule)) {
$value_map = get_opt_config_value($prefix, $merge_rule, 'value_map');
if (!check_opt_set_type($cmd, $value_map, 'value_map', 'array,yaml_array')) {
return false;
}
if (is_string($value_map)) {
$value_map = yaml_decode($value_map);
}
$rule[3] = 'vm';
$rule[4] = $value_map;
} else {
$rule[3] = 'cv';
}
# add rule to rules
$rules[] = $rule;
}
# set up counts
$data_count = count($data);
$field_count = count($fields);
$new_field_count = count($new_fields);
# set up new data
$new_data = array();
$new_data_indexes = array();
# loop over data and apply rules
debug_echo($cmd, "merging relational data (this can take some time) ...");
for ($i = 0; $i < $data_count; $i++) {
# get line and check if line already exists
$line = $data[$i];
$primary_value = $line[$primary_key_index];
unset($new_line);
$new_line =& $new_data_indexes[$primary_value];
# create new line
if (!$new_line) {
unset($new_line);
$new_line = array();
for ($j = 0; $j < $field_count; $j++) {
$new_line[$j] = $line[$j];
}
for ($j = $field_count; $j < $new_field_count; $j++) {
$new_line[] = null;
}
$new_data[] =& $new_line;
$new_data_indexes[$primary_value] =& $new_line;
}
# loop through the rules to modify the data
foreach ($rules as $rule) {
# set up rule parts
$check_field_index = $rule[0];
$value_field_type = $rule[1];
$value_field_index = $rule[2];
$value_type = $rule[3];
# intiiate check
$check_value = $line[$check_field_index];
if ($check_value === null) {
continue;
}
# get value field
switch ($value_field_type) {
case 'fm':
$value_field_index = @$value_field_index[$check_value];
break;
}
if ($value_field_index === null) {
continue;
}
# get value
switch ($value_type) {
case 'v':
$value = $rule[4];
break;
case 'vm':
$value = $rule[4][$check_value];
break;
case 'cv':
$value = $check_value;
break;
}
# set the value
$new_line[$value_field_index] = $value;
}
}
# build result
$res = array('data' => $new_data, 'fields' => $new_fields);
return $res;
}
function http_init_set_url($prefix, $func_args, $cmd, $debug)
{
# get opts and curlopts
$opts =& $func_args['opts'];
$curlopts =& $func_args['curlopts'];
# get base url from url|response_location|request opt
$base_url_opts = array('url', 'response_location', 'form_action', 'response_form_action', 'request_url');
foreach ($base_url_opts as $opt) {
$base_url = get_opt($prefix, $opts, $opt);
if (!check_opt_if_set_type($cmd, $base_url, $opt, 'string')) {
return false;
}
if ($base_url) {
if ($debug) {
debug_echo($cmd, "using the value from option '{$opt}' for the base url : {$base_url}");
}
break;
}
}
if (!$base_url) {
return opt_not_set_msg($cmd, implode(',', $base_url_opts));
}
# get get args opt
$get_args = get_opt($prefix, $opts, 'get_args');
if (!check_opt_if_set_type($cmd, $get_args, 'get_args', 'array,string')) {
return false;
}
# build url
if (is_array($get_args)) {
$get_args = http_build_query($get_args);
}
if (is_string($get_args)) {
$url = "{$base_url}?{$get_args}";
} else {
$url = $base_url;
}
# set url
$opts['curlopts']['url'] = $url;
return true;
}
