function array_strip_offset($opts, $pipe, $cmd = __FUNCTION__) { # merge opts $opts = merge_opts($opts, $pipe, 'offset'); # get the list and offset if (is_array($opts)) { if (array_key_exists('offset', $opts)) { $offset = $opts['offset']; } else { return error($cmd, "no offset defined"); } if (array_key_exists('list', $opts)) { $list = $opts; } else { $list = $pipe; } } else { $list = $pipe; $offset = (int) $opts; if ($offset <= 0) { return $list; } } # check validity of list and offset if (!check_opt_set_type($cmd, $list, 'list', 'array_of_strings')) { return false; } if (!check_opt_set_type($cmd, $offset, 'offset', 'integer')) { return false; } # strip offset recursively $list = array_strip_offset_util($list, $offset); return $list; }
function convert_data_to_objects($opts, $pipe, $cmd = __FUNCTION__) { # set prefix $prefix = 'convert_data_to_objects'; # merge opts $opts = merge_opts($opts, $pipe); # get fields opt $fields = get_opt($prefix, $opts, 'fields'); if (!check_opt_set_type($cmd, $fields, 'fields', 'array_of_strings')) { return false; } # get data opt $data = get_opt($prefix, $opts, 'data'); if (!check_opt_set_type($cmd, $data, 'data', 'array_of_arrays')) { return false; } # initiate response $field_count = count($fields); $data_objects = array(); # loop over data debug_echo($cmd, "converting data to objects"); foreach ($data as $line) { $data_object = array(); for ($i = 0; $i < $field_count; $i++) { $data_object[$fields[$i]] = $line[$i]; } $data_objects[] = $data_object; } # build res $res = array('data_objects' => $data_objects); return $res; }
function create_csv_file($opts, $pipe, $cmd = __FUNCTION__) { # set prefix $prefix = 'create_csv'; # merge opts $opts = merge_opts($opts, $pipe, 'file'); # get file opt $file = get_opt($prefix, $opts, 'file'); if (!check_opt_set_type($cmd, $file, 'file', 'string')) { return false; } # get fields opt $fields = get_opt($prefix, $opts, 'fields'); if (!check_opt_set_type($cmd, $fields, 'fields', 'array_of_strings')) { return false; } # get data opt $data = get_opt($prefix, $opts, 'data'); if (!check_opt_set_type($cmd, $data, 'data', 'array')) { return false; } # make parent dir if (!make_dir($opts, dirname($file))) { return false; } # create temp file $file_handle = @fopen('php://temp/csv-out', 'w'); if (!$file_handle) { return error($cmd, "cannot create temp CSV file"); } # output the fields if (!fputcsv($file_handle, $fields)) { return error($cmd, "could not output the fields CSV file : {$file}"); } # output the data foreach ($data as $line_no => $line) { if (!fputcsv($file_handle, $line)) { return error($cmd, "could not output line {$line_no} to CSV file : {$file}"); } } # grab contents and close temp file rewind($file_handle); $text = stream_get_contents($file_handle); fclose($file_handle); # create file and if (!file_put_contents($file, $text)) { return error($cmd, "cannot store CSV data to file : {$file}"); } # report $lines = count($data); debug_echo($cmd, "CSV file created ({$lines} lines) : {$file}"); # return if ($pipe === false) { return $file; } return array('file' => $file); }
function message($cmd, $msg, $type, $debug = false) { # adjust output for if is a normal command if (is_null($msg) || is_array($msg)) { # set the message if (is_string($cmd)) { $msg = $cmd; } elseif (is_array($cmd)) { $msg = get_opt($cmd, 'msg'); if (!check_opt_set_type($cmd, $msg, 'msg', 'string')) { return false; } } # set the command if ($debug) { $cmd = "debug_{$type}"; } else { $cmd = $type; } } else { # split the substring from the command $cmd = explode(':', $cmd); $cmd = $cmd[0]; } # switch on type switch ($type) { case 'error': case 'warning': case 'notice': case 'echo': break; case 'none': return true; default: $msg = "unknown message type '{$type}'"; $type = 'error'; dump_yaml(debug_backtrace); } # build output array $out = array(); if ($cmd && $cmd != $type && $cmd != "debug_{$type}") { $out[] = "[{$cmd}] "; } if ($type != 'echo') { $out[] = '[' . strtoupper($type) . '] '; } $out[] = ucfirst($msg); $out = implode($out); # echo message echo "{$out}\n"; # return false on error if ($type == 'error') { $GLOBALS['error_msg'] = trim($out); return false; } }
function build_indexes($opts, $pipe, $cmd = __FUNCTION__, $msg_suffix = null) { # set prefix $prefix = 'build_indexes'; # merge opts if ($pipe !== null) { $opts = merge_opts($opts, $pipe, 'indexes'); } # get fields opt $fields = get_opt($prefix, $opts, 'fields'); if (!check_opt_set_type($cmd, $fields, 'fields', 'array_of_strings')) { return false; } # get data opt $data =& get_opt($prefix, $opts, 'data'); if (!check_opt_set_type($cmd, $data, 'data', 'array')) { return false; } # get indexes opt $indexes =& get_opt($prefix, $opts, 'indexes'); if (!check_opt_set_type($cmd, $indexes, 'indexes', 'array_of_strings')) { return false; } # set up indexes $res_indexes_assoc = array(); $res_indexes_list = array(); foreach ($indexes as $index_name) { $index = array(); $res_indexes_assoc[$index_name] =& $index; $res_indexes_list[] =& $index; } # set up field indexes $field_indexes = build_field_indexes($fields, $indexes, $cmd, $msg_suffix); if ($field_indexes === false) { return false; } $index_count = count($indexes); # loop over data to build indexes for ($i = 0; $i < count($data); $i++) { unset($line); $line =& $data[$i]; for ($j = 0; $j < $index_count; $j++) { $idx = $field_indexes[$j]; $key = $line[$idx]; $res_indexes_list[$j][$key] =& $line; } } # sort the indexes foreach ($res_indexes_assoc as $name => $index) { ksort($index); } # save the indxes $opts['indexes'] = $res_indexes_assoc; return $opts; }
function clean_data($opts, $pipe, $cmd = __FUNCTION__) { # set prefix $prefix = 'clean_data'; # merge opts $opts = merge_opts($opts, $pipe); # get data opt $data = get_opt_config_value($prefix, $opts, 'data'); if (!check_opt_set_type($cmd, $data, 'data', 'array')) { return false; } # get fields opt $fields = get_opt_config_value($prefix, $opts, 'fields'); if (!check_opt_set_type($cmd, $fields, 'fields', 'array_of_strings')) { return false; } # get old value opt $old_value = get_opt_config_value($prefix, $opts, 'old_value'); # get new value opt $new_value = get_opt_config_value($prefix, $opts, 'new_value'); # set up counts $data_count = count($data); $field_count = count($fields); # messages debug_echo($cmd, "cleaning data (old value : {$old_value}, new value : {$new_value})"); # loop through data for ($i = 0; $i < $data_count; $i++) { unset($line); $line =& $data[$i]; for ($j = 0; $j < $field_count; $j++) { if ($line[$j] === $old_value) { $line[$j] = $new_value; } } } # set up result $res = array('data' => $data, 'fields' => $fields); return $res; }
function modify_rows($opts, $pipe, $cmd = __FUNCTION__) { # set prefix $prefix = 'modify_rows'; # merge opts $opts = merge_opts($opts, $pipe); # get data opt $data = get_opt_config_value($prefix, $opts, 'data'); if (!check_opt_set_type($cmd, $data, 'data', 'array')) { return false; } # get fields opt $res_fields = get_opt_config_value($prefix, $opts, 'fields'); if (!check_opt_set_type($cmd, $res_fields, 'fields', 'array_of_strings')) { return false; } # get limit opt $limit = get_opt_config_value($prefix, $opts, 'limit'); if (!check_opt_set_type($cmd, $limit, 'limit', 'integer')) { return false; } # get limit opt $offset = get_opt_config_value($prefix, $opts, 'offset', 0); if (!check_opt_set_type($cmd, $offset, 'offset', 'integer')) { return false; } # display message debug_echo($cmd, "modifying rows (limit = {$limit}, offset = {$offset})"); # set up end $end = $offset + $limit; # process the data $res_data = array(); for ($i = $offset; $i < $end; $i++) { $res_data[] = $data[$i]; } # build response object return build_result_data($res_fields, $res_data); }
function ftp_get_files($opts, $pipe, $cmd = __FUNCTION__) { # set prefix $prefix = $cmd; # merge opts $opts = merge_opts($opts, $pipe, 'files'); # get files opt $files = get_opt_config_value($prefix, $opts, 'files'); if (!check_opt_set_type($cmd, $files, 'files', 'array_of_strings')) { return false; } # get local dir opt $local_dir = get_opt_config_value($prefix, $opts, 'local_dir'); if (!check_opt_if_set_type($cmd, $local_dir, 'local_dir', 'string')) { return false; } # loop over files $res_files = array(); foreach ($files as $file) { # get the save_to_file value if ($local_dir) { $file_name = basename($file); $save_to_file = "{$local_dir}/{$file_name}"; } else { $save_to_file = $file; } # process request $req = array('file' => $file, 'save_to_file' => $save_to_file); $r = ftp_get_file($req, $opts, $cmd); if ($r === false) { return false; } # add file to return list $res_files[] = $save_to_file; } return array('files' => $res_files); }
function process_data($opts, $pipe, $cmd = __FUNCTION__, $opt_prefix = false) { # set prefix $prefix = 'process_data'; # merge opts $opts = merge_opts($opts, $pipe); # adjust opt prefix $cmd = adjust_opt_prefix($cmd, $opts, $opt_prefix, 'edit'); # get debug opt $debug = get_opt_config_value($prefix, $opts, 'debug', false); if (!check_opt_set_type($cmd, $debug, 'debug', 'boolean')) { return false; } # get type opt $join_type = get_opt_config_value($prefix, $opts, 'join_type', 'outer'); if (!check_opt_set_type($cmd, $join_type, 'type', 'data_join_type')) { return false; } # get sources opt $sources = get_opt($prefix, $opts, 'sources'); if (!check_opt_if_set_type($cmd, $sources, 'sources', 'array')) { return false; } if (!$sources) { $source = build_source_data($prefix, $opts, $cmd); if (!$source) { return error($cmd, "sources opt is not set and the data and fields opts are not set either, so can't be used to create one"); } $sources = array($source); } # get sources values foreach ($sources as $i => $val) { $sources[$i] = value($val); } # get create_indexes opt $create_indexes = get_opt($prefix, $opts, 'create_indexes'); if (!check_opt_if_set_type($cmd, $create_indexes, 'create_indexes', 'array_of_strings')) { return false; } # get join field opt $primary_key = get_opt_config_value($prefix, $opts, 'primary_key', 'guid'); if (!check_opt_set_type($cmd, $primary_key, 'primary_key', 'string')) { return false; } # get mawk rules opt $mawk = get_opt_config_value($prefix, $opts, 'mawk'); if (!check_opt_set_type($cmd, $mawk, 'mawk', 'string')) { return false; } # get update frequency opt $update_frequency = get_opt_config_value($prefix, $opts, 'update_frequency', 0); if (!check_opt_set_type($cmd, $update_frequency, 'update_frequency', 'integer')) { return false; } # create fields and data arrays $fields_arr = array(); $data_arr = array(); /* for ($i=0; $i<count ($sources); $i++) { $source_no = $i+1; $fields = $sources[$i]['fields']; if (!$fields) return error ($cmd, "no fields defined for source $source_no"); $fields_arr[] = $fields; if ($i == 0) { $data_arr[] = $sources[$i]['data']; } else { $data = $sources[$i]['indexes'][$primary_key]; if ($data === null) { return error ($cmd, "index on primary key '$primary_key' does not exis for source $source_no"); # TODO: auto-create } $data_arr[] = $data; } }*/ $sources_count = count($sources); for ($i = 0; $i < $sources_count; $i++) { $source_no = $i + 1; # add fields $fields = $sources[$i]['fields']; if (!$fields) { return error($cmd, "no fields defined for source {$source_no}"); } if ($sources_count > 1) { if (!in_array($primary_key, $fields)) { return error($cmd, "primary key '{$primary_key}' not defined for source {$source_no}"); } } $fields_arr[] = $fields; # add data $data = $sources[$i]['data']; if (!$data) { return error($cmd, "no data defined for source {$source_no}"); } $data_arr[] = $data; } # display info about the join, if there is one if (count($fields_arr) > 1) { debug_echo($cmd, "performing {$join_type} join on primary key / field '{$primary_key}'"); } for ($i = 1; $i <= count($fields_arr); $i++) { debug_echo($cmd, "fields for data source {$i} :"); debug_dump_list($fields_arr[$i - 1], true); } # parse the line rules $parsed_mawk = mawk_parse_code($mawk, $fields_arr, $cmd); if (is_int($parsed_mawk)) { return error($cmd, "code line {$i} of the mawk has an invalid syntax"); } $parsed_mawk_code = $parsed_mawk['code']; # set up indexes $indexes_arr = $parsed_mawk['indexes']; if (count($fields_arr) > 1) { $r = mawk_add_primary_index($primary_key, $indexes_arr, $fields_arr, $cmd); if ($r === false) { return false; } } # display info about the line rules debug_echo($cmd, "the following mawk code will be used :"); debug_echo_txt(mawk_clean_code($mawk), true); if ($debug) { debug_echo($cmd, "which translates to the following parsed code :"); debug_dump_yaml($parsed_mawk_code, true); if ($indexes_arr) { debug_echo($cmd, "and includes the following indexes :"); debug_dump_yaml($indexes_arr, true); } } /* if (count ($fields_arr) > 1) { $primary_indexes = mawk_get_indexes ($primary_key, $fields_arr, $cmd); if ($primary_indexes === false) return false; } else { $primary_indexes = array (); } # set up create indexes $new_indexes = mawk_get_indexes ($create_indexes, $fields_arr, $cmd); if ($new_indexes === false) return false; */ # set up discard if ($join_type == 'inner') { $inner_join = true; } else { $inner_join = false; } # run mawk code try { debug_echo($cmd, "processing the data (this can take some time depending on the inputs) ..."); $mawk_res = mawk_process_data($parsed_mawk_code, $fields_arr, $data_arr, $indexes_arr, $update_frequency, $inner_join, $cmd, $debug); } catch (MawkError $e) { $msg = $e->getMessage(); return error($cmd, $msg); } return $mawk_res; }
function load_data_from_doc_file($opts, $pipe, $cmd = __FUNCTION__) { # set prefix $prefix = $cmd; # merge opts $opts = merge_opts($opts, $pipe, 'doc_type'); # get file opt $file = get_opt($prefix, $opts, 'file'); if (!check_opt_set_type($cmd, $file, 'file', 'string')) { return false; } # get defined fields opt $defined_fields = get_opt($prefix, $opts, 'defined_fields'); if (!check_opt_if_set_type($cmd, $defined_fields, 'defined_fields', 'array_of_strings')) { return false; } # get save fields $save_fields = get_opt($prefix, $opts, 'save_fields'); if (!check_opt_if_set_type($cmd, $save_fields, 'save_fields', 'array_of_strings')) { return false; } # get doc type opt $doc_type = get_opt($prefix, $opts, 'doc_type', 'json'); if (!check_opt_set_type($cmd, $doc_type, 'doc_type', 'document_file_type')) { return false; } $doc_type_upper = strtoupper($doc_type); # get data structure opt $data_structure = get_opt($prefix, $opts, 'data_structure', 'list_of_objects'); if (!check_opt_set_type($cmd, $data_structure, 'data_structure', 'data_structure_type')) { return false; } # get limit opt $limit = get_opt($prefix, $opts, 'limit', 0); if (!check_opt_set_type($cmd, $limit, 'limit', 'positive_integer')) { return false; } # get offset opt $offset = get_opt($prefix, $opts, 'offset', 0); if (!check_opt_set_type($cmd, $offset, 'offset', 'positive_integer')) { return false; } # check the file exists if (!file_exists($file)) { return error($cmd, "file does not exist : {$file}"); } # display generic info about the data $data_structure_str = str_replace('_', ' ', $data_structure); debug_echo($cmd, "creating data from {$doc_type_upper} file : {$file}"); debug_echo($cmd, "source data of type '{$data_structure_str}'"); # read the file into memory $data_str = @file_get_contents($file); if (!is_string($data_str)) { return error($cmd, "could not read file : {$file}"); } # load data depending on type switch ($doc_type) { case 'json': $data = @json_decode($data_str, true); break; case 'yaml': $data = @yaml_decode($data_str); break; } if (!$data) { return error($cmd, "invalid {$doc_type_upper} in file : {$file}"); } $data_count = count($data); # set up end point if ($limit == 0) { $end = $data_count; } else { $end = $offset + $limit; } # set up the fields and build data $res_data = array(); switch ($data_structure) { case 'list_of_columns': # set up fields if ($defined_fields) { $defined_fields_source = '(from the config)'; } else { $defined_fields = array_shift($data); $defined_fields_source = '(from the source file)'; } # set up response field indexes if ($save_fields) { $res_fields = $save_fields; $res_field_count = count($res_fields); $res_field_indexes = build_field_indexes($defined_fields, $res_fields, $cmd); if ($res_field_indexes === false) { return; } } else { $res_fields = $defined_fields; } # display info about the fields debug_echo($cmd, "defined fields {$defined_fields_source} : "); debug_dump_list($defined_fields, true); if ($save_fields) { debug_echo($cmd, "saved fields :"); debug_dump_list($save_fields, true); } # gather the data # TODO: change for having offset / limit $res_data = array(); if ($save_fields) { $res_data = array(); for ($i = $offset; $i < $end; $i++) { $line = $data[$i]; $new_line = array(); for ($j = 0; $j < $res_field_count; $j++) { $new_line[] = $line[$res_field_indexes[$j]]; } $res_data[] = $new_line; } } elseif ($offset == 0 && $limit == 0) { $res_data = $data; } else { $res_data = array(); for ($i = $offset; $i < $end; $i++) { $res_data[] = $data[$i]; } } break; case 'list_of_objects': $res_data = array(); if ($save_fields) { # set up result fields $res_fields = $save_fields; $res_fields_count = count($res_fields); # display info about what fields will be saved debug_echo($cmd, "saved fields :"); debug_dump_list($res_fields, true); # build the data for ($i = $offset; $i < $end; $i++) { $obj = $data[$i]; $new_line = array(); for ($j = 0; $j < $res_fields_count; $j++) { $new_line[] = @$obj[$res_fields[$j]]; } $res_data[] = $new_line; } $res_fields = $save_fields; } else { # display info about saved files debug_echo($cmd, "saving all fields - they will be listed as they are added"); # build the data $res_fields = array(); $res_fields_count = 0; for ($i = $offset; $i < $end; $i++) { $obj = $data[$i]; $new_line = array(); # save all the existing data in order for ($j = 0; $j < $res_fields_count; $j++) { $new_line[] = @$obj[$res_fields[$j]]; } # index and save any new fields $res_fields_added = array(); foreach ($obj as $key => $value) { if (in_array($key, $res_fields)) { continue; } $res_fields_added[] = $key; $res_fields[] = $key; $res_fields_count++; $new_line[] = $value; } $res_data[$i] = $new_line; # display list of fields added (if any) if ($res_fields_added) { $line_no = $i + 1; debug_echo($cmd, "the following fields were added on row {$line_no} :"); debug_dump_list($res_fields_added, true); } } # add back in any empty fields for ($i = 0; $i < count($res_data); $i++) { $line =& $res_data[$i]; $line_fields_count = count($line); if ($line_fields_count == $res_fields_count) { break; } for ($j = $line_fields_count; $j < $res_fields_count; $j++) { $line[] = ''; } } } break; } # detail results $line_count = count($res_data); debug_echo($cmd, "creation of data from JSON file complete ({$line_count} lines processed)"); # create result return build_result_data($res_fields, $res_data); }
function search_replace_data_by_key_value($opts, $pipe, $cmd = __FUNCTION__) { # set prefix $prefix = 'search_replace_data'; # merge opts $opts = merge_opts($opts, $pipe); # get fields opt $data = get_opt_config_value($prefix, $opts, 'data'); if (!check_opt_set_type($cmd, $data, 'data', 'array')) { return false; } # get fields opt $fields = get_opt_config_value($prefix, $opts, 'fields'); if (!check_opt_set_type($cmd, $fields, 'fields', 'array_of_strings')) { return false; } # get map opt $search = get_opt_config_value($prefix, $opts, 'search'); if (!check_opt_set_type($cmd, $search, 'search', 'array,yaml_array')) { return false; } if (is_string($search)) { $search = yaml_decode($search); } # get replace opt $replace = get_opt_config_value($prefix, $opts, 'replace'); if (!check_opt_set_type($cmd, $replace, 'replace', 'array,yaml_array')) { return false; } if (is_string($replace)) { $replace = yaml_decode($replace); } # set all the fields $search_fields = array_keys($search); $value_fields = array_values($search); $replace_fields = array_values($replace); # set up new fields $new_fields = array(); foreach ($fields as $field) { if (in_array($field, $search_fields) || in_array($field, $value_fields)) { continue; } $new_fields[] = $field; } foreach ($replace_fields as $field) { $new_fields[] = $field; } $new_fields_count = count($new_fields); # set up copy fields $new_fields_flipped = array_flip($new_fields); $copy_fields = array(); foreach ($fields as $old_index => $field) { if (!in_array($field, $new_fields) || in_array($field, $replace_fields)) { continue; } $new_index = $new_fields_flipped[$field]; $copy_fields[$old_index] = $new_index; } $copy_fields_count = count($copy_fields); # set up search field indexes $fields_flipped = array_flip($fields); $searches = array(); foreach ($search as $check_field => $value_field) { $check_field_idx = $fields_flipped[$check_field]; $value_field_idx = $fields_flipped[$value_field]; $searches[$check_field_idx] = $value_field_idx; } # set up replaces $replaces = array(); foreach ($replace as $value => $field) { $replaces[$value] = $new_fields_flipped[$field]; } # display messages debug_echo($cmd, "doing a search-replace by key value using the following pairs (check_field: value_field):"); debug_dump_yaml($search, true); debug_echo($cmd, "the following found replacements were made (checked_value: replacement_field):"); debug_dump_yaml($replace, true); # loop through data $new_data = array(); foreach ($data as $line) { # create new line $new_line = array(); for ($i = 0; $i < $new_fields_count; $i++) { $new_line[] = null; } # copy data foreach ($copy_fields as $old_index => $new_index) { $new_line[$new_index] = $line[$old_index]; } # search for values foreach ($searches as $check_field_index => $value_field_index) { $check = $line[$check_field_index]; $replace_field_index = @$replaces[$check]; if ($replace_field_index === null) { continue; } $new_line[$replace_field_index] = $line[$value_field_index]; } # add the new line to the data $new_data[] = $new_line; } # build res $res = array('fields' => $new_fields, 'data' => $new_data); return $res; }
function http_use_checked_saved_file($opts, $pipe, $cmd = __FUNCTION__) { # set prefix $prefix = 'http'; # merge opts $opts = merge_opts($opts, $pipe, 'use_saved_file'); # get use downloaded file opt $use_saved_file = get_opt_config_value($prefix, $opts, 'use_saved_file', true); if (!check_opt_set_type($cmd, $use_saved_file, 'use_saved_file', 'boolean')) { return false; } # reset use download file opt $old_use_saved_file = get_config_value('http_old_use_saved_file'); set_config_value('http_use_saved_file', $old_use_saved_file); # check to see if should use downloaded file or not if (!$use_saved_file) { return $pipe; } # restore http request execution config $http_execute = get_config_value('http_old_execute'); set_config_value('http_execute', $http_execute); # get checked downloaded file opt $file = get_config_value('http_checked_saved_file'); if (!check_opt_if_set_type($cmd, $file, 'http_checked_saved_file', 'string')) { return false; } # if file does not exist just pipe if (!$file) { return $pipe; } # reset config and return set_config_value('http_checked_saved_file', null); return array('file' => $file, 'response_file' => $file); }
function curl_use_checked_saved_file($opts, $pipe, $cmd = __FUNCTION__, $prefix = 'curl') { # merge opts $opts = merge_opts($opts, $pipe, "use_saved_file"); # get use downloaded file opt $use_saved_file = get_opt_config_value($prefix, $opts, "use_saved_file", true); if (!check_opt_set_type($cmd, $use_saved_file, "use_saved_file", "boolean")) { return false; } # reset use download file opt $old_use_saved_file = get_config_value("{$prefix}_old_use_saved_file"); set_config_value("{$prefix}_use_saved_file", $old_use_saved_file); # check to see if should use downloaded file or not if (!$use_saved_file) { return $pipe; } # restore http request execution config $execute = get_config_value("{$prefix}_old_execute"); set_config_value("{$prefix}_execute", $execute); # get checked downloaded file opt $file = get_config_value("{$prefix}_checked_saved_file"); if (!check_opt_if_set_type($cmd, $file, "{$prefix}_checked_saved_file", "string")) { return false; } # if file does not exist just pipe if (!$file) { return $pipe; } # reset config and return set_config_value("{$prefix}_checked_saved_file", null); return array("file" => $file, "response_file" => $file); }
function http_parse_form($prefix, &$opts, &$res, $form_name, $cmd, $debug, $response_form = false) { # TODO: put debug info around the debug_echo's # get request_url opt $request_url = get_opt($prefix, $opts, 'request_url'); if (!check_opt_set_type($cmd, $request_url, 'request_url', 'string')) { return false; } # get response body opt $response_body = get_opt($prefix, $opts, 'response_body'); if (!check_opt_if_set_type($cmd, $response_body, 'response_body', 'string')) { return false; } # return if no response body is found if (!$response_body) { warning("no response body to parse for hidden post args"); return $opts; } # get post form opt $form_name = get_opt($prefix, $opts, 'form_name', ''); if (!check_opt_set_type($cmd, $form_name, 'form_name', 'string')) { return false; } # display message if ($form_name) { debug_echo($cmd, "parsing response form '{$form_name}' for action, method and inputs"); } else { debug_echo($cmd, "parsing all forms (if any) for action, method and inputs"); } # parse the response body for forms $form_parse_logic = "\n\n elements:\n - form:\n attributes:\n name: {$form_name}\n action:\n method:\n \n elements:\n - input:\n label: inputs\n attributes:\n !type: hidden\n name:\n value:\n\n - button:\n label: inputs\n attributes:\n name:\n value:\n "; $form_parse = array('document' => $response_body, 'doc_type' => 'HTML', 'result_type' => 'forms', 'logic' => $form_parse_logic, 'merge_results' => true); $form = xml_parse_document($form_parse, null, $cmd); if (!$form) { if ($form_name) { warning($cmd, "form '{$form_name}' not found in response body"); } else { debug_echo($cmd, "no forms found in the response body"); } return $opts; } # set the variables $form_action = $form['action']; $form_method = $form['method']; $form_name = $form['name']; $form_inputs = $form['inputs']; # set the form method if ($response_form) { $res['response_form_method'] = $form_method; } else { $opts['form_method'] = $form_method; $res['form_method'] = $form_method; } # build the full form action if ($form_action) { # check for a full URL if (substr($form_action, 0, 7) != 'http://' && substr($form_action, 0, 8) != 'https://') { # split the url into parts $r = preg_match('/^(https?:\\/\\/[^\\/\\?]+)((\\/[^\\/\\?]*)*)?(\\?(.*))?$/', $request_url, $matches); $host = $matches[1]; $path = $matches[2]; $last_dir = $matches[3]; $get_args = @$matches[4]; # build the form action based on the found parts and how it's defined if ($form_action[0] == '/') { $form_action = "{$host}{$form_action}"; } else { $dir_path = substr($path, 0, -strlen($last_dir)); $form_action = "{$host}{$dir_path}/{$form_action}"; } } # save the form action debug_echo($cmd, "form action url found : {$form_action}"); if ($response_form) { $res['response_form_action'] = $form_action; } else { $opts['form_action'] = $form_action; $res['form_action'] = $form_action; } } # build string for displaying info about hidden post args if ($form_name) { $form_name_str = "in form '{$form_name}'"; } else { $form_name_str = "in all forms"; } # add the hidden arguments to the hidden inputs and piped post args if (count($form_inputs) == 0) { debug_echo($cmd, "no hidden form inputs were found {$form_name_str}"); return true; } # build the hidden inputs $inputs = array(); $form_inputs_count = count($form_inputs); for ($i = 0; $i < $form_inputs_count; $i++) { $form_input = $form_inputs[$i]; $name = $form_input['name']; $value = @$form_input['value']; $value = $value ? $value : ''; $inputs[$name] = $value; } $form_inputs = $inputs; # display info about hidden args found debug_echo($cmd, "the following hidden form inputs were found {$form_name_str} :"); debug_dump_yaml($form_inputs, true); # save the hidden inputs if ($response_form) { $res['response_form_inputs'] = $form_inputs; } else { $opts['form_inputs'] = $form_inputs; $res['form_inputs'] = $form_inputs; } # check for args $args_key = "{$form_method}_args"; # get appropriate args opt $args = get_opt($cmd, $opts, $args_key); if (!check_opt_if_set_type($cmd, $args, $args_key, 'array')) { return false; } # merge the arts if necessary if ($args) { # check to see if is a string or an array if (is_string($args)) { if (strlen($args) == 0) { $args = $form_inputs; } else { $args .= '&' . http_build_query($form_inputs); } } elseif (is_array($args)) { $args = array_merge($form_inputs, $args); } else { return error($cmd, "option '{$form_method}_args' is not a string or array"); } } else { $args = $form_inputs; } # save the args if ($response_form) { $opts["response_{$args_key}"] = $args; $res[$args_key] = $args; } else { $opts[$args_key] = $args; } # sort response and return ksort($opts); ksort($res); return true; }
function imap_search_for_email($opts, $pipe, $cmd = __FUNCTION__) { # set prefix $prefix = 'imap_search_for_email'; # unset opts that might affect the request unset($pipe['dir']); unset($pipe['file']); unset($pipe['path']); # merge opts $merged_opts = merge_opts($opts, $pipe); # get tests opt $tests = get_opt($prefix, $merged_opts, 'tests'); if (!check_opt_set_type($cmd, $tests, 'tests', 'array')) { return false; } # perform serach of mailbox $r = imap_get_mailbox_email_count($opts, $pipe, $cmd); if ($r === false) { return false; } # get the count $email_count = $r['email_count']; # split the tests $header_tests = @$tests['headers']; $body_tests = @$tests['body']; # loop over the emails to get the headers $found = false; for ($i = $email_count; $i >= 1; $i--) { # set the email id $email_id = $i; $part_id = false; $email_headers_res = null; $email_body_res = null; $opts['email_id'] = $i; # get the eamil headers if required if ($header_tests) { $email_headers_res = imap_get_email_headers($opts, $pipe, $cmd); if ($email_headers_res === false) { return false; } # get the returned headers and set as lower-case $email_headers = $email_headers_res['email_headers']; foreach ($email_headers as $name => $value) { $email_headers[strtolower($name)] = $value; } # test the headers $found_headers = imap_test_values($header_tests, $email_headers); if (!$found_headers) { continue; } } if ($body_tests) { $email_body_res = imap_get_email_body($opts, $pipe, $cmd); if ($email_body_res === false) { return false; } # get the returned headers and set as lower-case $email_body_parts = $email_body_res['email_body_parts']; # run through the parts foreach ($email_body_parts as $part_idx => $part) { $found_part = imap_test_values($body_tests, $part); if ($found_part) { $part_id = $part_idx + 1; break; } } if (!$found_part) { continue; } } $found = true; break; } # check if found or not if (!$found) { return error($cmd, "email not found with the search criteria"); } # display success message $msg = "email found with id {$email_id}"; if ($part_id) { $msg .= " part {$part_id}"; } debug_echo($cmd, $msg); # create res $res = array('email_id' => $email_id); if ($part_id) { $res['email_part_id'] = $part_id; foreach ($part as $name => $value) { $res["email_{$name}"] = $value; } } # merge res with headers and body $res = merge_opts_for_output($opts, $res); if ($email_headers_res) { $res = merge_opts_for_output($res, $email_headers_res); } if ($email_body_res) { $res = merge_opts_for_output($res, $email_body_res); } ksort($res); return $res; }
function backup_file($opts, $pipe = null, $cmd = __FUNCTION__) { # set prefix $prefix = 'file_backup'; # merge opts $opts = merge_opts($opts, $pipe, 'file'); # get backup file opt $backup_file = get_opt($prefix, $opts, 'backup_file', true); if (!check_opt_set_type($cmd, $backup_file, 'backup_file', 'boolean')) { return false; } if ($backup_file === false) { return true; } # get file opt $file = get_opt($prefix, $opts, 'file'); if (!check_opt_set_type($cmd, $file, 'file', 'string')) { return false; } # get suffix opt $suffix = get_opt_config_value($prefix, $opts, 'suffix', '.bak.'); if (!check_opt_set_type($cmd, $suffix, 'suffix', 'string')) { return false; } # create new path $index = 1; do { $new_file = "{$file}{$suffix}{$index}"; if (!is_file($new_file)) { break; } $index++; } while (true); # move file debug_echo($cmd, "backing up file : {$file} => {$new_file}"); if (!rename($file, $new_file)) { return error($cmd, "could not rename file : {$file} => {$new_file}"); } return array('file' => $new_file); }
function result_switch_keys($opts, $pipe, $cmd = __FUNCTION__) { # set prefix $prefix = 'result_switch_keys'; # merge_opts $opts = merge_opts($opts, $pipe); # get key opt $keys = get_opt($prefix, $opts, 'keys'); if (!check_opt_set_type($cmd, $keys, 'keys', 'array')) { return false; } # check validity of pipe if (!is_array($pipe)) { return error($cmd, "pipe is not an array"); } # store values $vals = array(); foreach ($keys as $old_key => $new_key) { $vals[$new_key] = $pipe[$old_key]; } # unset old keys foreach ($keys as $old_key => $new_key) { unset($pipe[$old_key]); } # set the new keys foreach ($vals as $new_key => $val) { $pipe[$new_key] = $val; } return $pipe; }
function merge_relational_data($opts, $pipe, $cmd = __FUNCTION__) { # set prefix $prefix = 'merge_relational_data'; # merge opts $opts = merge_opts($opts, $pipe); # get data opt $data = get_opt_config_value($prefix, $opts, 'data'); if (!check_opt_set_type($cmd, $data, 'data', 'array')) { return false; } # get fields opt $fields = get_opt_config_value($prefix, $opts, 'fields'); if (!check_opt_set_type($cmd, $fields, 'fields', 'array_of_strings')) { return false; } # get primary key opt $primary_key = get_opt_config_value($prefix, $opts, 'primary_key'); if (!check_opt_set_type($cmd, $primary_key, 'primary_key', 'string')) { return false; } # get check field opt $merge_rules = get_opt_config_value($prefix, $opts, 'merge_rules'); if (!check_opt_set_type($cmd, $merge_rules, 'merge_rules', 'array,yaml_array')) { return false; } if (is_string($merge_rules)) { $merge_rules = yaml_decode($merge_rules); } # set up primary key if (!in_array($primary_key, $fields)) { return error($cmd, "field '{$primary_key}' does not exist"); } $fields_flipped = array_flip($fields); $primary_key_index = $fields_flipped[$primary_key]; # set up new fields $new_fields = array(); foreach ($fields as $field) { $new_fields[] = $field; } # loop through all the rules $rules = array(); foreach ($merge_rules as $i => $merge_rule) { $rule = array(); # get check field opt $check_field = get_opt_config_value($prefix, $merge_rule, 'check_field'); if (!check_opt_set_type($cmd, $check_field, 'check_field', 'string')) { return false; } # check that the fields exist if (!in_array($check_field, $fields)) { return error($cmd, "field '{$check_field}' does not exist"); } $check_field_index = $fields_flipped[$check_field]; $rule[0] = $check_field_index; # get value field opt $value_field = get_opt_config_value($prefix, $merge_rule, 'value_field'); if (!check_opt_if_set_type($cmd, $value_field, 'value_field', 'string')) { return false; } # get value field map opt $value_field_map = get_opt_config_value($prefix, $merge_rule, 'value_field_map'); if (!check_opt_if_set_type($cmd, $value_field_map, 'value_field_map', 'array,yaml_array')) { return false; } if ($value_field_map && !is_array($value_field_map)) { $value_field_map = yaml_decode($value_field_map); } if ($value_field === null && $value_field_map === null) { return opt_not_set_msg($cmd, 'value_field,value_field_map'); } # check the type of the rule if ($value_field !== null) { $rule[1] = 'f'; if (!in_array($value_field, $fields)) { return error($cmd, "field '{$value_field}' does not exist"); } $value_field_index = $fields_flipped[$value_field]; $rule[2] = $value_field_index; } else { $rule[1] = 'fm'; # set up value field map foreach ($value_field_map as $value => $field) { if (!in_array($field, $new_fields)) { $new_fields[] = $field; } } # create the value indexes $new_fields_flipped = array_flip($new_fields); $values_indexes = array(); foreach ($value_field_map as $value => $field) { $values_indexes[$value] = $new_fields_flipped[$field]; } $rule[2] = $values_indexes; } # get value opt # TODO: value_map if (array_key_exists('value', $merge_rule)) { $rule[3] = 'v'; $rule[4] = $merge_rule['value']; } elseif (array_key_exists('value_map', $merge_rule)) { $value_map = get_opt_config_value($prefix, $merge_rule, 'value_map'); if (!check_opt_set_type($cmd, $value_map, 'value_map', 'array,yaml_array')) { return false; } if (is_string($value_map)) { $value_map = yaml_decode($value_map); } $rule[3] = 'vm'; $rule[4] = $value_map; } else { $rule[3] = 'cv'; } # add rule to rules $rules[] = $rule; } # set up counts $data_count = count($data); $field_count = count($fields); $new_field_count = count($new_fields); # set up new data $new_data = array(); $new_data_indexes = array(); # loop over data and apply rules debug_echo($cmd, "merging relational data (this can take some time) ..."); for ($i = 0; $i < $data_count; $i++) { # get line and check if line already exists $line = $data[$i]; $primary_value = $line[$primary_key_index]; unset($new_line); $new_line =& $new_data_indexes[$primary_value]; # create new line if (!$new_line) { unset($new_line); $new_line = array(); for ($j = 0; $j < $field_count; $j++) { $new_line[$j] = $line[$j]; } for ($j = $field_count; $j < $new_field_count; $j++) { $new_line[] = null; } $new_data[] =& $new_line; $new_data_indexes[$primary_value] =& $new_line; } # loop through the rules to modify the data foreach ($rules as $rule) { # set up rule parts $check_field_index = $rule[0]; $value_field_type = $rule[1]; $value_field_index = $rule[2]; $value_type = $rule[3]; # intiiate check $check_value = $line[$check_field_index]; if ($check_value === null) { continue; } # get value field switch ($value_field_type) { case 'fm': $value_field_index = @$value_field_index[$check_value]; break; } if ($value_field_index === null) { continue; } # get value switch ($value_type) { case 'v': $value = $rule[4]; break; case 'vm': $value = $rule[4][$check_value]; break; case 'cv': $value = $check_value; break; } # set the value $new_line[$value_field_index] = $value; } } # build result $res = array('data' => $new_data, 'fields' => $new_fields); return $res; }
function xml_parse_document($opts, $pipe, $cmd = __FUNCTION__) { # set prefix $prefix = 'xml_parse_document'; # merge opts $opts = merge_opts($opts, $pipe, 'document'); # get document opt $document = get_opt($prefix, $opts, 'document'); if (!check_opt_set_type($cmd, $document, 'document', 'string')) { return false; } # get doc type opt $doc_type = get_opt_config_value($prefix, $opts, 'doc_type', 'xml'); if (!check_opt_set_type($cmd, $doc_type, 'doc_type', 'string')) { return false; } # get logic opt $logic = get_opt($prefix, $opts, 'logic'); if (!check_opt_set_type($cmd, $logic, 'logic', 'string,array')) { return false; } if (is_string($logic)) { $logic = @yaml_decode($logic); if (!$logic) { return error($cmd, "parse logic not valid YAML"); } } # get result type opt $result_type = get_opt_config_value($prefix, $opts, 'result_type', 'elements'); if (!check_opt_set_type($cmd, $result_type, 'result_type', 'string')) { return false; } # get merge results opt $merge_results = get_opt($prefix, $opts, 'merge_results', false); if (!check_opt_set_type($cmd, $merge_results, 'merge_results', 'boolean')) { return false; } # load the document $dom = new DOMDocument('1.0'); @$dom->loadHTML($document); #$dom->preserveWhiteSpace = false; # check that the document was parsed correctly $res = xml_parse_element($dom, $logic); if ($res === false) { return error($cmd, "could not parse {$type} document"); } # display info about the parsing debug_echo($cmd, "{$doc_type} document parsed successfully for {$result_type}"); if (empty($res)) { debug_echo($cmd, "no {$result_type} found while parsing the {$doc_type} document"); } else { debug_echo($cmd, "the following {$result_type} were found in the {$doc_type} document"); debug_dump_yaml($res, true); } # return if we are not merging the results if (!$merge_results) { return $res; } # merge the results $new_res = array(); $res_count = count($res); for ($i = 0; $i < $res_count; $i++) { $elt = $res[$i]; foreach ($elt as $key => $value) { if (is_array($value)) { $cur_value = @$new_res[$key]; if ($cur_value) { $value = array_merge($cur_value, $value); } } $new_res[$key] = $value; } } return $new_res; }
function modify_fields($opts, $pipe, $cmd = __FUNCTION__, $opt_prefix = false) { # set prefix $prefix = 'modify_fields'; # merge opts $opts = merge_opts($opts, $pipe); # adjust opt prefix $cmd = adjust_opt_prefix($cmd, $opts, $opt_prefix, 'add_fields'); # get data opt $data =& get_opt_config_value($prefix, $opts, 'data'); if (!check_opt_set_type($cmd, $data, 'data', 'array')) { return false; } # get fields opt $fields = get_opt_config_value($prefix, $opts, 'fields'); if (!check_opt_set_type($cmd, $fields, 'fields', 'array_of_strings')) { return false; } # get indexes fields opt $indexes =& get_opt_config_value($prefix, $opts, 'indexes'); if (!check_opt_if_set_type($cmd, $indexes, 'indexes', 'array')) { return false; } # get map fields opt $map_fields = get_opt_config_value($prefix, $opts, 'map'); if (!check_opt_if_set_type($cmd, $map_fields, 'map', 'string,array')) { return false; } if (is_string($map_fields)) { $map_fields = yaml_decode($map_fields); } # get swap fields opt $swap_fields = get_opt_config_value($prefix, $opts, 'swap'); if (!check_opt_if_set_type($cmd, $swap_fields, 'swap', 'string,array')) { return false; } if (is_string($swap_fields)) { $swap_fields = yaml_decode($swap_fields); } # get add fields opt $add_fields = get_opt_config_value($prefix, $opts, 'add'); if (!check_opt_if_set_type($cmd, $add_fields, 'add', 'array')) { return false; } # get delete fields opt $delete_fields = get_opt_config_value($prefix, $opts, 'delete'); if (!check_opt_if_set_type($cmd, $delete_fields, 'delete', 'array_of_strings')) { return false; } # get reorder fields opt $reorder_fields = get_opt_config_value($prefix, $opts, 'reorder'); if (!check_opt_if_set_type($cmd, $reorder_fields, 'reorder', 'array_of_strings')) { return false; } # perform the operations if ($map_fields && !map_data_fields($data, $fields, $map_fields, $cmd)) { return false; } if ($swap_fields && !swap_data_fields($fields, $swap_fields, $cmd)) { return false; } if ($add_fields && !add_data_fields($data, $fields, $add_fields, $cmd)) { return false; } if ($delete_fields && !delete_data_fields($data, $fields, $delete_fields, $cmd)) { return false; } if ($reorder_fields && !reorder_data_fields($data, $fields, $reorder_fields, $cmd)) { return false; } # create indexes /* var_dump ($fields); var_dump ($data[0]); exit; */ # set the response return build_result_data($fields, $data, $indexes); }