function array_strip_offset($opts, $pipe, $cmd = __FUNCTION__)
{
# merge opts
$opts = merge_opts($opts, $pipe, 'offset');
# get the list and offset
if (is_array($opts)) {
if (array_key_exists('offset', $opts)) {
$offset = $opts['offset'];
} else {
return error($cmd, "no offset defined");
}
if (array_key_exists('list', $opts)) {
$list = $opts;
} else {
$list = $pipe;
}
} else {
$list = $pipe;
$offset = (int) $opts;
if ($offset <= 0) {
return $list;
}
}
# check validity of list and offset
if (!check_opt_set_type($cmd, $list, 'list', 'array_of_strings')) {
return false;
}
if (!check_opt_set_type($cmd, $offset, 'offset', 'integer')) {
return false;
}
# strip offset recursively
$list = array_strip_offset_util($list, $offset);
return $list;
}
function result_switch_keys($opts, $pipe, $cmd = __FUNCTION__)
{
# set prefix
$prefix = 'result_switch_keys';
# merge_opts
$opts = merge_opts($opts, $pipe);
# get key opt
$keys = get_opt($prefix, $opts, 'keys');
if (!check_opt_set_type($cmd, $keys, 'keys', 'array')) {
return false;
}
# check validity of pipe
if (!is_array($pipe)) {
return error($cmd, "pipe is not an array");
}
# store values
$vals = array();
foreach ($keys as $old_key => $new_key) {
$vals[$new_key] = $pipe[$old_key];
}
# unset old keys
foreach ($keys as $old_key => $new_key) {
unset($pipe[$old_key]);
}
# set the new keys
foreach ($vals as $new_key => $val) {
$pipe[$new_key] = $val;
}
return $pipe;
}
function convert_data_to_objects($opts, $pipe, $cmd = __FUNCTION__)
{
# set prefix
$prefix = 'convert_data_to_objects';
# merge opts
$opts = merge_opts($opts, $pipe);
# get fields opt
$fields = get_opt($prefix, $opts, 'fields');
if (!check_opt_set_type($cmd, $fields, 'fields', 'array_of_strings')) {
return false;
}
# get data opt
$data = get_opt($prefix, $opts, 'data');
if (!check_opt_set_type($cmd, $data, 'data', 'array_of_arrays')) {
return false;
}
# initiate response
$field_count = count($fields);
$data_objects = array();
# loop over data
debug_echo($cmd, "converting data to objects");
foreach ($data as $line) {
$data_object = array();
for ($i = 0; $i < $field_count; $i++) {
$data_object[$fields[$i]] = $line[$i];
}
$data_objects[] = $data_object;
}
# build res
$res = array('data_objects' => $data_objects);
return $res;
}
function create_csv_file($opts, $pipe, $cmd = __FUNCTION__)
{
# set prefix
$prefix = 'create_csv';
# merge opts
$opts = merge_opts($opts, $pipe, 'file');
# get file opt
$file = get_opt($prefix, $opts, 'file');
if (!check_opt_set_type($cmd, $file, 'file', 'string')) {
return false;
}
# get fields opt
$fields = get_opt($prefix, $opts, 'fields');
if (!check_opt_set_type($cmd, $fields, 'fields', 'array_of_strings')) {
return false;
}
# get data opt
$data = get_opt($prefix, $opts, 'data');
if (!check_opt_set_type($cmd, $data, 'data', 'array')) {
return false;
}
# make parent dir
if (!make_dir($opts, dirname($file))) {
return false;
}
# create temp file
$file_handle = @fopen('php://temp/csv-out', 'w');
if (!$file_handle) {
return error($cmd, "cannot create temp CSV file");
}
# output the fields
if (!fputcsv($file_handle, $fields)) {
return error($cmd, "could not output the fields CSV file : {$file}");
}
# output the data
foreach ($data as $line_no => $line) {
if (!fputcsv($file_handle, $line)) {
return error($cmd, "could not output line {$line_no} to CSV file : {$file}");
}
}
# grab contents and close temp file
rewind($file_handle);
$text = stream_get_contents($file_handle);
fclose($file_handle);
# create file and
if (!file_put_contents($file, $text)) {
return error($cmd, "cannot store CSV data to file : {$file}");
}
# report
$lines = count($data);
debug_echo($cmd, "CSV file created ({$lines} lines) : {$file}");
# return
if ($pipe === false) {
return $file;
}
return array('file' => $file);
}
function build_indexes($opts, $pipe, $cmd = __FUNCTION__, $msg_suffix = null)
{
# set prefix
$prefix = 'build_indexes';
# merge opts
if ($pipe !== null) {
$opts = merge_opts($opts, $pipe, 'indexes');
}
# get fields opt
$fields = get_opt($prefix, $opts, 'fields');
if (!check_opt_set_type($cmd, $fields, 'fields', 'array_of_strings')) {
return false;
}
# get data opt
$data =& get_opt($prefix, $opts, 'data');
if (!check_opt_set_type($cmd, $data, 'data', 'array')) {
return false;
}
# get indexes opt
$indexes =& get_opt($prefix, $opts, 'indexes');
if (!check_opt_set_type($cmd, $indexes, 'indexes', 'array_of_strings')) {
return false;
}
# set up indexes
$res_indexes_assoc = array();
$res_indexes_list = array();
foreach ($indexes as $index_name) {
$index = array();
$res_indexes_assoc[$index_name] =& $index;
$res_indexes_list[] =& $index;
}
# set up field indexes
$field_indexes = build_field_indexes($fields, $indexes, $cmd, $msg_suffix);
if ($field_indexes === false) {
return false;
}
$index_count = count($indexes);
# loop over data to build indexes
for ($i = 0; $i < count($data); $i++) {
unset($line);
$line =& $data[$i];
for ($j = 0; $j < $index_count; $j++) {
$idx = $field_indexes[$j];
$key = $line[$idx];
$res_indexes_list[$j][$key] =& $line;
}
}
# sort the indexes
foreach ($res_indexes_assoc as $name => $index) {
ksort($index);
}
# save the indxes
$opts['indexes'] = $res_indexes_assoc;
return $opts;
}
function clean_data($opts, $pipe, $cmd = __FUNCTION__)
{
# set prefix
$prefix = 'clean_data';
# merge opts
$opts = merge_opts($opts, $pipe);
# get data opt
$data = get_opt_config_value($prefix, $opts, 'data');
if (!check_opt_set_type($cmd, $data, 'data', 'array')) {
return false;
}
# get fields opt
$fields = get_opt_config_value($prefix, $opts, 'fields');
if (!check_opt_set_type($cmd, $fields, 'fields', 'array_of_strings')) {
return false;
}
# get old value opt
$old_value = get_opt_config_value($prefix, $opts, 'old_value');
# get new value opt
$new_value = get_opt_config_value($prefix, $opts, 'new_value');
# set up counts
$data_count = count($data);
$field_count = count($fields);
# messages
debug_echo($cmd, "cleaning data (old value : {$old_value}, new value : {$new_value})");
# loop through data
for ($i = 0; $i < $data_count; $i++) {
unset($line);
$line =& $data[$i];
for ($j = 0; $j < $field_count; $j++) {
if ($line[$j] === $old_value) {
$line[$j] = $new_value;
}
}
}
# set up result
$res = array('data' => $data, 'fields' => $fields);
return $res;
}
function modify_rows($opts, $pipe, $cmd = __FUNCTION__)
{
# set prefix
$prefix = 'modify_rows';
# merge opts
$opts = merge_opts($opts, $pipe);
# get data opt
$data = get_opt_config_value($prefix, $opts, 'data');
if (!check_opt_set_type($cmd, $data, 'data', 'array')) {
return false;
}
# get fields opt
$res_fields = get_opt_config_value($prefix, $opts, 'fields');
if (!check_opt_set_type($cmd, $res_fields, 'fields', 'array_of_strings')) {
return false;
}
# get limit opt
$limit = get_opt_config_value($prefix, $opts, 'limit');
if (!check_opt_set_type($cmd, $limit, 'limit', 'integer')) {
return false;
}
# get limit opt
$offset = get_opt_config_value($prefix, $opts, 'offset', 0);
if (!check_opt_set_type($cmd, $offset, 'offset', 'integer')) {
return false;
}
# display message
debug_echo($cmd, "modifying rows (limit = {$limit}, offset = {$offset})");
# set up end
$end = $offset + $limit;
# process the data
$res_data = array();
for ($i = $offset; $i < $end; $i++) {
$res_data[] = $data[$i];
}
# build response object
return build_result_data($res_fields, $res_data);
}
function ftp_get_files($opts, $pipe, $cmd = __FUNCTION__)
{
# set prefix
$prefix = $cmd;
# merge opts
$opts = merge_opts($opts, $pipe, 'files');
# get files opt
$files = get_opt_config_value($prefix, $opts, 'files');
if (!check_opt_set_type($cmd, $files, 'files', 'array_of_strings')) {
return false;
}
# get local dir opt
$local_dir = get_opt_config_value($prefix, $opts, 'local_dir');
if (!check_opt_if_set_type($cmd, $local_dir, 'local_dir', 'string')) {
return false;
}
# loop over files
$res_files = array();
foreach ($files as $file) {
# get the save_to_file value
if ($local_dir) {
$file_name = basename($file);
$save_to_file = "{$local_dir}/{$file_name}";
} else {
$save_to_file = $file;
}
# process request
$req = array('file' => $file, 'save_to_file' => $save_to_file);
$r = ftp_get_file($req, $opts, $cmd);
if ($r === false) {
return false;
}
# add file to return list
$res_files[] = $save_to_file;
}
return array('files' => $res_files);
}
function split_file_on_cols($opts, $pipe)
{
# set prefix
$prefix = 'split_file_on_cols';
# merge opts
$opts = merge_opts($opts, $pipe, 'file');
# get file to split
$file = get_opt($prefix, $opts, 'file');
if (!$file) {
return error("No file specified to split");
}
# get split
$splits = get_opt($prefix, $opts, 'split');
if (!$splits) {
return error("No columns split specified to split file '{$file}'");
}
# check that the split is valid (i.e. an array of integers)
if (!is_array($splits)) {
return error("File split on columns does not have a split that is an array");
}
for ($i = 0; $i < count($splits); $i++) {
$val = $splits[$i];
if (!is_integer($val)) {
$i++;
$type = gettype($val);
return error("Value at position '{$i}' of split for file '{$file}' is not an integer (is of type '{$type}')");
}
}
# get trim (i.e. whether the values should have leading and trailing whitespace removed)
$trim = get_opt_config_value($prefix, $opts, 'trim', true);
# check file exists
if (!is_file($file)) {
return error("File '{$file}' does not exist");
}
# get lines
$lines = file($file);
# loop over lines to split the file
$data = array();
$total_splits = count($splits);
$max_split_index = $total_splits - 1;
foreach ($lines as $line) {
$line_data = array();
for ($i = 0; $i < $total_splits; $i++) {
$min = $splits[$i];
if ($i == 0) {
$min = 0;
$len = $splits[0];
} else {
$min = $splits[$i - 1];
$len = $splits[$i] - $min;
}
$val = substr($line, $min, $len);
if ($trim) {
$val = trim($val);
}
$line_data[] = $val;
}
$data[] = $line_data;
}
# return data depending on whether called internally or not
if ($pipe === false) {
return $data;
}
$res = array('data' => $data, 'lines' => count($lines));
return $res;
}
function process_data($opts, $pipe, $cmd = __FUNCTION__, $opt_prefix = false)
{
# set prefix
$prefix = 'process_data';
# merge opts
$opts = merge_opts($opts, $pipe);
# adjust opt prefix
$cmd = adjust_opt_prefix($cmd, $opts, $opt_prefix, 'edit');
# get debug opt
$debug = get_opt_config_value($prefix, $opts, 'debug', false);
if (!check_opt_set_type($cmd, $debug, 'debug', 'boolean')) {
return false;
}
# get type opt
$join_type = get_opt_config_value($prefix, $opts, 'join_type', 'outer');
if (!check_opt_set_type($cmd, $join_type, 'type', 'data_join_type')) {
return false;
}
# get sources opt
$sources = get_opt($prefix, $opts, 'sources');
if (!check_opt_if_set_type($cmd, $sources, 'sources', 'array')) {
return false;
}
if (!$sources) {
$source = build_source_data($prefix, $opts, $cmd);
if (!$source) {
return error($cmd, "sources opt is not set and the data and fields opts are not set either, so can't be used to create one");
}
$sources = array($source);
}
# get sources values
foreach ($sources as $i => $val) {
$sources[$i] = value($val);
}
# get create_indexes opt
$create_indexes = get_opt($prefix, $opts, 'create_indexes');
if (!check_opt_if_set_type($cmd, $create_indexes, 'create_indexes', 'array_of_strings')) {
return false;
}
# get join field opt
$primary_key = get_opt_config_value($prefix, $opts, 'primary_key', 'guid');
if (!check_opt_set_type($cmd, $primary_key, 'primary_key', 'string')) {
return false;
}
# get mawk rules opt
$mawk = get_opt_config_value($prefix, $opts, 'mawk');
if (!check_opt_set_type($cmd, $mawk, 'mawk', 'string')) {
return false;
}
# get update frequency opt
$update_frequency = get_opt_config_value($prefix, $opts, 'update_frequency', 0);
if (!check_opt_set_type($cmd, $update_frequency, 'update_frequency', 'integer')) {
return false;
}
# create fields and data arrays
$fields_arr = array();
$data_arr = array();
/*
for ($i=0; $i<count ($sources); $i++) {
$source_no = $i+1;
$fields = $sources[$i]['fields'];
if (!$fields)
return error ($cmd, "no fields defined for source $source_no");
$fields_arr[] = $fields;
if ($i == 0) {
$data_arr[] = $sources[$i]['data'];
} else {
$data = $sources[$i]['indexes'][$primary_key];
if ($data === null) {
return error ($cmd, "index on primary key '$primary_key' does not exis for source $source_no"); # TODO: auto-create
}
$data_arr[] = $data;
}
}*/
$sources_count = count($sources);
for ($i = 0; $i < $sources_count; $i++) {
$source_no = $i + 1;
# add fields
$fields = $sources[$i]['fields'];
if (!$fields) {
return error($cmd, "no fields defined for source {$source_no}");
}
if ($sources_count > 1) {
if (!in_array($primary_key, $fields)) {
return error($cmd, "primary key '{$primary_key}' not defined for source {$source_no}");
}
}
$fields_arr[] = $fields;
# add data
$data = $sources[$i]['data'];
if (!$data) {
return error($cmd, "no data defined for source {$source_no}");
}
$data_arr[] = $data;
}
# display info about the join, if there is one
if (count($fields_arr) > 1) {
debug_echo($cmd, "performing {$join_type} join on primary key / field '{$primary_key}'");
}
for ($i = 1; $i <= count($fields_arr); $i++) {
debug_echo($cmd, "fields for data source {$i} :");
debug_dump_list($fields_arr[$i - 1], true);
}
# parse the line rules
$parsed_mawk = mawk_parse_code($mawk, $fields_arr, $cmd);
if (is_int($parsed_mawk)) {
return error($cmd, "code line {$i} of the mawk has an invalid syntax");
}
$parsed_mawk_code = $parsed_mawk['code'];
# set up indexes
$indexes_arr = $parsed_mawk['indexes'];
if (count($fields_arr) > 1) {
$r = mawk_add_primary_index($primary_key, $indexes_arr, $fields_arr, $cmd);
if ($r === false) {
return false;
}
}
# display info about the line rules
debug_echo($cmd, "the following mawk code will be used :");
debug_echo_txt(mawk_clean_code($mawk), true);
if ($debug) {
debug_echo($cmd, "which translates to the following parsed code :");
debug_dump_yaml($parsed_mawk_code, true);
if ($indexes_arr) {
debug_echo($cmd, "and includes the following indexes :");
debug_dump_yaml($indexes_arr, true);
}
}
/*
if (count ($fields_arr) > 1) {
$primary_indexes = mawk_get_indexes ($primary_key, $fields_arr, $cmd);
if ($primary_indexes === false)
return false;
} else {
$primary_indexes = array ();
}
# set up create indexes
$new_indexes = mawk_get_indexes ($create_indexes, $fields_arr, $cmd);
if ($new_indexes === false)
return false;
*/
# set up discard
if ($join_type == 'inner') {
$inner_join = true;
} else {
$inner_join = false;
}
# run mawk code
try {
debug_echo($cmd, "processing the data (this can take some time depending on the inputs) ...");
$mawk_res = mawk_process_data($parsed_mawk_code, $fields_arr, $data_arr, $indexes_arr, $update_frequency, $inner_join, $cmd, $debug);
} catch (MawkError $e) {
$msg = $e->getMessage();
return error($cmd, $msg);
}
return $mawk_res;
}
function load_data_from_yaml_file($opts, $pipe, $cmd = __FUNCTION__)
{
$opts = merge_opts($opts, $pipe);
return load_data_from_doc_file($opts, 'yaml', $cmd);
}
function search_replace_data_by_key_value($opts, $pipe, $cmd = __FUNCTION__)
{
# set prefix
$prefix = 'search_replace_data';
# merge opts
$opts = merge_opts($opts, $pipe);
# get fields opt
$data = get_opt_config_value($prefix, $opts, 'data');
if (!check_opt_set_type($cmd, $data, 'data', 'array')) {
return false;
}
# get fields opt
$fields = get_opt_config_value($prefix, $opts, 'fields');
if (!check_opt_set_type($cmd, $fields, 'fields', 'array_of_strings')) {
return false;
}
# get map opt
$search = get_opt_config_value($prefix, $opts, 'search');
if (!check_opt_set_type($cmd, $search, 'search', 'array,yaml_array')) {
return false;
}
if (is_string($search)) {
$search = yaml_decode($search);
}
# get replace opt
$replace = get_opt_config_value($prefix, $opts, 'replace');
if (!check_opt_set_type($cmd, $replace, 'replace', 'array,yaml_array')) {
return false;
}
if (is_string($replace)) {
$replace = yaml_decode($replace);
}
# set all the fields
$search_fields = array_keys($search);
$value_fields = array_values($search);
$replace_fields = array_values($replace);
# set up new fields
$new_fields = array();
foreach ($fields as $field) {
if (in_array($field, $search_fields) || in_array($field, $value_fields)) {
continue;
}
$new_fields[] = $field;
}
foreach ($replace_fields as $field) {
$new_fields[] = $field;
}
$new_fields_count = count($new_fields);
# set up copy fields
$new_fields_flipped = array_flip($new_fields);
$copy_fields = array();
foreach ($fields as $old_index => $field) {
if (!in_array($field, $new_fields) || in_array($field, $replace_fields)) {
continue;
}
$new_index = $new_fields_flipped[$field];
$copy_fields[$old_index] = $new_index;
}
$copy_fields_count = count($copy_fields);
# set up search field indexes
$fields_flipped = array_flip($fields);
$searches = array();
foreach ($search as $check_field => $value_field) {
$check_field_idx = $fields_flipped[$check_field];
$value_field_idx = $fields_flipped[$value_field];
$searches[$check_field_idx] = $value_field_idx;
}
# set up replaces
$replaces = array();
foreach ($replace as $value => $field) {
$replaces[$value] = $new_fields_flipped[$field];
}
# display messages
debug_echo($cmd, "doing a search-replace by key value using the following pairs (check_field: value_field):");
debug_dump_yaml($search, true);
debug_echo($cmd, "the following found replacements were made (checked_value: replacement_field):");
debug_dump_yaml($replace, true);
# loop through data
$new_data = array();
foreach ($data as $line) {
# create new line
$new_line = array();
for ($i = 0; $i < $new_fields_count; $i++) {
$new_line[] = null;
}
# copy data
foreach ($copy_fields as $old_index => $new_index) {
$new_line[$new_index] = $line[$old_index];
}
# search for values
foreach ($searches as $check_field_index => $value_field_index) {
$check = $line[$check_field_index];
$replace_field_index = @$replaces[$check];
if ($replace_field_index === null) {
continue;
}
$new_line[$replace_field_index] = $line[$value_field_index];
}
# add the new line to the data
$new_data[] = $new_line;
}
# build res
$res = array('fields' => $new_fields, 'data' => $new_data);
return $res;
}
function curl_use_checked_saved_file($opts, $pipe, $cmd = __FUNCTION__, $prefix = 'curl')
{
# merge opts
$opts = merge_opts($opts, $pipe, "use_saved_file");
# get use downloaded file opt
$use_saved_file = get_opt_config_value($prefix, $opts, "use_saved_file", true);
if (!check_opt_set_type($cmd, $use_saved_file, "use_saved_file", "boolean")) {
return false;
}
# reset use download file opt
$old_use_saved_file = get_config_value("{$prefix}_old_use_saved_file");
set_config_value("{$prefix}_use_saved_file", $old_use_saved_file);
# check to see if should use downloaded file or not
if (!$use_saved_file) {
return $pipe;
}
# restore http request execution config
$execute = get_config_value("{$prefix}_old_execute");
set_config_value("{$prefix}_execute", $execute);
# get checked downloaded file opt
$file = get_config_value("{$prefix}_checked_saved_file");
if (!check_opt_if_set_type($cmd, $file, "{$prefix}_checked_saved_file", "string")) {
return false;
}
# if file does not exist just pipe
if (!$file) {
return $pipe;
}
# reset config and return
set_config_value("{$prefix}_checked_saved_file", null);
return array("file" => $file, "response_file" => $file);
}
function http_use_checked_saved_file($opts, $pipe, $cmd = __FUNCTION__)
{
# set prefix
$prefix = 'http';
# merge opts
$opts = merge_opts($opts, $pipe, 'use_saved_file');
# get use downloaded file opt
$use_saved_file = get_opt_config_value($prefix, $opts, 'use_saved_file', true);
if (!check_opt_set_type($cmd, $use_saved_file, 'use_saved_file', 'boolean')) {
return false;
}
# reset use download file opt
$old_use_saved_file = get_config_value('http_old_use_saved_file');
set_config_value('http_use_saved_file', $old_use_saved_file);
# check to see if should use downloaded file or not
if (!$use_saved_file) {
return $pipe;
}
# restore http request execution config
$http_execute = get_config_value('http_old_execute');
set_config_value('http_execute', $http_execute);
# get checked downloaded file opt
$file = get_config_value('http_checked_saved_file');
if (!check_opt_if_set_type($cmd, $file, 'http_checked_saved_file', 'string')) {
return false;
}
# if file does not exist just pipe
if (!$file) {
return $pipe;
}
# reset config and return
set_config_value('http_checked_saved_file', null);
return array('file' => $file, 'response_file' => $file);
}
function merge_relational_data($opts, $pipe, $cmd = __FUNCTION__)
{
# set prefix
$prefix = 'merge_relational_data';
# merge opts
$opts = merge_opts($opts, $pipe);
# get data opt
$data = get_opt_config_value($prefix, $opts, 'data');
if (!check_opt_set_type($cmd, $data, 'data', 'array')) {
return false;
}
# get fields opt
$fields = get_opt_config_value($prefix, $opts, 'fields');
if (!check_opt_set_type($cmd, $fields, 'fields', 'array_of_strings')) {
return false;
}
# get primary key opt
$primary_key = get_opt_config_value($prefix, $opts, 'primary_key');
if (!check_opt_set_type($cmd, $primary_key, 'primary_key', 'string')) {
return false;
}
# get check field opt
$merge_rules = get_opt_config_value($prefix, $opts, 'merge_rules');
if (!check_opt_set_type($cmd, $merge_rules, 'merge_rules', 'array,yaml_array')) {
return false;
}
if (is_string($merge_rules)) {
$merge_rules = yaml_decode($merge_rules);
}
# set up primary key
if (!in_array($primary_key, $fields)) {
return error($cmd, "field '{$primary_key}' does not exist");
}
$fields_flipped = array_flip($fields);
$primary_key_index = $fields_flipped[$primary_key];
# set up new fields
$new_fields = array();
foreach ($fields as $field) {
$new_fields[] = $field;
}
# loop through all the rules
$rules = array();
foreach ($merge_rules as $i => $merge_rule) {
$rule = array();
# get check field opt
$check_field = get_opt_config_value($prefix, $merge_rule, 'check_field');
if (!check_opt_set_type($cmd, $check_field, 'check_field', 'string')) {
return false;
}
# check that the fields exist
if (!in_array($check_field, $fields)) {
return error($cmd, "field '{$check_field}' does not exist");
}
$check_field_index = $fields_flipped[$check_field];
$rule[0] = $check_field_index;
# get value field opt
$value_field = get_opt_config_value($prefix, $merge_rule, 'value_field');
if (!check_opt_if_set_type($cmd, $value_field, 'value_field', 'string')) {
return false;
}
# get value field map opt
$value_field_map = get_opt_config_value($prefix, $merge_rule, 'value_field_map');
if (!check_opt_if_set_type($cmd, $value_field_map, 'value_field_map', 'array,yaml_array')) {
return false;
}
if ($value_field_map && !is_array($value_field_map)) {
$value_field_map = yaml_decode($value_field_map);
}
if ($value_field === null && $value_field_map === null) {
return opt_not_set_msg($cmd, 'value_field,value_field_map');
}
# check the type of the rule
if ($value_field !== null) {
$rule[1] = 'f';
if (!in_array($value_field, $fields)) {
return error($cmd, "field '{$value_field}' does not exist");
}
$value_field_index = $fields_flipped[$value_field];
$rule[2] = $value_field_index;
} else {
$rule[1] = 'fm';
# set up value field map
foreach ($value_field_map as $value => $field) {
if (!in_array($field, $new_fields)) {
$new_fields[] = $field;
}
}
# create the value indexes
$new_fields_flipped = array_flip($new_fields);
$values_indexes = array();
foreach ($value_field_map as $value => $field) {
$values_indexes[$value] = $new_fields_flipped[$field];
}
$rule[2] = $values_indexes;
}
# get value opt
# TODO: value_map
if (array_key_exists('value', $merge_rule)) {
$rule[3] = 'v';
$rule[4] = $merge_rule['value'];
} elseif (array_key_exists('value_map', $merge_rule)) {
$value_map = get_opt_config_value($prefix, $merge_rule, 'value_map');
if (!check_opt_set_type($cmd, $value_map, 'value_map', 'array,yaml_array')) {
return false;
}
if (is_string($value_map)) {
$value_map = yaml_decode($value_map);
}
$rule[3] = 'vm';
$rule[4] = $value_map;
} else {
$rule[3] = 'cv';
}
# add rule to rules
$rules[] = $rule;
}
# set up counts
$data_count = count($data);
$field_count = count($fields);
$new_field_count = count($new_fields);
# set up new data
$new_data = array();
$new_data_indexes = array();
# loop over data and apply rules
debug_echo($cmd, "merging relational data (this can take some time) ...");
for ($i = 0; $i < $data_count; $i++) {
# get line and check if line already exists
$line = $data[$i];
$primary_value = $line[$primary_key_index];
unset($new_line);
$new_line =& $new_data_indexes[$primary_value];
# create new line
if (!$new_line) {
unset($new_line);
$new_line = array();
for ($j = 0; $j < $field_count; $j++) {
$new_line[$j] = $line[$j];
}
for ($j = $field_count; $j < $new_field_count; $j++) {
$new_line[] = null;
}
$new_data[] =& $new_line;
$new_data_indexes[$primary_value] =& $new_line;
}
# loop through the rules to modify the data
foreach ($rules as $rule) {
# set up rule parts
$check_field_index = $rule[0];
$value_field_type = $rule[1];
$value_field_index = $rule[2];
$value_type = $rule[3];
# intiiate check
$check_value = $line[$check_field_index];
if ($check_value === null) {
continue;
}
# get value field
switch ($value_field_type) {
case 'fm':
$value_field_index = @$value_field_index[$check_value];
break;
}
if ($value_field_index === null) {
continue;
}
# get value
switch ($value_type) {
case 'v':
$value = $rule[4];
break;
case 'vm':
$value = $rule[4][$check_value];
break;
case 'cv':
$value = $check_value;
break;
}
# set the value
$new_line[$value_field_index] = $value;
}
}
# build result
$res = array('data' => $new_data, 'fields' => $new_fields);
return $res;
}
function imap_search_for_email($opts, $pipe, $cmd = __FUNCTION__)
{
# set prefix
$prefix = 'imap_search_for_email';
# unset opts that might affect the request
unset($pipe['dir']);
unset($pipe['file']);
unset($pipe['path']);
# merge opts
$merged_opts = merge_opts($opts, $pipe);
# get tests opt
$tests = get_opt($prefix, $merged_opts, 'tests');
if (!check_opt_set_type($cmd, $tests, 'tests', 'array')) {
return false;
}
# perform serach of mailbox
$r = imap_get_mailbox_email_count($opts, $pipe, $cmd);
if ($r === false) {
return false;
}
# get the count
$email_count = $r['email_count'];
# split the tests
$header_tests = @$tests['headers'];
$body_tests = @$tests['body'];
# loop over the emails to get the headers
$found = false;
for ($i = $email_count; $i >= 1; $i--) {
# set the email id
$email_id = $i;
$part_id = false;
$email_headers_res = null;
$email_body_res = null;
$opts['email_id'] = $i;
# get the eamil headers if required
if ($header_tests) {
$email_headers_res = imap_get_email_headers($opts, $pipe, $cmd);
if ($email_headers_res === false) {
return false;
}
# get the returned headers and set as lower-case
$email_headers = $email_headers_res['email_headers'];
foreach ($email_headers as $name => $value) {
$email_headers[strtolower($name)] = $value;
}
# test the headers
$found_headers = imap_test_values($header_tests, $email_headers);
if (!$found_headers) {
continue;
}
}
if ($body_tests) {
$email_body_res = imap_get_email_body($opts, $pipe, $cmd);
if ($email_body_res === false) {
return false;
}
# get the returned headers and set as lower-case
$email_body_parts = $email_body_res['email_body_parts'];
# run through the parts
foreach ($email_body_parts as $part_idx => $part) {
$found_part = imap_test_values($body_tests, $part);
if ($found_part) {
$part_id = $part_idx + 1;
break;
}
}
if (!$found_part) {
continue;
}
}
$found = true;
break;
}
# check if found or not
if (!$found) {
return error($cmd, "email not found with the search criteria");
}
# display success message
$msg = "email found with id {$email_id}";
if ($part_id) {
$msg .= " part {$part_id}";
}
debug_echo($cmd, $msg);
# create res
$res = array('email_id' => $email_id);
if ($part_id) {
$res['email_part_id'] = $part_id;
foreach ($part as $name => $value) {
$res["email_{$name}"] = $value;
}
}
# merge res with headers and body
$res = merge_opts_for_output($opts, $res);
if ($email_headers_res) {
$res = merge_opts_for_output($res, $email_headers_res);
}
if ($email_body_res) {
$res = merge_opts_for_output($res, $email_body_res);
}
ksort($res);
return $res;
}
function backup_file($opts, $pipe = null, $cmd = __FUNCTION__)
{
# set prefix
$prefix = 'file_backup';
# merge opts
$opts = merge_opts($opts, $pipe, 'file');
# get backup file opt
$backup_file = get_opt($prefix, $opts, 'backup_file', true);
if (!check_opt_set_type($cmd, $backup_file, 'backup_file', 'boolean')) {
return false;
}
if ($backup_file === false) {
return true;
}
# get file opt
$file = get_opt($prefix, $opts, 'file');
if (!check_opt_set_type($cmd, $file, 'file', 'string')) {
return false;
}
# get suffix opt
$suffix = get_opt_config_value($prefix, $opts, 'suffix', '.bak.');
if (!check_opt_set_type($cmd, $suffix, 'suffix', 'string')) {
return false;
}
# create new path
$index = 1;
do {
$new_file = "{$file}{$suffix}{$index}";
if (!is_file($new_file)) {
break;
}
$index++;
} while (true);
# move file
debug_echo($cmd, "backing up file : {$file} => {$new_file}");
if (!rename($file, $new_file)) {
return error($cmd, "could not rename file : {$file} => {$new_file}");
}
return array('file' => $new_file);
}
function xml_parse_document($opts, $pipe, $cmd = __FUNCTION__)
{
# set prefix
$prefix = 'xml_parse_document';
# merge opts
$opts = merge_opts($opts, $pipe, 'document');
# get document opt
$document = get_opt($prefix, $opts, 'document');
if (!check_opt_set_type($cmd, $document, 'document', 'string')) {
return false;
}
# get doc type opt
$doc_type = get_opt_config_value($prefix, $opts, 'doc_type', 'xml');
if (!check_opt_set_type($cmd, $doc_type, 'doc_type', 'string')) {
return false;
}
# get logic opt
$logic = get_opt($prefix, $opts, 'logic');
if (!check_opt_set_type($cmd, $logic, 'logic', 'string,array')) {
return false;
}
if (is_string($logic)) {
$logic = @yaml_decode($logic);
if (!$logic) {
return error($cmd, "parse logic not valid YAML");
}
}
# get result type opt
$result_type = get_opt_config_value($prefix, $opts, 'result_type', 'elements');
if (!check_opt_set_type($cmd, $result_type, 'result_type', 'string')) {
return false;
}
# get merge results opt
$merge_results = get_opt($prefix, $opts, 'merge_results', false);
if (!check_opt_set_type($cmd, $merge_results, 'merge_results', 'boolean')) {
return false;
}
# load the document
$dom = new DOMDocument('1.0');
@$dom->loadHTML($document);
#$dom->preserveWhiteSpace = false;
# check that the document was parsed correctly
$res = xml_parse_element($dom, $logic);
if ($res === false) {
return error($cmd, "could not parse {$type} document");
}
# display info about the parsing
debug_echo($cmd, "{$doc_type} document parsed successfully for {$result_type}");
if (empty($res)) {
debug_echo($cmd, "no {$result_type} found while parsing the {$doc_type} document");
} else {
debug_echo($cmd, "the following {$result_type} were found in the {$doc_type} document");
debug_dump_yaml($res, true);
}
# return if we are not merging the results
if (!$merge_results) {
return $res;
}
# merge the results
$new_res = array();
$res_count = count($res);
for ($i = 0; $i < $res_count; $i++) {
$elt = $res[$i];
foreach ($elt as $key => $value) {
if (is_array($value)) {
$cur_value = @$new_res[$key];
if ($cur_value) {
$value = array_merge($cur_value, $value);
}
}
$new_res[$key] = $value;
}
}
return $new_res;
}
function modify_fields($opts, $pipe, $cmd = __FUNCTION__, $opt_prefix = false)
{
# set prefix
$prefix = 'modify_fields';
# merge opts
$opts = merge_opts($opts, $pipe);
# adjust opt prefix
$cmd = adjust_opt_prefix($cmd, $opts, $opt_prefix, 'add_fields');
# get data opt
$data =& get_opt_config_value($prefix, $opts, 'data');
if (!check_opt_set_type($cmd, $data, 'data', 'array')) {
return false;
}
# get fields opt
$fields = get_opt_config_value($prefix, $opts, 'fields');
if (!check_opt_set_type($cmd, $fields, 'fields', 'array_of_strings')) {
return false;
}
# get indexes fields opt
$indexes =& get_opt_config_value($prefix, $opts, 'indexes');
if (!check_opt_if_set_type($cmd, $indexes, 'indexes', 'array')) {
return false;
}
# get map fields opt
$map_fields = get_opt_config_value($prefix, $opts, 'map');
if (!check_opt_if_set_type($cmd, $map_fields, 'map', 'string,array')) {
return false;
}
if (is_string($map_fields)) {
$map_fields = yaml_decode($map_fields);
}
# get swap fields opt
$swap_fields = get_opt_config_value($prefix, $opts, 'swap');
if (!check_opt_if_set_type($cmd, $swap_fields, 'swap', 'string,array')) {
return false;
}
if (is_string($swap_fields)) {
$swap_fields = yaml_decode($swap_fields);
}
# get add fields opt
$add_fields = get_opt_config_value($prefix, $opts, 'add');
if (!check_opt_if_set_type($cmd, $add_fields, 'add', 'array')) {
return false;
}
# get delete fields opt
$delete_fields = get_opt_config_value($prefix, $opts, 'delete');
if (!check_opt_if_set_type($cmd, $delete_fields, 'delete', 'array_of_strings')) {
return false;
}
# get reorder fields opt
$reorder_fields = get_opt_config_value($prefix, $opts, 'reorder');
if (!check_opt_if_set_type($cmd, $reorder_fields, 'reorder', 'array_of_strings')) {
return false;
}
# perform the operations
if ($map_fields && !map_data_fields($data, $fields, $map_fields, $cmd)) {
return false;
}
if ($swap_fields && !swap_data_fields($fields, $swap_fields, $cmd)) {
return false;
}
if ($add_fields && !add_data_fields($data, $fields, $add_fields, $cmd)) {
return false;
}
if ($delete_fields && !delete_data_fields($data, $fields, $delete_fields, $cmd)) {
return false;
}
if ($reorder_fields && !reorder_data_fields($data, $fields, $reorder_fields, $cmd)) {
return false;
}
# create indexes
/*
var_dump ($fields);
var_dump ($data[0]);
exit;
*/
# set the response
return build_result_data($fields, $data, $indexes);
}
