save_array_to_mongo($kurs, $currency, $data_collection); $collection->update(array(), array("ts" => time())); echo " updating capsidea.com datasets:"; $secret = sha1($capsidea_client_secret . $capsidea_permanent_access_token); $clients_collection = $db->cbrclients; $cursor = $clients_collection->find(); foreach ($cursor as $this_item) { $selected = unserialize(base64_decode($this_item["selected"])); $schemakey = $this_item["schemakey"]; echo " {$schemakey}"; // show my progress $rangeQuery = array('ts' => array('$gt' => $last_ts, '$lt' => time())); // select all new records $kurs = load_array_from_mongo($data_collection, $rangeQuery, $selected); // save array as csv $fname = save_array_as_csv($selected, $kurs); $stime = get_timer(); $host_reply = askhost($server_url . "&schemakey=" . $schemakey, array('file_contents' => '@' . $fname), "", "", "", 80000, array("appid: {$capsidea_appid}", "sig: {$secret}"), true); // defined in askhost.php $capsidea_time = get_timer() - $stime; unlink($fname); $result = $host_reply["data"]; $error_log = "secret: {$secret} response: \n" . $host_reply["data"] . "\nconnection debug:\n" . $host_reply["d"]; if (200 != $host_reply["httpcode"]) { mylog("ERR: {$error_log} " . $host_reply["httpcode"]); die; } // if !200 mylog("OK: {$schemakey} in {$capsidea_time} sec"); } echo " complete\n";
file_put_contents($my_data_dir . "/paypal_ds.log", print_r($ret, true) . " \n cdata:" . print_r($cdata, true) . "\n reply:" . print_r($host_reply, true) . "\n pgerr:" . pg_errormessage() . "\n", FILE_APPEND); $resp = "system error, unable update client in database. Please contact support."; file_put_contents($my_data_dir . "/" . $hash . ".prc", "100"); file_put_contents($my_data_dir . "/" . $hash . ".resp", $resp); die; } } else { if (1 != pg_affected_rows(@pg_query("insert into client (iname, ikey, ihash, ieml, cname, sftplogin, sftppwd,idate, todo, iparent, active, app ) values ('{$company_name}',{$ikey},'{$hash}','{$eml}','{$contact_name}','{$sftplog}','{$sftppwd}',CURRENT_TIMESTAMP,1,{$parent_key}, 1, {$capsidea_appid})"))) { file_put_contents($my_data_dir . "/paypal_ds.log", date(DATE_ATOM) . " ERR CPS ERR: client {$eml} [{$company_name}] {$contact_name} l: {$sftplog} p: {$sftppwd} \n", FILE_APPEND); file_put_contents($my_data_dir . "/paypal_ds.log", print_r($ret, true) . " \n cdata:" . print_r($cdata, true) . "\n reply:" . print_r($host_reply, true) . "\n pgerr:" . pg_errormessage() . "\n", FILE_APPEND); $resp = "system error, unable to insert client into database. Please contact support."; file_put_contents($my_data_dir . "/" . $hash . ".prc", "100"); file_put_contents($my_data_dir . "/" . $hash . ".resp", $resp); die; } } // all ok, proceed to cps // create json for DB if (0 == $parent_key) { $source_id = $ikey; } else { $source_id = $parent_key; } $resp = "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">\n<html xmlns=\"http://www.w3.org/1999/xhtml\"><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\">\n<script type=\"text/javascript\" src=\"http://beta.capsidea.com/api.js\"></script><script>CI.updateSource({$ikey});CI.closeApp();</script>\n<title>Success</title><link rel=\"stylesheet\" type=\"text/css\" href=\"view.css\" media=\"all\"></head><body id=\"main_body\"\n<img id=\"top\" src=\"top.png\" alt=\"\"><div id=\"form_container\"><h1><b>Source {$ikey} created</b></h1><br>\n<div id=\"footer\"></div></div><img id=\"bottom\" src=\"bottom.png\" alt=\"\"></body></html>"; // report time and mem $mem = floor(memory_get_peak_usage(true) / (1024 * 1024)); $full_time = get_timer() - $stime1; mystat("signup [{$ikey}] mem: {$mem} MB list: {$list_time} upload: {$cps_time} size: {$fsize} MB full: {$full_time}"); file_put_contents($my_data_dir . "/" . $hash . ".resp", $resp); file_put_contents($my_data_dir . "/" . $hash . ".prc", "100"); die;
} // position recovered, proceed if (1 == $need_recover) { continue; } //if (("41424800"!=$series)&&("20237110"!=$ series)&&("10212100"!=$series)) continue; echo "\n[{$series}] {$value} ({$save_counter})"; //$keys_ind[]=$value; //if (0==$counter) {echo $resp['data'];die("\nerr no data in whole industry!");} // debug if ($save_counter > 0) { //if (10==$save_counter) { echo "\nautosaving work "; $stime = get_timer(); file_put_contents("/tmp/save-array.tmp", gzcompress(serialize($fields))); file_put_contents("/tmp/save-industry.tmp", serialize($value)); echo get_timer() - $stime . "sec\n"; $save_counter = 0; } $save_counter++; $counter = 0; // $years=array(1930,1941,1952,1963,1974,1985,1996,2007); $years = array(2014); foreach ($years as $this_year) { $sy = $this_year; $ey = $this_year + 10; if ($ey > (int) date("Y")) { $ey = (int) date("Y"); } $series_num = 1; $list_series = ""; //\"CEU{$series}{$datatype}\"
function generate_application_report_to_cvs($client_dir, $dbconn, $source_id, $cases, &$merchant, $ware, $cntryarr) { global $dbg; $fname = generatehash(); $report_fname = $client_dir . "/" . $fname . ".csv"; $fw = fopen($report_fname, "w"); fputs($fw, "txn_lag,fee,amount,start_ts,stop_ts,status,tcode,currency,case_reason,case_status,case_money_move,due_date,case_type,case_lag,case_date,case_amount,days_before_case_closure,project,country\n"); $qry = "select * from txn where mid<>'' and amount>0 and refid='' and ikey={$source_id} and stop_ts>(date_trunc('month',CURRENT_TIMESTAMP)- interval '1 month') order by stop_ts;"; //$qry="select * from txn where mid<>'' and amount>0 and (refid='' or refid is null) and ikey=1591 and (stop_ts>(date_trunc('month',CURRENT_TIMESTAMP)- interval '1 month') or (stop_ts between (date_trunc('month',CURRENT_TIMESTAMP)- interval '13 month') and (CURRENT_TIMESTAMP - interval '12 month'))) order by stop_ts;"; //$qry="select * from txn where mid<>'' and amount>0 and (refid='' or refid is null) and ikey=1591 order by stop_ts;"; //$dbg=$dbg."\n executing txn qry $qry... "; //$stime=get_timer(); $pgres = pg_query($qry); if (false === $pgres) { $qr = pg_errormessage($dbconn); log_fatal("generate_application_report_to_cvs: src {$source_id} ERR {$qr} in query: {$qry} "); } //$tcount=pg_num_rows($pgres); //$dbg=$dbg."($tcount) time ".((get_timer()-$stime)/1)." sec "; // $stime=get_timer(); $stime = get_timer(); while ($row = pg_fetch_assoc($pgres)) { // process txn row $line = array(); //if (isset($row["txnlag"])) $line["lag"]=$row["txnlag"]; else $line["lag"]="null"; if (isset($row["txnlag"])) { $line["lag"] = $row["txnlag"]; } else { $line["lag"] = "0"; } $line["fee"] = -1 * ((int) $row["fee"] / 100); $line["amount"] = (int) $row["amount"] / 100; $line["stopts"] = convert_to_cps_date($row["stop_ts"]); if (isset($row["start_ts"])) { $line["startts"] = convert_to_cps_date($row["start_ts"]); } else { $line["startts"] = $line["stopts"]; } $line["status"] = (string) $row["status"]; if (isset($row["tcode"])) { $line["tcode"] = (string) $row["tcode"]; } else { $line["tcode"] = " "; } $line["currency"] = (string) $row["currency"]; $ppid = $row["ppid"]; $mid = $row["mid"]; if (isset($cases[$mid])) { $row2 = $cases[$mid]; $line["creason"] = $row2["creason"]; $line["cstatus"] = $row2["cstatus"]; $line["cmm"] = $row2["cmm"]; $line["cdd"] = convert_to_cps_date($row2["cdd"]); $line["ctype"] = $row2["ctype"]; $line["clag"] = $row2["clag"]; $line["cdate"] = convert_to_cps_date($row2["cdate"]); $line["camount"] = $row2["camount"]; $line["cdays"] = $row2["cdays"]; //$dbg=$dbg."*"; } else { // null values $line["creason"] = "null"; $line["cstatus"] = "null"; $line["cmm"] = " "; $line["cdd"] = convert_to_cps_date("0"); $line["ctype"] = " "; $line["clag"] = "0"; $line["cdate"] = convert_to_cps_date("0"); $line["camount"] = "0"; $line["cdays"] = "0"; } // no case - empty values if (isset($merchant[$mid])) { $row2 = $merchant[$mid]; $line["prj"] = process_prj_names($ware[$row2[1]]); if (!isset($cntryarr[$row2[2]])) { //$dbg=$dbg."country not found! '".$row2[2]."' \n"; $line["country"] = "unknown country"; } else { $line["country"] = $cntryarr[$row2[2]]; } } else { $line["prj"] = "null"; $line["country"] = "null"; } fputcsv($fw, $line); } // process txn's fclose($fw); return $report_fname; }
$res = @pg_query($qry); if (false === $res) { $qr = pg_errormessage($dbconn); $line = implode(",", $this_item); file_put_contents("{$my_data_dir}/loader.log", date(DATE_ATOM) . " (line: {$line}) client {$key} ERR {$qr} in query: {$qry} file: {$nfname}\n", FILE_APPEND); log_fatal("ERR error in " . implode(",", array_keys($this_item)) . "values" . implode(",", $this_item)); } } // case? if ($i > 10000) { // @pg_query("commit;"); echo "."; ob_flush(); flush(); ob_flush(); flush(); //@pg_query("begin;"); $lines_processed = $lines_processed + $i; $i = 0; } } @pg_query("commit;"); $lines_processed = $lines_processed + $i; echo "\nOK {$lines_processed} records accepted"; @pg_query("update client set todo=1 where ikey={$key};"); // flag client as updated @pg_query("update client set todo=1 where iparent={$key};"); $time = get_timer() - $stime; //total timer $mem = floor(memory_get_peak_usage(true) / (1024 * 1024)); file_put_contents("{$my_data_dir}/txn-stat.log", date(DATE_ATOM) . " client {$key} lines: {$lines_processed} time: {$time} sec mem: {$mem} MB\n", FILE_APPEND);
// $err="cube: $merchant_cube_id"."secret: ".$secret."<br>response:<pre>".$host_reply["data"]."</pre>"."<br>connection debug:<pre>".$host_reply["d"]."</pre>"; // if (500==$host_reply["httpcode"]) { // echo "ERR: $err\n".$host_reply["httpcode"]; // log_fatal("error 500 from cps: \n $dbg"); // } // if 500 // if (401==$host_reply["httpcode"]) { // echo "ERR: unauthorized $err\n".$host_reply["httpcode"]; // log_fatal("error 401 from cps \n $dbg"); // } // if 500 // if (200==$host_reply["httpcode"]) { // echo "unlinking ".$report_fname."\n"; // unlink($report_fname); // echo "OK (http == 200)\n"; // } else {echo "UNK: $err\n".$host_reply["httpcode"];} pg_query("update client set pdate=CURRENT_TIMESTAMP, todo=0 where ikey=" . $merchant_cube_id); $time4 = get_timer() - $time4; //total timer $mem = floor(memory_get_peak_usage(true) / (1024 * 1024)); //file_put_contents("/tmp/paypal-gen.log", date(DATE_ATOM)." client $client_name [$source_id] cases: $ccount merchant $mcount / $time2 sec] mem: $mem MB loading: $time1 uploading: $time3 size: $fsize MB full: $time4\n", FILE_APPEND); file_put_contents("/tmp/paypal-gen.log", date(DATE_ATOM) . " link client {$client_name} [{$source_id}] mem: {$mem} MB loading: {$time1} full: {$time4}\n", FILE_APPEND); } // select client which need update if (isset($force_run)) { echo "forced run done\n"; die; } echo $dbg; if ($data_modified) { $sleep_interval = 1; } else { $sleep_interval = 30;