save_array_to_mongo($kurs, $currency, $data_collection);
$collection->update(array(), array("ts" => time()));
echo " updating capsidea.com datasets:";
$secret = sha1($capsidea_client_secret . $capsidea_permanent_access_token);
$clients_collection = $db->cbrclients;
$cursor = $clients_collection->find();
foreach ($cursor as $this_item) {
$selected = unserialize(base64_decode($this_item["selected"]));
$schemakey = $this_item["schemakey"];
echo " {$schemakey}";
// show my progress
$rangeQuery = array('ts' => array('$gt' => $last_ts, '$lt' => time()));
// select all new records
$kurs = load_array_from_mongo($data_collection, $rangeQuery, $selected);
// save array as csv
$fname = save_array_as_csv($selected, $kurs);
$stime = get_timer();
$host_reply = askhost($server_url . "&schemakey=" . $schemakey, array('file_contents' => '@' . $fname), "", "", "", 80000, array("appid: {$capsidea_appid}", "sig: {$secret}"), true);
// defined in askhost.php
$capsidea_time = get_timer() - $stime;
unlink($fname);
$result = $host_reply["data"];
$error_log = "secret: {$secret} response: \n" . $host_reply["data"] . "\nconnection debug:\n" . $host_reply["d"];
if (200 != $host_reply["httpcode"]) {
mylog("ERR: {$error_log} " . $host_reply["httpcode"]);
die;
}
// if !200
mylog("OK: {$schemakey} in {$capsidea_time} sec");
}
echo " complete\n";
file_put_contents($my_data_dir . "/paypal_ds.log", print_r($ret, true) . " \n cdata:" . print_r($cdata, true) . "\n reply:" . print_r($host_reply, true) . "\n pgerr:" . pg_errormessage() . "\n", FILE_APPEND);
$resp = "system error, unable update client in database. Please contact support.";
file_put_contents($my_data_dir . "/" . $hash . ".prc", "100");
file_put_contents($my_data_dir . "/" . $hash . ".resp", $resp);
die;
}
} else {
if (1 != pg_affected_rows(@pg_query("insert into client (iname, ikey, ihash, ieml, cname, sftplogin, sftppwd,idate, todo, iparent, active, app ) values ('{$company_name}',{$ikey},'{$hash}','{$eml}','{$contact_name}','{$sftplog}','{$sftppwd}',CURRENT_TIMESTAMP,1,{$parent_key}, 1, {$capsidea_appid})"))) {
file_put_contents($my_data_dir . "/paypal_ds.log", date(DATE_ATOM) . " ERR CPS ERR: client {$eml} [{$company_name}] {$contact_name} l: {$sftplog} p: {$sftppwd} \n", FILE_APPEND);
file_put_contents($my_data_dir . "/paypal_ds.log", print_r($ret, true) . " \n cdata:" . print_r($cdata, true) . "\n reply:" . print_r($host_reply, true) . "\n pgerr:" . pg_errormessage() . "\n", FILE_APPEND);
$resp = "system error, unable to insert client into database. Please contact support.";
file_put_contents($my_data_dir . "/" . $hash . ".prc", "100");
file_put_contents($my_data_dir . "/" . $hash . ".resp", $resp);
die;
}
}
// all ok, proceed to cps
// create json for DB
if (0 == $parent_key) {
$source_id = $ikey;
} else {
$source_id = $parent_key;
}
$resp = "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">\n<html xmlns=\"http://www.w3.org/1999/xhtml\"><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\">\n<script type=\"text/javascript\" src=\"http://beta.capsidea.com/api.js\"></script><script>CI.updateSource({$ikey});CI.closeApp();</script>\n<title>Success</title><link rel=\"stylesheet\" type=\"text/css\" href=\"view.css\" media=\"all\"></head><body id=\"main_body\"\n<img id=\"top\" src=\"top.png\" alt=\"\"><div id=\"form_container\"><h1><b>Source {$ikey} created</b></h1><br>\n<div id=\"footer\"></div></div><img id=\"bottom\" src=\"bottom.png\" alt=\"\"></body></html>";
// report time and mem
$mem = floor(memory_get_peak_usage(true) / (1024 * 1024));
$full_time = get_timer() - $stime1;
mystat("signup [{$ikey}] mem: {$mem} MB list: {$list_time} upload: {$cps_time} size: {$fsize} MB full: {$full_time}");
file_put_contents($my_data_dir . "/" . $hash . ".resp", $resp);
file_put_contents($my_data_dir . "/" . $hash . ".prc", "100");
die;
}
// position recovered, proceed
if (1 == $need_recover) {
continue;
}
//if (("41424800"!=$series)&&("20237110"!=$ series)&&("10212100"!=$series)) continue;
echo "\n[{$series}] {$value} ({$save_counter})";
//$keys_ind[]=$value;
//if (0==$counter) {echo $resp['data'];die("\nerr no data in whole industry!");} // debug
if ($save_counter > 0) {
//if (10==$save_counter) {
echo "\nautosaving work ";
$stime = get_timer();
file_put_contents("/tmp/save-array.tmp", gzcompress(serialize($fields)));
file_put_contents("/tmp/save-industry.tmp", serialize($value));
echo get_timer() - $stime . "sec\n";
$save_counter = 0;
}
$save_counter++;
$counter = 0;
// $years=array(1930,1941,1952,1963,1974,1985,1996,2007);
$years = array(2014);
foreach ($years as $this_year) {
$sy = $this_year;
$ey = $this_year + 10;
if ($ey > (int) date("Y")) {
$ey = (int) date("Y");
}
$series_num = 1;
$list_series = "";
//\"CEU{$series}{$datatype}\"
function generate_application_report_to_cvs($client_dir, $dbconn, $source_id, $cases, &$merchant, $ware, $cntryarr)
{
global $dbg;
$fname = generatehash();
$report_fname = $client_dir . "/" . $fname . ".csv";
$fw = fopen($report_fname, "w");
fputs($fw, "txn_lag,fee,amount,start_ts,stop_ts,status,tcode,currency,case_reason,case_status,case_money_move,due_date,case_type,case_lag,case_date,case_amount,days_before_case_closure,project,country\n");
$qry = "select * from txn where mid<>'' and amount>0 and refid='' and ikey={$source_id} and stop_ts>(date_trunc('month',CURRENT_TIMESTAMP)- interval '1 month') order by stop_ts;";
//$qry="select * from txn where mid<>'' and amount>0 and (refid='' or refid is null) and ikey=1591 and (stop_ts>(date_trunc('month',CURRENT_TIMESTAMP)- interval '1 month') or (stop_ts between (date_trunc('month',CURRENT_TIMESTAMP)- interval '13 month') and (CURRENT_TIMESTAMP - interval '12 month'))) order by stop_ts;";
//$qry="select * from txn where mid<>'' and amount>0 and (refid='' or refid is null) and ikey=1591 order by stop_ts;";
//$dbg=$dbg."\n executing txn qry $qry... ";
//$stime=get_timer();
$pgres = pg_query($qry);
if (false === $pgres) {
$qr = pg_errormessage($dbconn);
log_fatal("generate_application_report_to_cvs: src {$source_id} ERR {$qr} in query: {$qry} ");
}
//$tcount=pg_num_rows($pgres);
//$dbg=$dbg."($tcount) time ".((get_timer()-$stime)/1)." sec "; // $stime=get_timer();
$stime = get_timer();
while ($row = pg_fetch_assoc($pgres)) {
// process txn row
$line = array();
//if (isset($row["txnlag"])) $line["lag"]=$row["txnlag"]; else $line["lag"]="null";
if (isset($row["txnlag"])) {
$line["lag"] = $row["txnlag"];
} else {
$line["lag"] = "0";
}
$line["fee"] = -1 * ((int) $row["fee"] / 100);
$line["amount"] = (int) $row["amount"] / 100;
$line["stopts"] = convert_to_cps_date($row["stop_ts"]);
if (isset($row["start_ts"])) {
$line["startts"] = convert_to_cps_date($row["start_ts"]);
} else {
$line["startts"] = $line["stopts"];
}
$line["status"] = (string) $row["status"];
if (isset($row["tcode"])) {
$line["tcode"] = (string) $row["tcode"];
} else {
$line["tcode"] = " ";
}
$line["currency"] = (string) $row["currency"];
$ppid = $row["ppid"];
$mid = $row["mid"];
if (isset($cases[$mid])) {
$row2 = $cases[$mid];
$line["creason"] = $row2["creason"];
$line["cstatus"] = $row2["cstatus"];
$line["cmm"] = $row2["cmm"];
$line["cdd"] = convert_to_cps_date($row2["cdd"]);
$line["ctype"] = $row2["ctype"];
$line["clag"] = $row2["clag"];
$line["cdate"] = convert_to_cps_date($row2["cdate"]);
$line["camount"] = $row2["camount"];
$line["cdays"] = $row2["cdays"];
//$dbg=$dbg."*";
} else {
// null values
$line["creason"] = "null";
$line["cstatus"] = "null";
$line["cmm"] = " ";
$line["cdd"] = convert_to_cps_date("0");
$line["ctype"] = " ";
$line["clag"] = "0";
$line["cdate"] = convert_to_cps_date("0");
$line["camount"] = "0";
$line["cdays"] = "0";
}
// no case - empty values
if (isset($merchant[$mid])) {
$row2 = $merchant[$mid];
$line["prj"] = process_prj_names($ware[$row2[1]]);
if (!isset($cntryarr[$row2[2]])) {
//$dbg=$dbg."country not found! '".$row2[2]."' \n";
$line["country"] = "unknown country";
} else {
$line["country"] = $cntryarr[$row2[2]];
}
} else {
$line["prj"] = "null";
$line["country"] = "null";
}
fputcsv($fw, $line);
}
// process txn's
fclose($fw);
return $report_fname;
}
$res = @pg_query($qry);
if (false === $res) {
$qr = pg_errormessage($dbconn);
$line = implode(",", $this_item);
file_put_contents("{$my_data_dir}/loader.log", date(DATE_ATOM) . " (line: {$line}) client {$key} ERR {$qr} in query: {$qry} file: {$nfname}\n", FILE_APPEND);
log_fatal("ERR error in " . implode(",", array_keys($this_item)) . "values" . implode(",", $this_item));
}
}
// case?
if ($i > 10000) {
// @pg_query("commit;");
echo ".";
ob_flush();
flush();
ob_flush();
flush();
//@pg_query("begin;");
$lines_processed = $lines_processed + $i;
$i = 0;
}
}
@pg_query("commit;");
$lines_processed = $lines_processed + $i;
echo "\nOK {$lines_processed} records accepted";
@pg_query("update client set todo=1 where ikey={$key};");
// flag client as updated
@pg_query("update client set todo=1 where iparent={$key};");
$time = get_timer() - $stime;
//total timer
$mem = floor(memory_get_peak_usage(true) / (1024 * 1024));
file_put_contents("{$my_data_dir}/txn-stat.log", date(DATE_ATOM) . " client {$key} lines: {$lines_processed} time: {$time} sec mem: {$mem} MB\n", FILE_APPEND);
// $err="cube: $merchant_cube_id"."secret: ".$secret."<br>response:<pre>".$host_reply["data"]."</pre>"."<br>connection debug:<pre>".$host_reply["d"]."</pre>";
// if (500==$host_reply["httpcode"]) {
// echo "ERR: $err\n".$host_reply["httpcode"];
// log_fatal("error 500 from cps: \n $dbg");
// } // if 500
// if (401==$host_reply["httpcode"]) {
// echo "ERR: unauthorized $err\n".$host_reply["httpcode"];
// log_fatal("error 401 from cps \n $dbg");
// } // if 500
// if (200==$host_reply["httpcode"]) {
// echo "unlinking ".$report_fname."\n";
// unlink($report_fname);
// echo "OK (http == 200)\n";
// } else {echo "UNK: $err\n".$host_reply["httpcode"];}
pg_query("update client set pdate=CURRENT_TIMESTAMP, todo=0 where ikey=" . $merchant_cube_id);
$time4 = get_timer() - $time4;
//total timer
$mem = floor(memory_get_peak_usage(true) / (1024 * 1024));
//file_put_contents("/tmp/paypal-gen.log", date(DATE_ATOM)." client $client_name [$source_id] cases: $ccount merchant $mcount / $time2 sec] mem: $mem MB loading: $time1 uploading: $time3 size: $fsize MB full: $time4\n", FILE_APPEND);
file_put_contents("/tmp/paypal-gen.log", date(DATE_ATOM) . " link client {$client_name} [{$source_id}] mem: {$mem} MB loading: {$time1} full: {$time4}\n", FILE_APPEND);
}
// select client which need update
if (isset($force_run)) {
echo "forced run done\n";
die;
}
echo $dbg;
if ($data_modified) {
$sleep_interval = 1;
} else {
$sleep_interval = 30;
