function download_data_from_cbr($date1, $currency) { $date2 = date("d/m/Y"); $kurs = array(); echo " download"; foreach ($currency as $c_key => $c_value) { $data = askhost("http://www.cbr.ru/scripts/XML_dynamic.asp?date_req1={$date1}&date_req2={$date2}&VAL_NM_RQ={$c_key}"); $xml = simplexml_load_string($data); foreach ($xml->Record as $item) { $this_record_ts = strtotime((string) $item['Date']); $this_currency_quantity = floatval((string) $item->Nominal); $this_currency_cost = floatval(str_replace(",", ".", (string) $item->Value)) / $this_currency_quantity; $kurs[$this_record_ts][$c_key] = $this_currency_cost; } // foreach record } // foreach curr return $kurs; }
// add new investment $newround = array(); $newround['id'] = $round_uuid; //$round_fname="/tmp/cb/$round_uuid.json"; //$newround['recipient']=strip_characters($funding->invested_in->name); $rcpt_path = strip_first_part($funding->invested_in->path); $newround['recipient'] = $rcpt_path; $fround = null; if (isset($fullinv[$round_uuid])) { $mnginv = load_json_from_mongo($round_uuid, $jsoninv_collection); $fround = json_decode(bzdecompress($mnginv['jsondata']->bin)); echo "+"; } if (!isset($fround->data->uuid)) { echo "\nbad investment data: " . print_r($fround, true) . "\ntrying to reload "; $res2 = askhost("http://api.crunchbase.com/v/2/funding-round/" . $round_uuid . "?user_key={$apikey}", FALSE, "", "", "1", 60000, "", true); $fround = json_decode($res2['data']); if (isset($fround->data->uuid)) { save_json_to_mongo($fround, $round_uuid, $jsoninv_collection); //file_put_contents($round_fname, json_encode($fround,JSON_PRETTY_PRINT) ); echo "-"; } else { echo "!"; $apikey = get_api_key(); } } if (null == $fround) { echo "no investment data - skip"; //print_r($fround); continue; }
if (!isset($js->data->uuid)) { continue; echo "#"; } //bad data if (!isset($js->data->properties->funding_type)) { continue; echo "!"; } //bad data //update invround if ($need_update_jsons) { $objname = $doc['id']; $url = "http://api.crunchbase.com/v/2/funding-round/" . $objname . "?user_key={$apikey}"; //echo $doc['id'].": ".$js->data->uuid."\n"; $res = askhost($url, FALSE, "", "", "1", 60000, "", true); $js = json_decode($res['data']); if (isset($js->data->uuid)) { save_json_to_mongo($js, $objname, $jsoninv_collection); } else { echo "_"; $apikey = get_api_key(); $js = $js2; } } $ftype = $js->data->properties->funding_type; if (!isset($js->data->properties->money_raised_usd)) { echo "no usdsum, skip\n"; continue; } $usdsum = $js->data->properties->money_raised_usd;
function convert_to_usd(&$curr, $txn_cur, $amount) { if (!isset($curr[$txn_cur])) { $result = askhost("http://rate-exchange.appspot.com/currency?from={$txn_cur}&to=USD"); $jsonres = json_decode($result, true); if (!isset($jsonres['rate'])) { log_fatal("unknown currency {$txn_cur}"); } $curr[$txn_cur] = $jsonres['rate']; } // lookup new currency return floor($curr[$txn_cur] * $amount); }
// do it $key = (int) $_GET["obj_key"]; include_once 'askhost.php'; $dt = get_datatype1(); $industry = get_datatype2(); $fname = tempnam("/tmp", "bls1"); $csvname = get_data_from_mongo($dt, $industry, $fname); zip_compress_csv($fname . "zip", $csvname); @unlink($fname); $secret = sha1($capsidea_client_secret . $capsidea_permanent_access_token); $array1 = create_list_of_measures($dt); $measures = ""; foreach ($array1 as $val) { $measures = $measures . "{Name: \"{$val}\", TypeName:\"double\" },"; } $schemajson = "&fields=" . urlencode('[{Name: "date", TypeName:"timestamp"},' . $measures . '{Name: "industry", TypeName:"string"}]'); $host_reply = askhost($server_url . "&schemakey=" . $key . $schemajson, array('file_contents' => '@' . $fname . ".zip"), "", "", "", 800000, array("appid: {$capsidea_appid}", "sig: {$secret}"), true); // defined in askhost.php //echo "done\n"; $httpcode = $host_reply["httpcode"]; // $jsonres=json_decode($host_reply["data"], true); // $key=(int)$jsonres["Key"]; // if (0==$key) $key=$cdata["k"]; $error_log = "response:<pre>" . $host_reply["data"] . "</pre>" . "<br>connection debug:<pre>" . $host_reply["d"] . "</pre>"; if (200 == $httpcode) { unlink($fname . ".zip"); } else { file_put_contents("/tmp/bls-errors.log", date(DATE_ATOM) . print_r($_REQUEST, true) . "\n" . print_r($host_reply, true) . " \n", FILE_APPEND); } die("OK"); }
<?php // crunchbase daily updater // http://static.crunchbase.com/daily/content_web.html include_once 'cb-inc.php'; $m = new MongoClient(); $db = $m->crunchbase; $jsoninv_collection = $db->jsoninv; $jsondata_collection = $db->jsondata; $jsoninv_collection = $db->jsoninv; $fulldata_collection = $db->fulldata; $investments_collection = $db->investments; $investments = load_investments_from_mongo($investments_collection); $fulljson = load_objs_list_from_mongo($jsondata_collection); $data = askhost("http://static.crunchbase.com/daily/content_web.html"); // person // organization $ret = $data; while ($ret = stristr($ret, "person/")) { $pos = strpos($ret, "\""); $obj = substr($ret, 0, $pos); $ret = substr($ret, $pos + 1); echo $obj . "\n"; update_object_and_related($obj); } $ret = $data; while ($ret = stristr($ret, "organization/")) { $pos = strpos($ret, "\""); $obj = substr($ret, 0, $pos); $ret = substr($ret, $pos + 1); echo $obj . "\n";
<?php // federalreserve application for capsidea include_once 'usdrates-inc.php'; $cdata = get_capsidea_data($capsidea_client_secret); $secret = $cdata["t"]; //$my_data_dir="/tmp"; include_once 'askhost.php'; $host_reply = askhost($server_url, array('file_contents' => '@' . "{$my_data_dir}/usrates.csv"), "", "", "", 80000, array("appid: {$capsidea_appid}", "sig: {$secret}"), true); // defined in askhost.php $httpcode = $host_reply["httpcode"]; $jsonres = json_decode($host_reply["data"], true); $key = (int) $jsonres["Key"]; if (0 == $key) { $key = $cdata["k"]; } //construct dashboard json $i = 0; $sheets = ""; $datasources = ""; $usedkeys = array(); $ds_template = file_get_contents("cbr1-dst.json"); $sheet_template = file_get_contents("cbr1-st.json"); $jsondash = "{ \"Sources\":{ myfulldatasources }, \"CalcMembers\":null, \"Sheets\":[ myfulldatasheets ],\n \"ActiveSheetId\":\"myfirstsheetid\", \"SheetsVisible\":true, \"FixedSheetWidth\":false, \"SheetWidth\":842, \"Font\":{\"FontFamily\":\"DefaultFont\", \"Size\":12,\n\"Color\":\"#000000\",\"IsBold\":false,\"IsItalic\":false,\"IsUnderline\":false},\n \"Name\":\"Currency generated\", \"Description\":\"Currency dashboard\", \"Version\":\"6\"} "; $orig = array("currencyname", "mydatasourceid", "currencyvname", "mycapsideacomdatasetkey", "mysheetname", "mysheetid", "mydatablockid", "\$currency"); foreach ($cname as $value) { $kname = $value; $dskey = "ds_" . generateuniqid($usedkeys); $skey = "sheet_" . generateuniqid($usedkeys); $dbkey = "datablock_" . generateuniqid($usedkeys); $capsideaname = strtolower(str_replace(" ", "_", $kname));
} fwrite($fp, ",{$val}"); } // for fwrite($fp, "\n"); } // foreach kurs fclose($fp); //$zip = new ZipArchive(); // zip support, for future use //$filename = $fname.".zip"; //if ($zip->open($filename, ZIPARCHIVE::CREATE)!==TRUE) { die("cant open <$filename>\n");} //$zip->addFile($fname); //$zip->close(); return $fname; } $data = askhost("http://www.federalreserve.gov/datadownload/Output.aspx?rel=H10&filetype=zip"); $zipname = "/tmp/1.zip"; file_put_contents($zipname, $data); $zip = zip_open($zipname); do { $entry = zip_read($zip); } while ($entry && zip_entry_name($entry) != "H10_data.xml"); // open entry zip_entry_open($zip, $entry, "r"); // read entry $data = zip_entry_read($entry, zip_entry_filesize($entry)); zip_entry_close($entry); // close zip zip_close($zip); $filename = "/tmp/H10_data.xml"; file_put_contents($filename, $data);
$selected = array(); for ($i = 1; $i < 10; $i++) { if (isset($_POST["element_3_{$i}"])) { $selected[$i] = $_POST["element_3_{$i}"]; } } $date1 = strtotime(sprintf("%02d", $_POST["element_1_1"]) . "/" . sprintf("%02d", $_POST["element_1_2"]) . "/" . $_POST["element_1_3"]); // d/m/Y $date2 = strtotime(sprintf("%02d", $_POST["element_2_1"]) . "/" . sprintf("%02d", $_POST["element_2_2"]) . "/" . $_POST["element_2_3"]); $rangeQuery = array('ts' => array('$gt' => 0, '$lt' => time())); // select all new records $kurs = load_array_from_mongo($data_collection, $rangeQuery, $selected); //file_put_contents("$my_data_dir/c.log",print_r($kurs,true)); $fname = save_array_as_csv($selected, $kurs); $stime = get_timer(); $host_reply = askhost($server_url . $schemakey, array('file_contents' => '@' . $fname), "", "", "", 80000, array("appid: {$capsidea_appid}", "sig: {$secret}"), true); // defined in askhost.php $capsidea_time = get_timer() - $stime; unlink($fname); $httpcode = $host_reply["httpcode"]; $jsonres = json_decode($host_reply["data"], true); $key = (int) $jsonres["Key"]; if (0 == $key) { $key = $cdata["k"]; } //construct dashboard json $i = 0; $sheets = ""; $datasources = ""; $usedkeys = array(); $ds_template = file_get_contents("cbr1-dst.json");
//print_r($res); // seriesID //print_r($res->Results->series); //die(); $counter = 0; //if ("REQUEST_FAILED"==$res->status) if (FALSE === strpos($res->status, "REQUEST_SUCCEEDED")) { //file_put_contents("/tmp/save-array-err.tmp", serialize($fields)); //file_put_contents("/tmp/save-industry-err.tmp", serialize($value)); //echo "\n".$res->status."\n aborting!\n"; //echo "\n $json_rq \n"; //echo $resp['data']."\n".$resp['d']."\n"; while (true) { echo ","; sleep(60); $resp = askhost($url, $json_rq, "", "", "", 90000, array("Content-Type: application/json"), true); $res = json_decode($resp['data']); if (FALSE !== strpos($res->status, "REQUEST_SUCCEEDED")) { break; } // ok! } //die(); } foreach ($res->Results->series as $result_series) { if (!isset($result_series->data[0])) { echo "!"; continue; // no data } //echo $result_series."\n";
<?php // kickstarter data extractor for capsidea include 'k-inc.php'; $projects = load_prj_from_db(); $locations = load_loc_from_db(); $category = load_cat_from_db(); $creators = load_creators_from_db(); // https://www.kickstarter.com/projects/alradeck/caelum-sky-illustrated-1?format=json $i = 0; while (true) { $res = askhost("https://www.kickstarter.com/discover/advanced?format=json&page={$i}", FALSE, "", "", "1", 60000, "", true); $i++; if (500 == $res['httpcode']) { die('\\n some error 500\\n'); } $js = json_decode($res['data']); echo "\n{$i}: "; //.$res['data']."\n"; if (!isset($js->projects[0]->id)) { die("complete\n"); } foreach ($js->projects as $prj) { // init with empty values $cat_pid = 0; $creator_slug = ""; // get real values $pid = (int) $prj->id; $cat_id = (int) $prj->category->id; @($loc_id = (int) $prj->location->id); $creator_id = $prj->creator->id;
$report_fname = prepare_report(); echo "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">\n<html xmlns=\"http://www.w3.org/1999/xhtml\"><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\">\n<script type=\"text/javascript\" src=\"http://beta.capsidea.com/api.js\"></script>"; echo "<script>function doit(key) {CI.updateSource(key);CI.openSource(key);CI.closeApp();}</script>"; //echo "<script>CI.closeApp();</script>"; echo "<title>Success</title>\n<link rel=\"stylesheet\" type=\"text/css\" href=\"view.css\" media=\"all\"></head>\n<body id=\"main_body\"> "; echo "please standby while Capsidea.com processing report<br>NB: This can take a while, so be patient<br>"; flush(); ob_flush(); flush(); ob_flush(); flush(); ob_flush(); flush(); ob_flush(); $name = date("YMdHi"); $host_reply = askhost($server_url . "&name={$name}" . $schemakey . $schemajson, array('file_contents' => '@' . $report_fname), "", "", "", 1800000, array("appid: {$capsidea_appid}", "sig: {$secret}"), true); // defined in askhost.php //echo " time ".$time3=((get_timer()-$stime)/1)." sec "; // $stime=get_timer(); $result = $host_reply["data"]; $error_log = "cube: {$schemakey}" . "secret: " . $secret . "<br>response:<pre>" . $host_reply["data"] . "</pre>" . "<br>connection debug:<pre>" . $host_reply["d"] . "</pre>"; if (500 == $host_reply["httpcode"]) { echo "ERR: {$error_log}\n" . $host_reply["httpcode"]; log_fatal("error 500 from cps: \n {$error_log}"); } // if 500 if (401 == $host_reply["httpcode"]) { echo "ERR: unauthorized {$error_log}\n" . $host_reply["httpcode"]; log_fatal("error 401 from cps \n {$error_log}"); } // if 500 if (200 == $host_reply["httpcode"]) {
// federalreserve data connector for capsidea include_once 'usdrates-inc.php'; file_put_contents("/tmp/debug.log", date(DATE_ATOM) . " usdrates-dc " . print_r($_REQUEST, true) . " \n", FILE_APPEND); $capsidea_appid = 2233; $capsidea_client_secret = "put-your-data-here"; $cdata = get_capsidea_data($capsidea_client_secret); $secret = $cdata["t"]; $schemakey = ""; if (isset($cdata["k"])) { // check, if this is reinitialisation $cps_key = $cdata["k"]; $schemakey = "&schemakey={$cps_key}"; } //$my_data_dir="/tmp"; include_once 'askhost.php'; $host_reply = askhost($server_url . $schemakey . "&frequency=daily", array('file_contents' => '@' . "{$my_data_dir}/usrates.csv"), "", "", "", 80000, array("appid: {$capsidea_appid}", "sig: {$secret}"), true); // defined in askhost.php $httpcode = $host_reply["httpcode"]; $jsonres = json_decode($host_reply["data"], true); $key = (int) $jsonres["Key"]; if (0 == $key) { $key = $cdata["k"]; } //construct dashboard json $i = 0; $sheets = ""; $datasources = ""; //$usedkeys=array(); //$ds_template=file_get_contents("cbr1-dst.json"); //$sheet_template=file_get_contents("cbr1-st.json"); // $jsondash="{ \"Sources\":{ myfulldatasources }, \"CalcMembers\":null, \"Sheets\":[ myfulldatasheets ],
// all ok, proceed to cps // create json for DB if (0 == $parent_key) { $source_id = $ikey; } else { $source_id = $parent_key; } // get element codes $url = $base_url . "?s=DataService&m=OpenDataSource"; $post_data = "[\"{$ikey}\$paypal\"]"; $res = askhost($url, $post_data, "", "", "", 180000, array("appid: {$capsidea_appid}", "sig: {$secret}", "Content-Type: application/json;charset=UTF-8"), true); mylog($res['d'] . "| " . $post_data . " json: " . $res['data']); //$secret1=sha1($capsidea_client_secret.$capsidea_permanent_access_token); $url = $base_url . "?s=DataService&m=GetChildEls"; $post_data = "[\"{$ikey}\$case_reason\",\"\"]"; $res = askhost($url, $post_data, "", "", "", 180000, array("appid: {$capsidea_appid}", "sig: {$secret}", "Content-Type: application/json;charset=UTF-8"), true); mylog($res['d'] . "| " . $post_data . " json: " . $res['data']); $js = json_decode($res['data']); //print_r($js); $selected = ""; $i = 0; foreach ($js as $this_item) { if ($this_item->n == "") { continue; } if ($i > 0) { $selected = $selected . ","; } $selected = $selected . "\"" . $this_item->k . "\""; $i++; }