function download_data_from_cbr($date1, $currency)
{
$date2 = date("d/m/Y");
$kurs = array();
echo " download";
foreach ($currency as $c_key => $c_value) {
$data = askhost("http://www.cbr.ru/scripts/XML_dynamic.asp?date_req1={$date1}&date_req2={$date2}&VAL_NM_RQ={$c_key}");
$xml = simplexml_load_string($data);
foreach ($xml->Record as $item) {
$this_record_ts = strtotime((string) $item['Date']);
$this_currency_quantity = floatval((string) $item->Nominal);
$this_currency_cost = floatval(str_replace(",", ".", (string) $item->Value)) / $this_currency_quantity;
$kurs[$this_record_ts][$c_key] = $this_currency_cost;
}
// foreach record
}
// foreach curr
return $kurs;
}
// add new investment
$newround = array();
$newround['id'] = $round_uuid;
//$round_fname="/tmp/cb/$round_uuid.json";
//$newround['recipient']=strip_characters($funding->invested_in->name);
$rcpt_path = strip_first_part($funding->invested_in->path);
$newround['recipient'] = $rcpt_path;
$fround = null;
if (isset($fullinv[$round_uuid])) {
$mnginv = load_json_from_mongo($round_uuid, $jsoninv_collection);
$fround = json_decode(bzdecompress($mnginv['jsondata']->bin));
echo "+";
}
if (!isset($fround->data->uuid)) {
echo "\nbad investment data: " . print_r($fround, true) . "\ntrying to reload ";
$res2 = askhost("http://api.crunchbase.com/v/2/funding-round/" . $round_uuid . "?user_key={$apikey}", FALSE, "", "", "1", 60000, "", true);
$fround = json_decode($res2['data']);
if (isset($fround->data->uuid)) {
save_json_to_mongo($fround, $round_uuid, $jsoninv_collection);
//file_put_contents($round_fname, json_encode($fround,JSON_PRETTY_PRINT) );
echo "-";
} else {
echo "!";
$apikey = get_api_key();
}
}
if (null == $fround) {
echo "no investment data - skip";
//print_r($fround);
continue;
}
if (!isset($js->data->uuid)) {
continue;
echo "#";
}
//bad data
if (!isset($js->data->properties->funding_type)) {
continue;
echo "!";
}
//bad data
//update invround
if ($need_update_jsons) {
$objname = $doc['id'];
$url = "http://api.crunchbase.com/v/2/funding-round/" . $objname . "?user_key={$apikey}";
//echo $doc['id'].": ".$js->data->uuid."\n";
$res = askhost($url, FALSE, "", "", "1", 60000, "", true);
$js = json_decode($res['data']);
if (isset($js->data->uuid)) {
save_json_to_mongo($js, $objname, $jsoninv_collection);
} else {
echo "_";
$apikey = get_api_key();
$js = $js2;
}
}
$ftype = $js->data->properties->funding_type;
if (!isset($js->data->properties->money_raised_usd)) {
echo "no usdsum, skip\n";
continue;
}
$usdsum = $js->data->properties->money_raised_usd;
function convert_to_usd(&$curr, $txn_cur, $amount)
{
if (!isset($curr[$txn_cur])) {
$result = askhost("http://rate-exchange.appspot.com/currency?from={$txn_cur}&to=USD");
$jsonres = json_decode($result, true);
if (!isset($jsonres['rate'])) {
log_fatal("unknown currency {$txn_cur}");
}
$curr[$txn_cur] = $jsonres['rate'];
}
// lookup new currency
return floor($curr[$txn_cur] * $amount);
}
// do it
$key = (int) $_GET["obj_key"];
include_once 'askhost.php';
$dt = get_datatype1();
$industry = get_datatype2();
$fname = tempnam("/tmp", "bls1");
$csvname = get_data_from_mongo($dt, $industry, $fname);
zip_compress_csv($fname . "zip", $csvname);
@unlink($fname);
$secret = sha1($capsidea_client_secret . $capsidea_permanent_access_token);
$array1 = create_list_of_measures($dt);
$measures = "";
foreach ($array1 as $val) {
$measures = $measures . "{Name: \"{$val}\", TypeName:\"double\" },";
}
$schemajson = "&fields=" . urlencode('[{Name: "date", TypeName:"timestamp"},' . $measures . '{Name: "industry", TypeName:"string"}]');
$host_reply = askhost($server_url . "&schemakey=" . $key . $schemajson, array('file_contents' => '@' . $fname . ".zip"), "", "", "", 800000, array("appid: {$capsidea_appid}", "sig: {$secret}"), true);
// defined in askhost.php
//echo "done\n";
$httpcode = $host_reply["httpcode"];
// $jsonres=json_decode($host_reply["data"], true);
// $key=(int)$jsonres["Key"];
// if (0==$key) $key=$cdata["k"];
$error_log = "response:<pre>" . $host_reply["data"] . "</pre>" . "<br>connection debug:<pre>" . $host_reply["d"] . "</pre>";
if (200 == $httpcode) {
unlink($fname . ".zip");
} else {
file_put_contents("/tmp/bls-errors.log", date(DATE_ATOM) . print_r($_REQUEST, true) . "\n" . print_r($host_reply, true) . " \n", FILE_APPEND);
}
die("OK");
}
<?php
// crunchbase daily updater
// http://static.crunchbase.com/daily/content_web.html
include_once 'cb-inc.php';
$m = new MongoClient();
$db = $m->crunchbase;
$jsoninv_collection = $db->jsoninv;
$jsondata_collection = $db->jsondata;
$jsoninv_collection = $db->jsoninv;
$fulldata_collection = $db->fulldata;
$investments_collection = $db->investments;
$investments = load_investments_from_mongo($investments_collection);
$fulljson = load_objs_list_from_mongo($jsondata_collection);
$data = askhost("http://static.crunchbase.com/daily/content_web.html");
// person
// organization
$ret = $data;
while ($ret = stristr($ret, "person/")) {
$pos = strpos($ret, "\"");
$obj = substr($ret, 0, $pos);
$ret = substr($ret, $pos + 1);
echo $obj . "\n";
update_object_and_related($obj);
}
$ret = $data;
while ($ret = stristr($ret, "organization/")) {
$pos = strpos($ret, "\"");
$obj = substr($ret, 0, $pos);
$ret = substr($ret, $pos + 1);
echo $obj . "\n";
<?php
// federalreserve application for capsidea
include_once 'usdrates-inc.php';
$cdata = get_capsidea_data($capsidea_client_secret);
$secret = $cdata["t"];
//$my_data_dir="/tmp";
include_once 'askhost.php';
$host_reply = askhost($server_url, array('file_contents' => '@' . "{$my_data_dir}/usrates.csv"), "", "", "", 80000, array("appid: {$capsidea_appid}", "sig: {$secret}"), true);
// defined in askhost.php
$httpcode = $host_reply["httpcode"];
$jsonres = json_decode($host_reply["data"], true);
$key = (int) $jsonres["Key"];
if (0 == $key) {
$key = $cdata["k"];
}
//construct dashboard json
$i = 0;
$sheets = "";
$datasources = "";
$usedkeys = array();
$ds_template = file_get_contents("cbr1-dst.json");
$sheet_template = file_get_contents("cbr1-st.json");
$jsondash = "{ \"Sources\":{ myfulldatasources }, \"CalcMembers\":null, \"Sheets\":[ myfulldatasheets ],\n \"ActiveSheetId\":\"myfirstsheetid\", \"SheetsVisible\":true, \"FixedSheetWidth\":false, \"SheetWidth\":842, \"Font\":{\"FontFamily\":\"DefaultFont\", \"Size\":12,\n\"Color\":\"#000000\",\"IsBold\":false,\"IsItalic\":false,\"IsUnderline\":false},\n \"Name\":\"Currency generated\", \"Description\":\"Currency dashboard\", \"Version\":\"6\"} ";
$orig = array("currencyname", "mydatasourceid", "currencyvname", "mycapsideacomdatasetkey", "mysheetname", "mysheetid", "mydatablockid", "\$currency");
foreach ($cname as $value) {
$kname = $value;
$dskey = "ds_" . generateuniqid($usedkeys);
$skey = "sheet_" . generateuniqid($usedkeys);
$dbkey = "datablock_" . generateuniqid($usedkeys);
$capsideaname = strtolower(str_replace(" ", "_", $kname));
}
fwrite($fp, ",{$val}");
}
// for
fwrite($fp, "\n");
}
// foreach kurs
fclose($fp);
//$zip = new ZipArchive(); // zip support, for future use
//$filename = $fname.".zip";
//if ($zip->open($filename, ZIPARCHIVE::CREATE)!==TRUE) { die("cant open <$filename>\n");}
//$zip->addFile($fname);
//$zip->close();
return $fname;
}
$data = askhost("http://www.federalreserve.gov/datadownload/Output.aspx?rel=H10&filetype=zip");
$zipname = "/tmp/1.zip";
file_put_contents($zipname, $data);
$zip = zip_open($zipname);
do {
$entry = zip_read($zip);
} while ($entry && zip_entry_name($entry) != "H10_data.xml");
// open entry
zip_entry_open($zip, $entry, "r");
// read entry
$data = zip_entry_read($entry, zip_entry_filesize($entry));
zip_entry_close($entry);
// close zip
zip_close($zip);
$filename = "/tmp/H10_data.xml";
file_put_contents($filename, $data);
$selected = array();
for ($i = 1; $i < 10; $i++) {
if (isset($_POST["element_3_{$i}"])) {
$selected[$i] = $_POST["element_3_{$i}"];
}
}
$date1 = strtotime(sprintf("%02d", $_POST["element_1_1"]) . "/" . sprintf("%02d", $_POST["element_1_2"]) . "/" . $_POST["element_1_3"]);
// d/m/Y
$date2 = strtotime(sprintf("%02d", $_POST["element_2_1"]) . "/" . sprintf("%02d", $_POST["element_2_2"]) . "/" . $_POST["element_2_3"]);
$rangeQuery = array('ts' => array('$gt' => 0, '$lt' => time()));
// select all new records
$kurs = load_array_from_mongo($data_collection, $rangeQuery, $selected);
//file_put_contents("$my_data_dir/c.log",print_r($kurs,true));
$fname = save_array_as_csv($selected, $kurs);
$stime = get_timer();
$host_reply = askhost($server_url . $schemakey, array('file_contents' => '@' . $fname), "", "", "", 80000, array("appid: {$capsidea_appid}", "sig: {$secret}"), true);
// defined in askhost.php
$capsidea_time = get_timer() - $stime;
unlink($fname);
$httpcode = $host_reply["httpcode"];
$jsonres = json_decode($host_reply["data"], true);
$key = (int) $jsonres["Key"];
if (0 == $key) {
$key = $cdata["k"];
}
//construct dashboard json
$i = 0;
$sheets = "";
$datasources = "";
$usedkeys = array();
$ds_template = file_get_contents("cbr1-dst.json");
//print_r($res);
// seriesID
//print_r($res->Results->series);
//die();
$counter = 0;
//if ("REQUEST_FAILED"==$res->status)
if (FALSE === strpos($res->status, "REQUEST_SUCCEEDED")) {
//file_put_contents("/tmp/save-array-err.tmp", serialize($fields));
//file_put_contents("/tmp/save-industry-err.tmp", serialize($value));
//echo "\n".$res->status."\n aborting!\n";
//echo "\n $json_rq \n";
//echo $resp['data']."\n".$resp['d']."\n";
while (true) {
echo ",";
sleep(60);
$resp = askhost($url, $json_rq, "", "", "", 90000, array("Content-Type: application/json"), true);
$res = json_decode($resp['data']);
if (FALSE !== strpos($res->status, "REQUEST_SUCCEEDED")) {
break;
}
// ok!
}
//die();
}
foreach ($res->Results->series as $result_series) {
if (!isset($result_series->data[0])) {
echo "!";
continue;
// no data
}
//echo $result_series."\n";
<?php
// kickstarter data extractor for capsidea
include 'k-inc.php';
$projects = load_prj_from_db();
$locations = load_loc_from_db();
$category = load_cat_from_db();
$creators = load_creators_from_db();
// https://www.kickstarter.com/projects/alradeck/caelum-sky-illustrated-1?format=json
$i = 0;
while (true) {
$res = askhost("https://www.kickstarter.com/discover/advanced?format=json&page={$i}", FALSE, "", "", "1", 60000, "", true);
$i++;
if (500 == $res['httpcode']) {
die('\\n some error 500\\n');
}
$js = json_decode($res['data']);
echo "\n{$i}: ";
//.$res['data']."\n";
if (!isset($js->projects[0]->id)) {
die("complete\n");
}
foreach ($js->projects as $prj) {
// init with empty values
$cat_pid = 0;
$creator_slug = "";
// get real values
$pid = (int) $prj->id;
$cat_id = (int) $prj->category->id;
@($loc_id = (int) $prj->location->id);
$creator_id = $prj->creator->id;
$report_fname = prepare_report();
echo "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">\n<html xmlns=\"http://www.w3.org/1999/xhtml\"><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\">\n<script type=\"text/javascript\" src=\"http://beta.capsidea.com/api.js\"></script>";
echo "<script>function doit(key) {CI.updateSource(key);CI.openSource(key);CI.closeApp();}</script>";
//echo "<script>CI.closeApp();</script>";
echo "<title>Success</title>\n<link rel=\"stylesheet\" type=\"text/css\" href=\"view.css\" media=\"all\"></head>\n<body id=\"main_body\"> ";
echo "please standby while Capsidea.com processing report<br>NB: This can take a while, so be patient<br>";
flush();
ob_flush();
flush();
ob_flush();
flush();
ob_flush();
flush();
ob_flush();
$name = date("YMdHi");
$host_reply = askhost($server_url . "&name={$name}" . $schemakey . $schemajson, array('file_contents' => '@' . $report_fname), "", "", "", 1800000, array("appid: {$capsidea_appid}", "sig: {$secret}"), true);
// defined in askhost.php
//echo " time ".$time3=((get_timer()-$stime)/1)." sec "; // $stime=get_timer();
$result = $host_reply["data"];
$error_log = "cube: {$schemakey}" . "secret: " . $secret . "<br>response:<pre>" . $host_reply["data"] . "</pre>" . "<br>connection debug:<pre>" . $host_reply["d"] . "</pre>";
if (500 == $host_reply["httpcode"]) {
echo "ERR: {$error_log}\n" . $host_reply["httpcode"];
log_fatal("error 500 from cps: \n {$error_log}");
}
// if 500
if (401 == $host_reply["httpcode"]) {
echo "ERR: unauthorized {$error_log}\n" . $host_reply["httpcode"];
log_fatal("error 401 from cps \n {$error_log}");
}
// if 500
if (200 == $host_reply["httpcode"]) {
// federalreserve data connector for capsidea
include_once 'usdrates-inc.php';
file_put_contents("/tmp/debug.log", date(DATE_ATOM) . " usdrates-dc " . print_r($_REQUEST, true) . " \n", FILE_APPEND);
$capsidea_appid = 2233;
$capsidea_client_secret = "put-your-data-here";
$cdata = get_capsidea_data($capsidea_client_secret);
$secret = $cdata["t"];
$schemakey = "";
if (isset($cdata["k"])) {
// check, if this is reinitialisation
$cps_key = $cdata["k"];
$schemakey = "&schemakey={$cps_key}";
}
//$my_data_dir="/tmp";
include_once 'askhost.php';
$host_reply = askhost($server_url . $schemakey . "&frequency=daily", array('file_contents' => '@' . "{$my_data_dir}/usrates.csv"), "", "", "", 80000, array("appid: {$capsidea_appid}", "sig: {$secret}"), true);
// defined in askhost.php
$httpcode = $host_reply["httpcode"];
$jsonres = json_decode($host_reply["data"], true);
$key = (int) $jsonres["Key"];
if (0 == $key) {
$key = $cdata["k"];
}
//construct dashboard json
$i = 0;
$sheets = "";
$datasources = "";
//$usedkeys=array();
//$ds_template=file_get_contents("cbr1-dst.json");
//$sheet_template=file_get_contents("cbr1-st.json");
// $jsondash="{ \"Sources\":{ myfulldatasources }, \"CalcMembers\":null, \"Sheets\":[ myfulldatasheets ],
// all ok, proceed to cps
// create json for DB
if (0 == $parent_key) {
$source_id = $ikey;
} else {
$source_id = $parent_key;
}
// get element codes
$url = $base_url . "?s=DataService&m=OpenDataSource";
$post_data = "[\"{$ikey}\$paypal\"]";
$res = askhost($url, $post_data, "", "", "", 180000, array("appid: {$capsidea_appid}", "sig: {$secret}", "Content-Type: application/json;charset=UTF-8"), true);
mylog($res['d'] . "| " . $post_data . " json: " . $res['data']);
//$secret1=sha1($capsidea_client_secret.$capsidea_permanent_access_token);
$url = $base_url . "?s=DataService&m=GetChildEls";
$post_data = "[\"{$ikey}\$case_reason\",\"\"]";
$res = askhost($url, $post_data, "", "", "", 180000, array("appid: {$capsidea_appid}", "sig: {$secret}", "Content-Type: application/json;charset=UTF-8"), true);
mylog($res['d'] . "| " . $post_data . " json: " . $res['data']);
$js = json_decode($res['data']);
//print_r($js);
$selected = "";
$i = 0;
foreach ($js as $this_item) {
if ($this_item->n == "") {
continue;
}
if ($i > 0) {
$selected = $selected . ",";
}
$selected = $selected . "\"" . $this_item->k . "\"";
$i++;
}
