function process_json_file_timeline($filepath, $dbh) { global $tweets_processed, $tweets_failed, $tweets_success, $valid_timeline, $empty_timeline, $invalid_timeline, $populated_timeline, $total_timeline, $all_tweet_ids, $all_users, $bin_name; $tweetQueue = new TweetQueue(); $total_timeline++; ini_set('auto_detect_line_endings', true); $handle = @fopen($filepath, "r"); if ($handle) { while (($buffer = fgets($handle, 40960)) !== false) { $tweet = json_decode($buffer, true); //var_export($tweet); print "\n\n"; $buffer = ""; $t = new Tweet(); $t->fromJSON($tweet); if (!$t->isInBin($bin_name)) { $tweetQueue->push($t, $bin_name); if ($tweetQueue->length() > 100) { $tweetQueue->insertDB(); } $all_users[] = $t->from_user_id; $all_tweet_ids[] = $t->id; $tweets_processed++; } print "."; } if (!feof($handle)) { echo "Error: unexpected fgets() fail\n"; } fclose($handle); } if ($tweetQueue->length() > 0) { $tweetQueue->insertDB(); } }
function process_json_file_timeline($filepath, $dbh) { global $tweets_processed, $tweets_failed, $tweets_success, $valid_timeline, $empty_timeline, $invalid_timeline, $populated_timeline, $total_timeline, $all_tweet_ids, $all_users, $bin_name; $tweetQueue = new TweetQueue(); $total_timeline++; $filestr = file_get_contents($filepath); // sylvester stores multiple json exports in the same file, // in order to decode it we will need to split it into its respective individual exports $jsons = explode("}][{", $filestr); print count($jsons) . " jsons found\n"; foreach ($jsons as $json) { if (substr($json, 0, 2) != "[{") { $json = "[{" . $json; } if (substr($json, -2) != "}]") { $json = $json . "}]"; } $timeline = json_decode($json); if (is_array($timeline)) { $valid_timeline++; if (!empty($timeline)) { $populated_timeline++; } else { $empty_timeline++; } } else { $invalid_timeline++; } foreach ($timeline as $tweet) { $t = new Tweet(); $t->fromJSON($tweet); if (!$t->isInBin($bin_name)) { $tweetQueue->push($t, $bin_name); if ($tweetQueue->length() > 100) { $tweetQueue->insertDB(); } $all_users[] = $t->user->id; $all_tweet_ids[] = $t->id; $tweets_processed++; } } } if ($tweetQueue->length() > 0) { $tweetQueue->insertDB(); } }
function process_json_file_timeline($filepath, $dbh) { print $filepath . "\n"; global $tweets_processed, $tweets_failed, $tweets_success, $all_tweet_ids, $all_users, $bin_name; $tweetQueue = new TweetQueue(); ini_set('auto_detect_line_endings', true); $handle = @fopen($filepath, "r"); if ($handle) { while (($buffer = fgets($handle, 40960)) !== false) { $buffer = trim($buffer); if (empty($buffer)) { continue; } $tweet = json_decode($buffer); $buffer = ""; $t = Tweet::fromGnip($tweet); if ($t === false) { continue; } if (!$t->isInBin($bin_name)) { $all_users[] = $t->from_user_id; $all_tweet_ids[] = $t->id; $tweetQueue->push($t, $bin_name); if ($tweetQueue->length() > 100) { $tweetQueue->insertDB(); } $tweets_processed++; } print "."; } if (!feof($handle)) { echo "Error: unexpected fgets() fail\n"; } fclose($handle); } if ($tweetQueue->length() > 0) { $tweetQueue->insertDB(); } }
if (empty($keywords)) { die("keywords not set\n"); } if (dbserver_has_utf8mb4_support() == false) { die("DMI-TCAT requires at least MySQL version 5.5.3 - please upgrade your server"); } $querybin_id = queryManagerBinExists($bin_name, $cronjob); $current_key = 0; $tweetQueue = new TweetQueue(); // ----- connection ----- $dbh = pdo_connect(); create_bin($bin_name, $dbh); $ratefree = 0; queryManagerCreateBinFromExistingTables($bin_name, $querybin_id, $type, explode("OR", $keywords)); search($keywords); if ($tweetQueue->length() > 0) { $tweetQueue->insertDB(); } function search($keywords, $max_id = null) { global $twitter_keys, $current_key, $ratefree, $bin_name, $dbh, $tweetQueue; $ratefree--; if ($ratefree < 1 || $ratefree % 10 == 0) { $keyinfo = getRESTKey($current_key, 'search', 'tweets'); $current_key = $keyinfo['key']; $ratefree = $keyinfo['remaining']; } $tmhOAuth = new tmhOAuth(array('consumer_key' => $twitter_keys[$current_key]['twitter_consumer_key'], 'consumer_secret' => $twitter_keys[$current_key]['twitter_consumer_secret'], 'token' => $twitter_keys[$current_key]['twitter_user_token'], 'secret' => $twitter_keys[$current_key]['twitter_user_secret'])); $params = array('q' => $keywords, 'count' => 100); if (isset($max_id)) { $params['max_id'] = $max_id;