function process_json_file_timeline($filepath, $dbh)
{
global $tweets_processed, $tweets_failed, $tweets_success, $valid_timeline, $empty_timeline, $invalid_timeline, $populated_timeline, $total_timeline, $all_tweet_ids, $all_users, $bin_name;
$tweetQueue = new TweetQueue();
$total_timeline++;
ini_set('auto_detect_line_endings', true);
$handle = @fopen($filepath, "r");
if ($handle) {
while (($buffer = fgets($handle, 40960)) !== false) {
$tweet = json_decode($buffer, true);
//var_export($tweet); print "\n\n";
$buffer = "";
$t = new Tweet();
$t->fromJSON($tweet);
if (!$t->isInBin($bin_name)) {
$tweetQueue->push($t, $bin_name);
if ($tweetQueue->length() > 100) {
$tweetQueue->insertDB();
}
$all_users[] = $t->from_user_id;
$all_tweet_ids[] = $t->id;
$tweets_processed++;
}
print ".";
}
if (!feof($handle)) {
echo "Error: unexpected fgets() fail\n";
}
fclose($handle);
}
if ($tweetQueue->length() > 0) {
$tweetQueue->insertDB();
}
}
function process_json_file_timeline($filepath, $dbh)
{
global $tweets_processed, $tweets_failed, $tweets_success, $valid_timeline, $empty_timeline, $invalid_timeline, $populated_timeline, $total_timeline, $all_tweet_ids, $all_users, $bin_name;
$tweetQueue = new TweetQueue();
$total_timeline++;
$filestr = file_get_contents($filepath);
// sylvester stores multiple json exports in the same file,
// in order to decode it we will need to split it into its respective individual exports
$jsons = explode("}][{", $filestr);
print count($jsons) . " jsons found\n";
foreach ($jsons as $json) {
if (substr($json, 0, 2) != "[{") {
$json = "[{" . $json;
}
if (substr($json, -2) != "}]") {
$json = $json . "}]";
}
$timeline = json_decode($json);
if (is_array($timeline)) {
$valid_timeline++;
if (!empty($timeline)) {
$populated_timeline++;
} else {
$empty_timeline++;
}
} else {
$invalid_timeline++;
}
foreach ($timeline as $tweet) {
$t = new Tweet();
$t->fromJSON($tweet);
if (!$t->isInBin($bin_name)) {
$tweetQueue->push($t, $bin_name);
if ($tweetQueue->length() > 100) {
$tweetQueue->insertDB();
}
$all_users[] = $t->user->id;
$all_tweet_ids[] = $t->id;
$tweets_processed++;
}
}
}
if ($tweetQueue->length() > 0) {
$tweetQueue->insertDB();
}
}
function process_json_file_timeline($filepath, $dbh)
{
print $filepath . "\n";
global $tweets_processed, $tweets_failed, $tweets_success, $all_tweet_ids, $all_users, $bin_name;
$tweetQueue = new TweetQueue();
ini_set('auto_detect_line_endings', true);
$handle = @fopen($filepath, "r");
if ($handle) {
while (($buffer = fgets($handle, 40960)) !== false) {
$buffer = trim($buffer);
if (empty($buffer)) {
continue;
}
$tweet = json_decode($buffer);
$buffer = "";
$t = Tweet::fromGnip($tweet);
if ($t === false) {
continue;
}
if (!$t->isInBin($bin_name)) {
$all_users[] = $t->from_user_id;
$all_tweet_ids[] = $t->id;
$tweetQueue->push($t, $bin_name);
if ($tweetQueue->length() > 100) {
$tweetQueue->insertDB();
}
$tweets_processed++;
}
print ".";
}
if (!feof($handle)) {
echo "Error: unexpected fgets() fail\n";
}
fclose($handle);
}
if ($tweetQueue->length() > 0) {
$tweetQueue->insertDB();
}
}
if (empty($keywords)) {
die("keywords not set\n");
}
if (dbserver_has_utf8mb4_support() == false) {
die("DMI-TCAT requires at least MySQL version 5.5.3 - please upgrade your server");
}
$querybin_id = queryManagerBinExists($bin_name, $cronjob);
$current_key = 0;
$tweetQueue = new TweetQueue();
// ----- connection -----
$dbh = pdo_connect();
create_bin($bin_name, $dbh);
$ratefree = 0;
queryManagerCreateBinFromExistingTables($bin_name, $querybin_id, $type, explode("OR", $keywords));
search($keywords);
if ($tweetQueue->length() > 0) {
$tweetQueue->insertDB();
}
function search($keywords, $max_id = null)
{
global $twitter_keys, $current_key, $ratefree, $bin_name, $dbh, $tweetQueue;
$ratefree--;
if ($ratefree < 1 || $ratefree % 10 == 0) {
$keyinfo = getRESTKey($current_key, 'search', 'tweets');
$current_key = $keyinfo['key'];
$ratefree = $keyinfo['remaining'];
}
$tmhOAuth = new tmhOAuth(array('consumer_key' => $twitter_keys[$current_key]['twitter_consumer_key'], 'consumer_secret' => $twitter_keys[$current_key]['twitter_consumer_secret'], 'token' => $twitter_keys[$current_key]['twitter_user_token'], 'secret' => $twitter_keys[$current_key]['twitter_user_secret']));
$params = array('q' => $keywords, 'count' => 100);
if (isset($max_id)) {
$params['max_id'] = $max_id;