function search($idlist) { global $twitter_keys, $current_key, $all_users, $all_tweet_ids, $bin_name, $dbh, $tweetQueue; $keyinfo = getRESTKey(0); $current_key = $keyinfo['key']; $ratefree = $keyinfo['remaining']; print "current key {$current_key} ratefree {$ratefree}\n"; $tmhOAuth = new tmhOAuth(array('consumer_key' => $twitter_keys[$current_key]['twitter_consumer_key'], 'consumer_secret' => $twitter_keys[$current_key]['twitter_consumer_secret'], 'token' => $twitter_keys[$current_key]['twitter_user_token'], 'secret' => $twitter_keys[$current_key]['twitter_user_secret'])); // by hundred for ($i = 0; $i < sizeof($idlist); $i += 100) { if ($ratefree <= 0 || $ratefree % 10 == 0) { $keyinfo = getRESTKey($current_key); $current_key = $keyinfo['key']; $ratefree = $keyinfo['remaining']; $tmhOAuth = new tmhOAuth(array('consumer_key' => $twitter_keys[$current_key]['twitter_consumer_key'], 'consumer_secret' => $twitter_keys[$current_key]['twitter_consumer_secret'], 'token' => $twitter_keys[$current_key]['twitter_user_token'], 'secret' => $twitter_keys[$current_key]['twitter_user_secret'])); } $q = $idlist[$i]; $n = $i + 1; while ($n < $i + 100) { if (!isset($idlist[$n])) { break; } $q .= "," . $idlist[$n]; $n++; } $params = array('id' => $q); $code = $tmhOAuth->user_request(array('method' => 'GET', 'url' => $tmhOAuth->url('1.1/statuses/lookup'), 'params' => $params)); $ratefree--; if ($tmhOAuth->response['code'] == 200) { $data = json_decode($tmhOAuth->response['response'], true); if (is_array($data) && empty($data)) { // all tweets in set are deleted continue; } $tweets = $data; $tweet_ids = array(); foreach ($tweets as $tweet) { $t = new Tweet(); $t->fromJSON($tweet); if (!$t->isInBin($bin_name)) { $all_users[] = $t->from_user_id; $all_tweet_ids[] = $t->id; $tweet_ids[] = $t->id; $tweetQueue->push($t, $bin_name); } print "."; } sleep(1); } else { echo "Failure with code " . $tmhOAuth->response['response']['code'] . "\n"; var_dump($tmhOAuth->response['response']['info']); var_dump($tmhOAuth->response['response']['error']); var_dump($tmhOAuth->response['response']['errno']); die; } $tweetQueue->insertDB(); } }
function search($keywords, $max_id = null) { global $twitter_keys, $current_key, $ratefree, $bin_name, $dbh, $tweetQueue; $ratefree--; if ($ratefree < 1 || $ratefree % 10 == 0) { $keyinfo = getRESTKey($current_key, 'search', 'tweets'); $current_key = $keyinfo['key']; $ratefree = $keyinfo['remaining']; } $tmhOAuth = new tmhOAuth(array('consumer_key' => $twitter_keys[$current_key]['twitter_consumer_key'], 'consumer_secret' => $twitter_keys[$current_key]['twitter_consumer_secret'], 'token' => $twitter_keys[$current_key]['twitter_user_token'], 'secret' => $twitter_keys[$current_key]['twitter_user_secret'])); $params = array('q' => $keywords, 'count' => 100); if (isset($max_id)) { $params['max_id'] = $max_id; } $code = $tmhOAuth->user_request(array('method' => 'GET', 'url' => $tmhOAuth->url('1.1/search/tweets'), 'params' => $params)); if ($tmhOAuth->response['code'] == 200) { $data = json_decode($tmhOAuth->response['response'], true); $tweets = $data['statuses']; $tweet_ids = array(); foreach ($tweets as $tweet) { $t = new Tweet(); $t->fromJSON($tweet); $tweet_ids[] = $t->id; if (!$t->isInBin($bin_name)) { $tweetQueue->push($t, $bin_name); if ($tweetQueue->length() > 100) { $tweetQueue->insertDB(); } print "."; } } if (!empty($tweet_ids)) { print "\n"; if (count($tweet_ids) <= 1) { print "no more tweets found\n\n"; return false; } $max_id = min($tweet_ids); print "max id: " . $max_id . "\n"; } else { print "0 tweets found\n\n"; return false; } sleep(1); search($keywords, $max_id); } else { echo $tmhOAuth->response['response'] . "\n"; if ($tmhOAuth->response['response']['errors']['code'] == 130) { // over capacity sleep(1); search($keywords, $max_id); } } }
function get_friends($screen_name, $cursor = -1) { print "getting friends of {$screen_name}\n"; global $twitter_keys, $current_key, $ratefree, $bin_name, $dbh; $ratefree--; if ($ratefree < 1 || $ratefree % 10 == 0) { $keyinfo = getRESTKey($current_key, 'friends', 'list'); $current_key = $keyinfo['key']; $ratefree = $keyinfo['remaining']; } $tmhOAuth = new tmhOAuth(array('consumer_key' => $twitter_keys[$current_key]['twitter_consumer_key'], 'consumer_secret' => $twitter_keys[$current_key]['twitter_consumer_secret'], 'token' => $twitter_keys[$current_key]['twitter_user_token'], 'secret' => $twitter_keys[$current_key]['twitter_user_secret'])); $params = array('cursor' => $cursor, 'skip_status' => true, 'count' => 200, 'include_user_entities' => false, 'screen_name' => $screen_name); $tmhOAuth->user_request(array('method' => 'GET', 'url' => $tmhOAuth->url('1.1/friends/list'), 'params' => $params)); if ($tmhOAuth->response['code'] == 200) { $data = json_decode($tmhOAuth->response['response'], true); $observed_at = strftime("%Y-%m-%d %H-%M-%S", date('U')); $friends = $data['users']; $cursor = $data['next_cursor']; echo count($friends) . " users found\n"; // store in db $tr = new TwitterRelations($screen_name, $friends, "friend", $observed_at); $tr->save($dbh, $bin_name); // continue if there are still things to do if (!empty($cursor) && $cursor != -1) { sleep(1); get_friends($screen_name, $cursor); } } else { $error_code = json_decode($tmhOAuth->response['response'])->errors[0]->code; if ($error_code == 130) { print "Twitter is over capacity, sleeping 5 seconds before retry\n"; sleep(5); get_friends($screen_name, $cursor); } elseif ($error_code == 88) { print "API key rate limit exceeded, sleeping 60 seconds before retry\n"; sleep(60); get_friends($screen_name, $cursor); } else { echo "\nAPI error: " . $tmhOAuth->response['response'] . "\n"; } } }
function get_timeline($user_id, $type, $max_id = null) { print "doing {$user_id}\n"; global $twitter_keys, $current_key, $ratefree, $looped, $bin_name, $dbh, $tweetQueue; $ratefree--; if ($ratefree < 1 || $ratefree % 10 == 0) { $keyinfo = getRESTKey($current_key, 'statuses', 'user_timeline'); $current_key = $keyinfo['key']; $ratefree = $keyinfo['remaining']; } $tmhOAuth = new tmhOAuth(array('consumer_key' => $twitter_keys[$current_key]['twitter_consumer_key'], 'consumer_secret' => $twitter_keys[$current_key]['twitter_consumer_secret'], 'token' => $twitter_keys[$current_key]['twitter_user_token'], 'secret' => $twitter_keys[$current_key]['twitter_user_secret'])); $params = array('count' => 200, 'trim_user' => false, 'exclude_replies' => false, 'contributor_details' => true, 'include_rts' => 1); if ($type == "user_id") { $params['user_id'] = $user_id; } else { $params['screen_name'] = $user_id; } if (isset($max_id)) { $params['max_id'] = $max_id; } $tmhOAuth->user_request(array('method' => 'GET', 'url' => $tmhOAuth->url('1.1/statuses/user_timeline'), 'params' => $params)); //var_export($params); print "\n"; if ($tmhOAuth->response['code'] == 200) { $tweets = json_decode($tmhOAuth->response['response'], true); // store in db $tweet_ids = array(); foreach ($tweets as $tweet) { $t = new Tweet(); $t->fromJSON($tweet); $tweet_ids[] = $t->id; if (!$t->isInBin($bin_name)) { $tweetQueue->push($t, $bin_name); print "."; if ($tweetQueue->length() > 100) { $tweetQueue->insertDB(); } } } if (!empty($tweet_ids)) { print "\n"; if (count($tweet_ids) <= 1) { print "no more tweets found\n\n"; return false; } $max_id = min($tweet_ids); print "max id: " . $max_id . "\n"; } else { print "0 tweets found\n\n"; return false; } sleep(1); get_timeline($user_id, $type, $max_id); } else { $error_code = json_decode($tmhOAuth->response['response'])->errors[0]->code; if ($error_code == 130) { print "Twitter is over capacity, sleeping 5 seconds before retry\n"; sleep(5); get_timeline($user_id, $type, $max_id); } elseif ($error_code == 88) { print "API key rate limit exceeded, sleeping 60 seconds before retry\n"; sleep(60); get_timeline($user_id, $type, $max_id); } else { echo "\nAPI error: " . $tmhOAuth->response['response'] . "\n"; } } }
function search($idlist) { global $twitter_keys, $current_key, $all_users, $all_tweet_ids, $bin_name, $dbh, $tweetQueue; $keyinfo = getRESTKey(0); $current_key = $keyinfo['key']; $ratefree = $keyinfo['remaining']; print "\ncurrent key {$current_key} ratefree {$ratefree}\n"; $tmhOAuth = new tmhOAuth(array('consumer_key' => $twitter_keys[$current_key]['twitter_consumer_key'], 'consumer_secret' => $twitter_keys[$current_key]['twitter_consumer_secret'], 'token' => $twitter_keys[$current_key]['twitter_user_token'], 'secret' => $twitter_keys[$current_key]['twitter_user_secret'])); // by hundred for ($i = 0; $i < sizeof($idlist); $i += 100) { if ($ratefree <= 0 || $ratefree % 10 == 0) { print "\n"; $keyinfo = getRESTKey($current_key); $current_key = $keyinfo['key']; $ratefree = $keyinfo['remaining']; $tmhOAuth = new tmhOAuth(array('consumer_key' => $twitter_keys[$current_key]['twitter_consumer_key'], 'consumer_secret' => $twitter_keys[$current_key]['twitter_consumer_secret'], 'token' => $twitter_keys[$current_key]['twitter_user_token'], 'secret' => $twitter_keys[$current_key]['twitter_user_secret'])); } $q = $idlist[$i]; $n = $i + 1; while ($n < $i + 100) { if (!isset($idlist[$n])) { break; } $q .= "," . $idlist[$n]; $n++; } $params = array('id' => $q); $code = $tmhOAuth->user_request(array('method' => 'GET', 'url' => $tmhOAuth->url('1.1/statuses/lookup'), 'params' => $params)); $ratefree--; $reset_connection = false; if ($tmhOAuth->response['code'] == 200) { $data = json_decode($tmhOAuth->response['response'], true); if (is_array($data) && empty($data)) { // all tweets in set are deleted continue; } $tweets = $data; $tweet_ids = array(); foreach ($tweets as $tweet) { $t = new Tweet(); $t->fromJSON($tweet); if (!$t->isInBin($bin_name)) { $all_users[] = $t->from_user_id; $all_tweet_ids[] = $t->id; $tweet_ids[] = $t->id; $tweetQueue->push($t, $bin_name); } print "."; } sleep(1); $retries = 0; // reset retry counter on success } else { if ($retries < 4 && $tmhOAuth->response['code'] == 503) { /* this indicates problems on the Twitter side, such as overcapacity. we slow down and retry the connection */ print "!"; sleep(7); $i--; // rewind $retries++; $reset_connection = true; } else { if ($retries < 4) { print "\n"; print "Failure with code " . $tmhOAuth->response['response']['code'] . "\n"; var_dump($tmhOAuth->response['response']['info']); var_dump($tmhOAuth->response['response']['error']); var_dump($tmhOAuth->response['response']['errno']); print "The above error may not be permanent. We will sleep and retry the request.\n"; sleep(7); $i--; // rewind $retries++; $reset_connection = true; } else { print "\n"; print "Permanent error when querying the Twitter API. Please investigate the error output. Now stopping.\n"; exit(1); } } } if ($reset_connection) { $tmhOAuth = new tmhOAuth(array('consumer_key' => $twitter_keys[$current_key]['twitter_consumer_key'], 'consumer_secret' => $twitter_keys[$current_key]['twitter_consumer_secret'], 'token' => $twitter_keys[$current_key]['twitter_user_token'], 'secret' => $twitter_keys[$current_key]['twitter_user_secret'])); $reset_connection = false; } else { $tweetQueue->insertDB(); } } }