function tracker_streamCallback($data, $length, $metrics)
{
global $capturebucket, $lastinsert;
$now = time();
$data = json_decode($data, true);
if ($data) {
if (array_key_exists('disconnect', $data)) {
$discerror = implode(",", $data["disconnect"]);
logit(CAPTURE . ".error.log", "connection dropped or timed out - error " . $discerror);
logit(CAPTURE . ".error.log", "(debug) dump of result data on disconnect" . var_export($data, true));
return;
// exit will take place in the previous function
}
if (array_key_exists('warning', $data)) {
// Twitter sent us a warning
$code = $data['warning']['code'];
$message = $data['warning']['message'];
if ($code === 'FALLING_BEHIND') {
$full = $data['warning']['percent_full'];
logit(CAPTURE . ".error.log", "twitter api warning received: ({$code}) {$message} [percentage full {$full}]");
} else {
logit(CAPTURE . ".error.log", "twitter api warning received: ({$code}) {$message}");
}
}
// handle rate limiting
if (array_key_exists('limit', $data)) {
global $ratelimit, $exceeding, $ex_start;
if (isset($data['limit'][CAPTURE])) {
$current = $data['limit'][CAPTURE];
if ($current > $ratelimit) {
// currently exceeding rate limit
if (!$exceeding) {
// new disturbance!
$ex_start = time();
ratelimit_report_problem();
// logit(CAPTURE . ".error.log", "you have hit a rate limit. consider reducing your query bin sizes");
}
$ratelimit = $current;
$exceeding = 1;
if (time() > $ex_start + RATELIMIT_SILENCE * 6) {
// every half an hour (or: heartbeat x 6), record, but keep the exceeding flag set
ratelimit_record($ratelimit, $ex_start);
$ex_start = time();
}
} elseif ($exceeding && time() < $ex_start + RATELIMIT_SILENCE) {
// we are now no longer exceeding the rate limit
// to avoid flip-flop we only reset our values after the minimal heartbeat has passed
// store rate limit disturbance information in the database
ratelimit_record($ratelimit, $ex_start);
$ex_start = 0;
$exceeding = 0;
}
}
unset($data['limit']);
}
if (empty($data)) {
return;
}
// sometimes we only get rate limit info
$capturebucket[] = $data;
if (count($capturebucket) == 100 || $now > $lastinsert + 5) {
processtweets($capturebucket);
$lastinsert = time();
$capturebucket = array();
}
}
}
function tracker_streamCallback($data, $length, $metrics)
{
global $capturebucket, $lastinsert;
$now = time();
$data = json_decode($data, true);
if ($data) {
if (array_key_exists('disconnect', $data)) {
$discerror = implode(",", $data["disconnect"]);
logit(CAPTURE . ".error.log", "connection dropped or timed out - error " . $discerror);
logit(CAPTURE . ".error.log", "(debug) dump of result data on disconnect" . var_export($data, true));
return;
// exit will take place in the previous function
}
if (array_key_exists('warning', $data)) {
// Twitter sent us a warning
$code = $data['warning']['code'];
$message = $data['warning']['message'];
if ($code === 'FALLING_BEHIND') {
$full = $data['warning']['percent_full'];
logit(CAPTURE . ".error.log", "twitter api warning received: ({$code}) {$message} [percentage full {$full}]");
} else {
logit(CAPTURE . ".error.log", "twitter api warning received: ({$code}) {$message}");
}
}
// handle rate limiting at intervals of a single minute
global $rl_current_record, $rl_registering_minute;
global $tracker_started_at;
$current = 0;
$current_minute = get_current_minute();
// we keep a a counter of the nr. of tweets rate limited and reset it at intervals of one minute
// read possible rate limit information from Twitter
if (array_key_exists('limit', $data) && isset($data['limit']['track'])) {
$current = $data['limit'][CAPTURE];
// we have a new rate limit, grow the record
$rl_current_record += $current;
} else {
// when no new rate limits occur, sustain our current record
$current = $rl_current_record;
}
if ($rl_registering_minute != $current_minute) {
// the current minute is no longer our registering minute; we have to record our ratelimit information in the database
if ($current_minute == 0 && $rl_registering_minute < 59 || $current_minute > 0 && $current_minute < $rl_registering_minute || $current_minute > $rl_registering_minute + 1) {
// there was a more than 1 minute silence (i.e. a response from Curl took longer than our 1 minute interval, thus we need to fill in zeros backwards in time)
$tracker_running = round((time() - $tracker_started_at) / 60);
ratelimit_holefiller($tracker_running);
}
$rl_registering_minute = $current_minute;
// we now have rate limit information for the last minute
ratelimit_record($rl_current_record);
if ($rl_current_record > 0) {
ratelimit_report_problem();
$rl_current_record = 0;
}
}
if (array_key_exists('limit', $data)) {
unset($data['limit']);
}
if (empty($data)) {
return;
}
// sometimes we only get rate limit info
$capturebucket[] = $data;
if (count($capturebucket) == 100 || $now > $lastinsert + 5) {
processtweets($capturebucket);
$lastinsert = time();
$capturebucket = array();
}
}
}
