예제 #1
0
function tracker_streamCallback($data, $length, $metrics)
{
    global $capturebucket, $lastinsert;
    $now = time();
    $data = json_decode($data, true);
    if ($data) {
        if (array_key_exists('disconnect', $data)) {
            $discerror = implode(",", $data["disconnect"]);
            logit(CAPTURE . ".error.log", "connection dropped or timed out - error " . $discerror);
            logit(CAPTURE . ".error.log", "(debug) dump of result data on disconnect" . var_export($data, true));
            return;
            // exit will take place in the previous function
        }
        if (array_key_exists('warning', $data)) {
            // Twitter sent us a warning
            $code = $data['warning']['code'];
            $message = $data['warning']['message'];
            if ($code === 'FALLING_BEHIND') {
                $full = $data['warning']['percent_full'];
                logit(CAPTURE . ".error.log", "twitter api warning received: ({$code}) {$message} [percentage full {$full}]");
            } else {
                logit(CAPTURE . ".error.log", "twitter api warning received: ({$code}) {$message}");
            }
        }
        // handle rate limiting
        if (array_key_exists('limit', $data)) {
            global $ratelimit, $exceeding, $ex_start;
            if (isset($data['limit'][CAPTURE])) {
                $current = $data['limit'][CAPTURE];
                if ($current > $ratelimit) {
                    // currently exceeding rate limit
                    if (!$exceeding) {
                        // new disturbance!
                        $ex_start = time();
                        ratelimit_report_problem();
                        // logit(CAPTURE . ".error.log", "you have hit a rate limit. consider reducing your query bin sizes");
                    }
                    $ratelimit = $current;
                    $exceeding = 1;
                    if (time() > $ex_start + RATELIMIT_SILENCE * 6) {
                        // every half an hour (or: heartbeat x 6), record, but keep the exceeding flag set
                        ratelimit_record($ratelimit, $ex_start);
                        $ex_start = time();
                    }
                } elseif ($exceeding && time() < $ex_start + RATELIMIT_SILENCE) {
                    // we are now no longer exceeding the rate limit
                    // to avoid flip-flop we only reset our values after the minimal heartbeat has passed
                    // store rate limit disturbance information in the database
                    ratelimit_record($ratelimit, $ex_start);
                    $ex_start = 0;
                    $exceeding = 0;
                }
            }
            unset($data['limit']);
        }
        if (empty($data)) {
            return;
        }
        // sometimes we only get rate limit info
        $capturebucket[] = $data;
        if (count($capturebucket) == 100 || $now > $lastinsert + 5) {
            processtweets($capturebucket);
            $lastinsert = time();
            $capturebucket = array();
        }
    }
}
예제 #2
0
function tracker_streamCallback($data, $length, $metrics)
{
    global $capturebucket, $lastinsert;
    $now = time();
    $data = json_decode($data, true);
    if ($data) {
        if (array_key_exists('disconnect', $data)) {
            $discerror = implode(",", $data["disconnect"]);
            logit(CAPTURE . ".error.log", "connection dropped or timed out - error " . $discerror);
            logit(CAPTURE . ".error.log", "(debug) dump of result data on disconnect" . var_export($data, true));
            return;
            // exit will take place in the previous function
        }
        if (array_key_exists('warning', $data)) {
            // Twitter sent us a warning
            $code = $data['warning']['code'];
            $message = $data['warning']['message'];
            if ($code === 'FALLING_BEHIND') {
                $full = $data['warning']['percent_full'];
                logit(CAPTURE . ".error.log", "twitter api warning received: ({$code}) {$message} [percentage full {$full}]");
            } else {
                logit(CAPTURE . ".error.log", "twitter api warning received: ({$code}) {$message}");
            }
        }
        // handle rate limiting at intervals of a single minute
        global $rl_current_record, $rl_registering_minute;
        global $tracker_started_at;
        $current = 0;
        $current_minute = get_current_minute();
        // we keep a a counter of the nr. of tweets rate limited and reset it at intervals of one minute
        // read possible rate limit information from Twitter
        if (array_key_exists('limit', $data) && isset($data['limit']['track'])) {
            $current = $data['limit'][CAPTURE];
            // we have a new rate limit, grow the record
            $rl_current_record += $current;
        } else {
            // when no new rate limits occur, sustain our current record
            $current = $rl_current_record;
        }
        if ($rl_registering_minute != $current_minute) {
            // the current minute is no longer our registering minute; we have to record our ratelimit information in the database
            if ($current_minute == 0 && $rl_registering_minute < 59 || $current_minute > 0 && $current_minute < $rl_registering_minute || $current_minute > $rl_registering_minute + 1) {
                // there was a more than 1 minute silence (i.e. a response from Curl took longer than our 1 minute interval, thus we need to fill in zeros backwards in time)
                $tracker_running = round((time() - $tracker_started_at) / 60);
                ratelimit_holefiller($tracker_running);
            }
            $rl_registering_minute = $current_minute;
            // we now have rate limit information for the last minute
            ratelimit_record($rl_current_record);
            if ($rl_current_record > 0) {
                ratelimit_report_problem();
                $rl_current_record = 0;
            }
        }
        if (array_key_exists('limit', $data)) {
            unset($data['limit']);
        }
        if (empty($data)) {
            return;
        }
        // sometimes we only get rate limit info
        $capturebucket[] = $data;
        if (count($capturebucket) == 100 || $now > $lastinsert + 5) {
            processtweets($capturebucket);
            $lastinsert = time();
            $capturebucket = array();
        }
    }
}