protected function processUrls($status)
 {
     $dbh = $this->getDbh();
     if (!$this->_urlExpander) {
         $this->_urlExpander = new UrlExpander();
     }
     $urlExpander = $this->_urlExpander;
     $urls = $status['entities']['urls'];
     foreach ($urls as $url) {
         $expanded_url = $url['expanded_url'];
         $output_url = $urlExpander->expand($expanded_url);
         // sometimes "expanded url" returned by t.co is a bitly link, etc.
         $output_url = rtrim($output_url, '/');
         $first_seen = $status['created_at'];
         $first_seen = strtotime($first_seen);
         $first_seen = date('Y-m-d H:i:s', $first_seen);
         $first_tweet = $status['id_str'];
         $first_user = $status['user']['screen_name'];
         $first_user_id = $status['user']['id_str'];
         $weighted_count = Universe::getInfluence($first_user_id);
         try {
             $sql = "INSERT INTO openfuego_links (\n\t\t\t\t\turl,\n\t\t\t\t\tfirst_seen,\n\t\t\t\t\tfirst_tweet,\n\t\t\t\t\tfirst_user,\n\t\t\t\t\tfirst_user_id,\n\t\t\t\t\tweighted_count,\n\t\t\t\t\tcount,\n\t\t\t\t\tlast_seen\n\t\t\t\t)\n\t\t\t\tVALUES (\n\t\t\t\t\t:url,\n\t\t\t\t\t:first_seen,\n\t\t\t\t\t:first_tweet,\n\t\t\t\t\t:first_user,\n\t\t\t\t\t:first_user_id,\n\t\t\t\t\t:weighted_count,\n\t\t\t\t\t1,\n\t\t\t\t\t:first_seen\n\t\t\t\t)\n\t\t\t\tON DUPLICATE KEY UPDATE\n\t\t\t\tweighted_count = CASE WHEN\n\t\t\t\t\tfirst_tweet != VALUES(first_tweet) \n\t\t\t\t\tAND first_user = VALUES(first_user)\n\t\t\t\tTHEN\n\t\t\t\t\tweighted_count\n\t\t\t\tELSE\n\t\t\t\t\tweighted_count + VALUES(weighted_count)\n\t\t\t\tEND,\n\t\t\t\tcount = CASE WHEN\n\t\t\t\t\tfirst_tweet != VALUES(first_tweet) \n\t\t\t\t\tAND first_user = VALUES(first_user)\n\t\t\t\tTHEN\n\t\t\t\t\tcount\n\t\t\t\tELSE\n\t\t\t\t\tcount + 1\n\t\t\t\tEND,\n\t\t\t\tlast_seen = CASE WHEN\n\t\t\t\t\tfirst_tweet != VALUES(first_tweet)\n\t\t\t\t\tAND first_user = VALUES(first_user)\n\t\t\t\tTHEN\n\t\t\t\t\tlast_seen\n\t\t\t\tELSE\n\t\t\t\t\tVALUES(last_seen)\n\t\t\t\tEND;";
             $sth = $dbh->prepare($sql);
             $sth->bindParam('url', $output_url);
             $sth->bindParam('first_seen', $first_seen);
             $sth->bindParam('first_tweet', $first_tweet);
             $sth->bindParam('first_user', $first_user);
             $sth->bindParam('first_user_id', $first_user_id);
             $sth->bindParam('weighted_count', $weighted_count);
             $sth->execute();
         } catch (\PDOException $e) {
             echo 'PDO exception in ' . __FUNCTION__ . ', ' . date('Y-m-d H:i:s'), $e;
             continue;
             // on to the next url
         }
     }
 }
Exemple #2
0
    $error_message = "\n" . 'Do not run this script directly. Run fetch.php to start.' . "\n\n";
    die($error_message);
}
require_once __DIR__ . '/init.php';
register_shutdown_function(function () {
    Logger::fatal(__NAMESPACE__ . " collector was terminated.");
});
$twitter = new TwitterHandle();
$twitter->get("account/verify_credentials", array("include_entities" => 0, "skip_status" => 1));
if ($twitter->http_code !== 200) {
    $error_message = "Cannot continue. Your Twitter credentials appear to be invalid. Error code {$twitter->http_code}";
    Logger::info($error_message);
    die($error_message);
}
unset($twitter_handle);
$authorities = unserialize(\OpenFuego\AUTHORITIES);
$universe = new Universe();
/** The next line is commented out by default.
 * Uncomment it to repopulate the universe on each fetch. */
// $universe->populate($authorities, 1);
$citizens = $universe->getCitizens(1);
if (!$citizens) {
    $universe->populate($authorities, 1);
    $citizens = $universe->getCitizens(1);
}
$citizens = array_slice($citizens, 0, TWITTER_PREDICATE_LIMIT);
// Start streaming/collecting
$collector = new Collector(TWITTER_OAUTH_TOKEN, TWITTER_OAUTH_SECRET);
$collector->setFollow($citizens);
$collector->consume();
exit;