protected function processUrls($status)
{
$dbh = $this->getDbh();
if (!$this->_urlExpander) {
$this->_urlExpander = new UrlExpander();
}
$urlExpander = $this->_urlExpander;
$urls = $status['entities']['urls'];
foreach ($urls as $url) {
$expanded_url = $url['expanded_url'];
$output_url = $urlExpander->expand($expanded_url);
// sometimes "expanded url" returned by t.co is a bitly link, etc.
$output_url = rtrim($output_url, '/');
$first_seen = $status['created_at'];
$first_seen = strtotime($first_seen);
$first_seen = date('Y-m-d H:i:s', $first_seen);
$first_tweet = $status['id_str'];
$first_user = $status['user']['screen_name'];
$first_user_id = $status['user']['id_str'];
$weighted_count = Universe::getInfluence($first_user_id);
try {
$sql = "INSERT INTO openfuego_links (\n\t\t\t\t\turl,\n\t\t\t\t\tfirst_seen,\n\t\t\t\t\tfirst_tweet,\n\t\t\t\t\tfirst_user,\n\t\t\t\t\tfirst_user_id,\n\t\t\t\t\tweighted_count,\n\t\t\t\t\tcount,\n\t\t\t\t\tlast_seen\n\t\t\t\t)\n\t\t\t\tVALUES (\n\t\t\t\t\t:url,\n\t\t\t\t\t:first_seen,\n\t\t\t\t\t:first_tweet,\n\t\t\t\t\t:first_user,\n\t\t\t\t\t:first_user_id,\n\t\t\t\t\t:weighted_count,\n\t\t\t\t\t1,\n\t\t\t\t\t:first_seen\n\t\t\t\t)\n\t\t\t\tON DUPLICATE KEY UPDATE\n\t\t\t\tweighted_count = CASE WHEN\n\t\t\t\t\tfirst_tweet != VALUES(first_tweet) \n\t\t\t\t\tAND first_user = VALUES(first_user)\n\t\t\t\tTHEN\n\t\t\t\t\tweighted_count\n\t\t\t\tELSE\n\t\t\t\t\tweighted_count + VALUES(weighted_count)\n\t\t\t\tEND,\n\t\t\t\tcount = CASE WHEN\n\t\t\t\t\tfirst_tweet != VALUES(first_tweet) \n\t\t\t\t\tAND first_user = VALUES(first_user)\n\t\t\t\tTHEN\n\t\t\t\t\tcount\n\t\t\t\tELSE\n\t\t\t\t\tcount + 1\n\t\t\t\tEND,\n\t\t\t\tlast_seen = CASE WHEN\n\t\t\t\t\tfirst_tweet != VALUES(first_tweet)\n\t\t\t\t\tAND first_user = VALUES(first_user)\n\t\t\t\tTHEN\n\t\t\t\t\tlast_seen\n\t\t\t\tELSE\n\t\t\t\t\tVALUES(last_seen)\n\t\t\t\tEND;";
$sth = $dbh->prepare($sql);
$sth->bindParam('url', $output_url);
$sth->bindParam('first_seen', $first_seen);
$sth->bindParam('first_tweet', $first_tweet);
$sth->bindParam('first_user', $first_user);
$sth->bindParam('first_user_id', $first_user_id);
$sth->bindParam('weighted_count', $weighted_count);
$sth->execute();
} catch (\PDOException $e) {
echo 'PDO exception in ' . __FUNCTION__ . ', ' . date('Y-m-d H:i:s'), $e;
continue;
// on to the next url
}
}
}
$error_message = "\n" . 'Do not run this script directly. Run fetch.php to start.' . "\n\n";
die($error_message);
}
require_once __DIR__ . '/init.php';
register_shutdown_function(function () {
Logger::fatal(__NAMESPACE__ . " collector was terminated.");
});
$twitter = new TwitterHandle();
$twitter->get("account/verify_credentials", array("include_entities" => 0, "skip_status" => 1));
if ($twitter->http_code !== 200) {
$error_message = "Cannot continue. Your Twitter credentials appear to be invalid. Error code {$twitter->http_code}";
Logger::info($error_message);
die($error_message);
}
unset($twitter_handle);
$authorities = unserialize(\OpenFuego\AUTHORITIES);
$universe = new Universe();
/** The next line is commented out by default.
* Uncomment it to repopulate the universe on each fetch. */
// $universe->populate($authorities, 1);
$citizens = $universe->getCitizens(1);
if (!$citizens) {
$universe->populate($authorities, 1);
$citizens = $universe->getCitizens(1);
}
$citizens = array_slice($citizens, 0, TWITTER_PREDICATE_LIMIT);
// Start streaming/collecting
$collector = new Collector(TWITTER_OAUTH_TOKEN, TWITTER_OAUTH_SECRET);
$collector->setFollow($citizens);
$collector->consume();
exit;