public function consume($filter)
{
parent::consume($filter);
$query = $filter->query;
$url = 'https://www.googleapis.com/plus/v1/activities?query=' . urlencode($query) . '&key=' . urlencode(GOOGLE_API_KEY) . '&orderBy=recent&maxResults=20';
$page_token = null;
for ($pageNo = 0; $pageNo < $this->maxPages; $pageNo++) {
// Set current url
$current_url = $url;
if (!is_null($page_token)) {
$current_url .= '&pageToken=' . urlencode($page_token);
}
// Fetch data
$data = json_decode(file_get_contents($current_url));
foreach ($data->items as $item) {
$interaction = array('interaction' => array('schema' => array('version' => 3), 'author' => array('username' => $item->actor->displayName, 'name' => $item->actor->displayName, 'id' => $item->actor->id, 'avatar' => $item->actor->image->url, 'link' => $item->actor->url), 'type' => 'googleplus', 'created_at' => new \MongoDate(strtotime($item->published)), 'content' => $item->object->content, 'id' => $item->id, 'link' => $item->url), 'googleplus' => $item->object, 'internal' => array('filter_id' => (int) $filter->id));
\Consumer\Interaction::insert($interaction);
}
if (isset($data->nextPageToken)) {
$page_token = $data->nextPageToken;
} else {
break;
}
}
sleep(rand(40, 60));
// Slow things down so we don't hit API limits
}
public function consume($filter)
{
parent::consume($filter);
$parameters = array('q' => str_replace(PHP_EOL, ' ', $filter->query_simple), 'type' => 'post', 'access_token' => FACEBOOK_APP_ACCESS_TOKEN);
$url = 'https://graph.facebook.com/search?' . http_build_query($parameters);
$content = file_get_contents($url);
$response = json_decode($content);
foreach ($response->data as $status) {
if (!empty($status->message)) {
$interaction = array('interaction' => array('schema' => array('version' => 3), 'author' => array('username' => $status->from->id, 'name' => $status->from->name, 'id' => $status->from->id, 'avatar' => 'https://graph.facebook.com/picture?id=' . $status->from->id, 'link' => 'http://facebook.com/profile.php?id=' . $status->from->id), 'type' => 'facebook', 'created_at' => new \MongoDate(strtotime($status->created_time)), 'content' => isset($status->message) ? $status->message : NULL, 'id' => $status->id, 'link' => isset($status->link) ? $status->link : NULL), 'facebook' => $status, 'internal' => array('filter_id' => (int) $filter->id));
\Consumer\Interaction::insert($interaction);
}
}
}
private function handlePost($post)
{
if ($post->verb != 'post') {
Log::info('Skipping verb ' . $post->verb);
return;
}
if (isset($post->disqusType)) {
// Disqus
$interaction = array('interaction' => array('schema' => array('version' => 3), 'author' => array('username' => isset($post->actor->preferredUsername) ? $post->actor->preferredUsername : '******', 'name' => $post->actor->preferredUsername, 'id' => $post->actor->id), 'type' => 'disqus', 'created_at' => new \MongoDate(time()), 'content' => $post->body, 'id' => $post->id, 'link' => $post->object->link), 'internal' => array('filter_id' => (int) $post->gnip->matching_rules[0]->tag));
} else {
// WordPress
$interaction = array('interaction' => array('schema' => array('version' => 3), 'author' => array('username' => $post->actor->displayName, 'name' => $post->actor->displayName, 'id' => $post->actor->id, 'avatar' => 'http://1.gravatar.com/avatar/' . $post->actor->wpEmailMd5, 'link' => $post->actor->link), 'type' => strtolower($post->provider->displayName), 'created_at' => new \MongoDate(time()), 'content' => $post->object->summary, 'id' => $post->object->id, 'link' => $post->object->link), 'internal' => array('filter_id' => (int) $post->gnip->matching_rules[0]->tag));
}
\Consumer\Interaction::insert($interaction);
}
public function consume($filter)
{
parent::consume($filter);
$twitter = new \Endroid\Twitter\Twitter(TWITTER_CONSUMER_KEY, TWITTER_CONSUMER_SECRET, TWITTER_ACCESSTOKEN, TWITTER_ACCESSTOKEN_SECRET);
$requestsPerFilter = 0;
$requestsCount = 0;
$parameters = array('q' => str_replace(PHP_EOL, ' ', $filter->query), 'result_type' => 'recent', 'count' => 100);
do {
try {
$response = $twitter->query('search/tweets', 'GET', 'json', $parameters);
} catch (Exception $e) {
Log::error('Request failed: ' . $e->getMessage());
break;
}
$requestsCount++;
if ($requestsPerFilter <= 0) {
$limit = $response->getHeader('x-rate-limit-limit');
$intervalMinutes = 4;
// We perform query every 4 minutes
$requestsPerRun = $limit / 15 * $intervalMinutes;
$activeFilters = Filter::where('active', 1)->count();
$requestsPerFilter = floor($requestsPerRun / $activeFilters);
// Internal max so the requests doesn't take to long time to run
if ($requestsPerFilter > 2) {
$requestsPerFilter = 2;
}
}
$response = json_decode($response->getContent());
foreach ($response->statuses as $status) {
$interaction = array('interaction' => array('schema' => array('version' => 3), 'author' => array('username' => $status->user->screen_name, 'name' => $status->user->name, 'id' => $status->user->id_str, 'avatar' => $status->user->profile_image_url_https, 'link' => 'http://twitter.com/' . $status->user->screen_name), 'type' => 'twitter', 'created_at' => new \MongoDate(strtotime($status->created_at)), 'content' => $status->text, 'id' => $status->id_str, 'link' => 'http://twitter.com/' . $status->user->screen_name . '/status/' . $status->id_str), 'twitter' => $status, 'internal' => array('filter_id' => (int) $filter->id));
// If we have a tweet location
if (!is_null($status->geo)) {
$location = $this->reverseGeocode($status->geo->coordinates[0], $status->geo->coordinates[1]);
$interaction['internal']['location'] = array('source' => 'tweet', 'coords' => array($status->geo->coordinates[0], $status->geo->coordinates[1]), 'state' => $location['state'], 'county' => $location['county'], 'country' => $location['country']);
// Else get lat/lon from users bio location if available
} elseif (!is_null($status->user->location)) {
$location = $this->geocode($status->user->location);
if (!is_null($location)) {
$interaction['internal']['location'] = array('source' => 'bio', 'coords' => array((double) $location['lon'], (double) $location['lat']), 'state' => $location['state'], 'county' => $location['county'], 'country' => $location['country']);
}
}
\Consumer\Interaction::insert($interaction);
}
} while ($requestsCount < $requestsPerFilter);
}