protected function execute(InputInterface $input, OutputInterface $output)
{
register_shutdown_function(function () {
$errfile = 'unknown file';
$errstr = 'shutdown';
$errno = E_CORE_ERROR;
$errline = 0;
$error = error_get_last();
if ($error !== NULL) {
$errno = $error["type"];
$errfile = $error["file"];
$errline = $error["line"];
$errstr = $error["message"];
Log::info(print_r($error, true));
}
});
$consumer = \Consumer\Consumer::resolve($input->getArgument('consumer'));
$consumerName = get_class($consumer);
if ($consumer instanceof \Consumer\IndividualConsumer\IndividualConsumer || $consumer instanceof \Consumer\DatasiftConsumer) {
$filter = \Consumer\Model\Filter::find($input->getArgument('filter'));
$output->writeln('Consuming "' . $consumerName . '" with filter "' . $filter->title . '"');
$consumer->consume($filter);
} elseif ($consumer instanceof \Consumer\GnipConsumer) {
$endpoint = $input->getArgument('filter');
$output->writeln('Consuming "' . $consumerName . '" (' . $endpoint . ')');
$consumer->setEndpoint($endpoint);
$consumer->consume(\Consumer\Model\Filter::where('active', 1)->get());
} else {
$output->writeln('Consuming "' . $consumerName . '"');
$consumer->consume(\Consumer\Model\Filter::where('active', 1)->get());
}
}
private function prepareConsumers()
{
// Get all active filters
$filters = \Consumer\Model\Filter::where('active', 1)->get();
// Loop through all active consumers
foreach (explode(',', CONSUMERS) as $consumerName) {
$consumer = \Consumer\Consumer::resolve($consumerName);
if ($consumer instanceof \Consumer\IndividualConsumer\IndividualConsumer || $consumer instanceof \Consumer\DatasiftConsumer) {
foreach ($filters as $filter) {
$this->processes[$consumerName . ' - Filter ' . $filter->id] = new \Symfony\Component\Process\Process('./consume work ' . $consumerName . ' ' . $filter->id);
$this->processes[$consumerName . ' - Filter ' . $filter->id]->start();
}
} else {
if ($consumer instanceof \Consumer\GnipConsumer) {
$publishers = explode(',', GNIP_PUBLISHERS);
foreach ($publishers as $publisher) {
$this->processes[$consumerName . ' - ' . $publisher] = new \Symfony\Component\Process\Process('./consume work ' . $consumerName . ' ' . $publisher);
$this->processes[$consumerName . ' - ' . $publisher]->start();
}
} else {
$this->processes[$consumerName] = new \Symfony\Component\Process\Process('./consume work ' . $consumerName);
$this->processes[$consumerName]->start();
}
}
$this->output->writeln('<info>' . $consumerName . ':</info> Started');
}
}
public function consume($filter)
{
parent::consume($filter);
$twitter = new \Endroid\Twitter\Twitter(TWITTER_CONSUMER_KEY, TWITTER_CONSUMER_SECRET, TWITTER_ACCESSTOKEN, TWITTER_ACCESSTOKEN_SECRET);
$requestsPerFilter = 0;
$requestsCount = 0;
$parameters = array('q' => str_replace(PHP_EOL, ' ', $filter->query), 'result_type' => 'recent', 'count' => 100);
do {
try {
$response = $twitter->query('search/tweets', 'GET', 'json', $parameters);
} catch (Exception $e) {
Log::error('Request failed: ' . $e->getMessage());
break;
}
$requestsCount++;
if ($requestsPerFilter <= 0) {
$limit = $response->getHeader('x-rate-limit-limit');
$intervalMinutes = 4;
// We perform query every 4 minutes
$requestsPerRun = $limit / 15 * $intervalMinutes;
$activeFilters = Filter::where('active', 1)->count();
$requestsPerFilter = floor($requestsPerRun / $activeFilters);
// Internal max so the requests doesn't take to long time to run
if ($requestsPerFilter > 2) {
$requestsPerFilter = 2;
}
}
$response = json_decode($response->getContent());
foreach ($response->statuses as $status) {
$interaction = array('interaction' => array('schema' => array('version' => 3), 'author' => array('username' => $status->user->screen_name, 'name' => $status->user->name, 'id' => $status->user->id_str, 'avatar' => $status->user->profile_image_url_https, 'link' => 'http://twitter.com/' . $status->user->screen_name), 'type' => 'twitter', 'created_at' => new \MongoDate(strtotime($status->created_at)), 'content' => $status->text, 'id' => $status->id_str, 'link' => 'http://twitter.com/' . $status->user->screen_name . '/status/' . $status->id_str), 'twitter' => $status, 'internal' => array('filter_id' => (int) $filter->id));
// If we have a tweet location
if (!is_null($status->geo)) {
$location = $this->reverseGeocode($status->geo->coordinates[0], $status->geo->coordinates[1]);
$interaction['internal']['location'] = array('source' => 'tweet', 'coords' => array($status->geo->coordinates[0], $status->geo->coordinates[1]), 'state' => $location['state'], 'county' => $location['county'], 'country' => $location['country']);
// Else get lat/lon from users bio location if available
} elseif (!is_null($status->user->location)) {
$location = $this->geocode($status->user->location);
if (!is_null($location)) {
$interaction['internal']['location'] = array('source' => 'bio', 'coords' => array((double) $location['lon'], (double) $location['lat']), 'state' => $location['state'], 'county' => $location['county'], 'country' => $location['country']);
}
}
\Consumer\Interaction::insert($interaction);
}
} while ($requestsCount < $requestsPerFilter);
}
