protected function execute(InputInterface $input, OutputInterface $output) { register_shutdown_function(function () { $errfile = 'unknown file'; $errstr = 'shutdown'; $errno = E_CORE_ERROR; $errline = 0; $error = error_get_last(); if ($error !== NULL) { $errno = $error["type"]; $errfile = $error["file"]; $errline = $error["line"]; $errstr = $error["message"]; Log::info(print_r($error, true)); } }); $consumer = \Consumer\Consumer::resolve($input->getArgument('consumer')); $consumerName = get_class($consumer); if ($consumer instanceof \Consumer\IndividualConsumer\IndividualConsumer || $consumer instanceof \Consumer\DatasiftConsumer) { $filter = \Consumer\Model\Filter::find($input->getArgument('filter')); $output->writeln('Consuming "' . $consumerName . '" with filter "' . $filter->title . '"'); $consumer->consume($filter); } elseif ($consumer instanceof \Consumer\GnipConsumer) { $endpoint = $input->getArgument('filter'); $output->writeln('Consuming "' . $consumerName . '" (' . $endpoint . ')'); $consumer->setEndpoint($endpoint); $consumer->consume(\Consumer\Model\Filter::where('active', 1)->get()); } else { $output->writeln('Consuming "' . $consumerName . '"'); $consumer->consume(\Consumer\Model\Filter::where('active', 1)->get()); } }
private function prepareConsumers() { // Get all active filters $filters = \Consumer\Model\Filter::where('active', 1)->get(); // Loop through all active consumers foreach (explode(',', CONSUMERS) as $consumerName) { $consumer = \Consumer\Consumer::resolve($consumerName); if ($consumer instanceof \Consumer\IndividualConsumer\IndividualConsumer || $consumer instanceof \Consumer\DatasiftConsumer) { foreach ($filters as $filter) { $this->processes[$consumerName . ' - Filter ' . $filter->id] = new \Symfony\Component\Process\Process('./consume work ' . $consumerName . ' ' . $filter->id); $this->processes[$consumerName . ' - Filter ' . $filter->id]->start(); } } else { if ($consumer instanceof \Consumer\GnipConsumer) { $publishers = explode(',', GNIP_PUBLISHERS); foreach ($publishers as $publisher) { $this->processes[$consumerName . ' - ' . $publisher] = new \Symfony\Component\Process\Process('./consume work ' . $consumerName . ' ' . $publisher); $this->processes[$consumerName . ' - ' . $publisher]->start(); } } else { $this->processes[$consumerName] = new \Symfony\Component\Process\Process('./consume work ' . $consumerName); $this->processes[$consumerName]->start(); } } $this->output->writeln('<info>' . $consumerName . ':</info> Started'); } }
public function consume($filter) { parent::consume($filter); $twitter = new \Endroid\Twitter\Twitter(TWITTER_CONSUMER_KEY, TWITTER_CONSUMER_SECRET, TWITTER_ACCESSTOKEN, TWITTER_ACCESSTOKEN_SECRET); $requestsPerFilter = 0; $requestsCount = 0; $parameters = array('q' => str_replace(PHP_EOL, ' ', $filter->query), 'result_type' => 'recent', 'count' => 100); do { try { $response = $twitter->query('search/tweets', 'GET', 'json', $parameters); } catch (Exception $e) { Log::error('Request failed: ' . $e->getMessage()); break; } $requestsCount++; if ($requestsPerFilter <= 0) { $limit = $response->getHeader('x-rate-limit-limit'); $intervalMinutes = 4; // We perform query every 4 minutes $requestsPerRun = $limit / 15 * $intervalMinutes; $activeFilters = Filter::where('active', 1)->count(); $requestsPerFilter = floor($requestsPerRun / $activeFilters); // Internal max so the requests doesn't take to long time to run if ($requestsPerFilter > 2) { $requestsPerFilter = 2; } } $response = json_decode($response->getContent()); foreach ($response->statuses as $status) { $interaction = array('interaction' => array('schema' => array('version' => 3), 'author' => array('username' => $status->user->screen_name, 'name' => $status->user->name, 'id' => $status->user->id_str, 'avatar' => $status->user->profile_image_url_https, 'link' => 'http://twitter.com/' . $status->user->screen_name), 'type' => 'twitter', 'created_at' => new \MongoDate(strtotime($status->created_at)), 'content' => $status->text, 'id' => $status->id_str, 'link' => 'http://twitter.com/' . $status->user->screen_name . '/status/' . $status->id_str), 'twitter' => $status, 'internal' => array('filter_id' => (int) $filter->id)); // If we have a tweet location if (!is_null($status->geo)) { $location = $this->reverseGeocode($status->geo->coordinates[0], $status->geo->coordinates[1]); $interaction['internal']['location'] = array('source' => 'tweet', 'coords' => array($status->geo->coordinates[0], $status->geo->coordinates[1]), 'state' => $location['state'], 'county' => $location['county'], 'country' => $location['country']); // Else get lat/lon from users bio location if available } elseif (!is_null($status->user->location)) { $location = $this->geocode($status->user->location); if (!is_null($location)) { $interaction['internal']['location'] = array('source' => 'bio', 'coords' => array((double) $location['lon'], (double) $location['lat']), 'state' => $location['state'], 'county' => $location['county'], 'country' => $location['country']); } } \Consumer\Interaction::insert($interaction); } } while ($requestsCount < $requestsPerFilter); }