Exemple #1
0
  * in the queue by collect.php.
**/
use OpenFuego\lib\DbHandle;
use OpenFuego\app\Consumer;
use OpenFuego\lib\Logger;
if (!defined('PHP_VERSION_ID') || PHP_VERSION_ID < 50300) {
    die(__NAMESPACE__ . ' requires PHP 5.3.0 or higher.');
}
if (php_sapi_name() != 'cli') {
    die('This script must be invoked from the command line.');
}
if (!defined('OPENFUEGO') && function_exists('pcntl_fork')) {
    $error_message = "\n" . 'Do not run this script directly. Run fetch.php to start.' . "\n\n";
    die($error_message);
}
require_once __DIR__ . '/init.php';
register_shutdown_function(function () {
    Logger::fatal(__NAMESPACE__ . " consumer was terminated.");
});
$dbh = new DbHandle();
$sql = "\nCREATE TABLE IF NOT EXISTS `openfuego_citizens` (\n  `user_id` bigint(20) unsigned NOT NULL,\n  `influence` tinyint(2) unsigned NOT NULL,\n  PRIMARY KEY (`user_id`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;\n\nCREATE TABLE IF NOT EXISTS `openfuego_links` (\n  `link_id` mediumint(8) unsigned NOT NULL AUTO_INCREMENT,\n  `url` varchar(255) COLLATE utf8_unicode_ci NOT NULL DEFAULT '',\n  `first_seen` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',\n  `first_tweet` bigint(20) unsigned NOT NULL,\n  `first_user` varchar(20) COLLATE utf8_unicode_ci NOT NULL DEFAULT '',\n  `first_user_id` bigint(20) unsigned DEFAULT NULL,\n  `weighted_count` smallint(5) unsigned NOT NULL,\n  `count` smallint(5) unsigned NOT NULL DEFAULT '1',\n  `last_seen` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',\n  PRIMARY KEY (`link_id`),\n  UNIQUE KEY `url` (`url`)\n) ENGINE=InnoDB  DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;\n\nCREATE TABLE IF NOT EXISTS `openfuego_short_links` (\n  `id` mediumint(8) unsigned NOT NULL AUTO_INCREMENT,\n  `input_url` varchar(255) COLLATE utf8_unicode_ci NOT NULL,\n  `long_url` text COLLATE utf8_unicode_ci NOT NULL,\n  `last_seen` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,\n  PRIMARY KEY (`input_url`),\n  UNIQUE KEY `id` (`id`)\n) ENGINE=InnoDB  DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;\n\nCREATE TABLE IF NOT EXISTS `openfuego_tweets_cache` (\n  `link_id` mediumint(8) unsigned NOT NULL,\n  `id_str` bigint(20) unsigned NOT NULL,\n  `screen_name` varchar(15) COLLATE utf8_unicode_ci NOT NULL DEFAULT '',\n  `text` varchar(140) COLLATE utf8_unicode_ci NOT NULL DEFAULT '',\n  `profile_image_url` varchar(255) COLLATE utf8_unicode_ci NOT NULL DEFAULT '',\n  `modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,\n  PRIMARY KEY (`link_id`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;\n\t\nALTER TABLE `openfuego_tweets_cache`\n  ADD CONSTRAINT `FK.openfuego_tweets_cache.link_id` FOREIGN KEY (`link_id`) REFERENCES `openfuego_links` (`link_id`) ON DELETE CASCADE ON UPDATE CASCADE;\n";
try {
    $sth = $dbh->prepare($sql);
    $sth->execute();
} catch (\PDOException $e) {
    die($e);
}
$dbh = NULL;
$consumer = new Consumer();
$consumer->process();
exit;
 public static function getInfluence($user_id_str)
 {
     try {
         $dbh = self::getDbh();
         $sql = "SELECT influence FROM openfuego_citizens WHERE user_id = :user_id LIMIT 1;";
         $sth = self::$dbh->prepare($sql);
         $sth->bindParam('user_id', $user_id_str);
         $sth->execute();
         $influence = $sth->fetchColumn(0);
         return $influence;
     } catch (\PDOException $e) {
         Logger::error($e);
         return FALSE;
     }
 }
Exemple #3
0
if (php_sapi_name() != 'cli') {
    die('This script must be invoked from the command line.');
}
if (!defined('OPENFUEGO') && function_exists('pcntl_fork')) {
    $error_message = "\n" . 'Do not run this script directly. Run fetch.php to start.' . "\n\n";
    die($error_message);
}
require_once __DIR__ . '/init.php';
register_shutdown_function(function () {
    Logger::fatal(__NAMESPACE__ . " collector was terminated.");
});
$twitter = new TwitterHandle();
$twitter->get("account/verify_credentials", array("include_entities" => 0, "skip_status" => 1));
if ($twitter->http_code !== 200) {
    $error_message = "Cannot continue. Your Twitter credentials appear to be invalid. Error code {$twitter->http_code}";
    Logger::info($error_message);
    die($error_message);
}
unset($twitter_handle);
$authorities = unserialize(\OpenFuego\AUTHORITIES);
$universe = new Universe();
/** The next line is commented out by default.
 * Uncomment it to repopulate the universe on each fetch. */
// $universe->populate($authorities, 1);
$citizens = $universe->getCitizens(1);
if (!$citizens) {
    $universe->populate($authorities, 1);
    $citizens = $universe->getCitizens(1);
}
$citizens = array_slice($citizens, 0, TWITTER_PREDICATE_LIMIT);
// Start streaming/collecting
 /**
  * Processes a queue file and does something with it (example only)
  * @param string $queueFile The queue file
  */
 protected function processQueueFile($queueFile)
 {
     Logger::debug('Processing file: ' . $queueFile);
     // Open file
     $fp = fopen($queueFile, 'r');
     // Check if something has gone wrong, or perhaps the file is just locked by another process
     if (!is_resource($fp)) {
         Logger::error('WARN: Unable to open file or file already open: ' . $queueFile . ' - Skipping.');
         return FALSE;
     }
     // Lock file
     flock($fp, LOCK_EX);
     // Loop over each line (1 line per status)
     $statusCounter = 0;
     while ($rawStatus = fgets($fp, 8192)) {
         $statusCounter++;
         $status = json_decode($rawStatus, TRUE);
         // convert JSON data into PHP array
         // if data is invalid (e.g., if a user has deleted a tweet; surprisingly frequent)
         if (is_array($status) == FALSE || !isset($status['user']['id_str'])) {
             Logger::debug('Status is invalid, continuing.');
             continue;
             // skip it
         }
         if (array_key_exists(0, $status['entities']['urls']) == FALSE) {
             // if tweet does not contain link
             continue;
             // skip it
         }
         /* Weed out statuses created by undesired user. (The streaming API also returns _retweets of_
          ** statuses by desired user, which we don't want.) */
         if (!\OpenFuego\app\Universe::isCitizen($status['user']['id_str'])) {
             // if the tweeter is not a citizen
             continue;
             // skip it
         }
         $this->processUrls($status);
         Logger::debug('Decoded tweet: ' . $status['user']['screen_name'] . ': ' . urldecode($status['text']));
         set_time_limit(60);
         unset($status, $entities);
     }
     // End while
     // Release lock and close
     flock($fp, LOCK_UN);
     fclose($fp);
     // All done with this file
     Logger::debug('Successfully processed ' . $statusCounter . ' tweets from ' . $queueFile . ' - deleting.');
     unset($rawStatus);
     unlink($queueFile);
 }
Exemple #5
0
}
if (php_sapi_name() != 'cli') {
    die('This script must be invoked from the command line.');
}
require_once __DIR__ . '/init.php';
if (!function_exists('pcntl_fork')) {
    $error_message = "\n" . 'To start OpenFuego, run these commands:' . "\n\n" . "\tnohup " . \PHP_BINDIR . '/php ' . BASE_DIR . '/collect.php > /dev/null 2> /dev/null & echo $!' . "\n" . "\tnohup " . \PHP_BINDIR . '/php ' . BASE_DIR . '/consume.php > /dev/null 2> /dev/null & echo $!' . "\n\n";
    die($error_message);
}
// Ignore hangup signal (when user exits shell)
pcntl_signal(SIGHUP, SIG_IGN);
// Handle shutdown tasks
pcntl_signal(SIGTERM, function () {
    global $_should_stop;
    $_should_stop = TRUE;
    Logger::info("Received shutdown request, finishing up.");
    return;
});
$pids = array();
$pids[0] = pcntl_fork();
if (!$pids[0]) {
    include_once __DIR__ . '/collect.php';
}
$pids[1] = pcntl_fork();
if (!$pids[1]) {
    include_once __DIR__ . '/consume.php';
}
echo __NAMESPACE__ . ' collector running as PID ' . $pids[0] . "\n";
echo __NAMESPACE__ . ' consumer running as PID ' . $pids[1] . "\n";
@file_put_contents(\OpenFuego\TMP_DIR . '/OpenFuego-collect.pid', $pids[0]);
@file_put_contents(\OpenFuego\TMP_DIR . '/OpenFuego-consume.pid', $pids[1]);