* in the queue by collect.php.
**/
use OpenFuego\lib\DbHandle;
use OpenFuego\app\Consumer;
use OpenFuego\lib\Logger;
if (!defined('PHP_VERSION_ID') || PHP_VERSION_ID < 50300) {
die(__NAMESPACE__ . ' requires PHP 5.3.0 or higher.');
}
if (php_sapi_name() != 'cli') {
die('This script must be invoked from the command line.');
}
if (!defined('OPENFUEGO') && function_exists('pcntl_fork')) {
$error_message = "\n" . 'Do not run this script directly. Run fetch.php to start.' . "\n\n";
die($error_message);
}
require_once __DIR__ . '/init.php';
register_shutdown_function(function () {
Logger::fatal(__NAMESPACE__ . " consumer was terminated.");
});
$dbh = new DbHandle();
$sql = "\nCREATE TABLE IF NOT EXISTS `openfuego_citizens` (\n `user_id` bigint(20) unsigned NOT NULL,\n `influence` tinyint(2) unsigned NOT NULL,\n PRIMARY KEY (`user_id`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;\n\nCREATE TABLE IF NOT EXISTS `openfuego_links` (\n `link_id` mediumint(8) unsigned NOT NULL AUTO_INCREMENT,\n `url` varchar(255) COLLATE utf8_unicode_ci NOT NULL DEFAULT '',\n `first_seen` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',\n `first_tweet` bigint(20) unsigned NOT NULL,\n `first_user` varchar(20) COLLATE utf8_unicode_ci NOT NULL DEFAULT '',\n `first_user_id` bigint(20) unsigned DEFAULT NULL,\n `weighted_count` smallint(5) unsigned NOT NULL,\n `count` smallint(5) unsigned NOT NULL DEFAULT '1',\n `last_seen` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',\n PRIMARY KEY (`link_id`),\n UNIQUE KEY `url` (`url`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;\n\nCREATE TABLE IF NOT EXISTS `openfuego_short_links` (\n `id` mediumint(8) unsigned NOT NULL AUTO_INCREMENT,\n `input_url` varchar(255) COLLATE utf8_unicode_ci NOT NULL,\n `long_url` text COLLATE utf8_unicode_ci NOT NULL,\n `last_seen` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,\n PRIMARY KEY (`input_url`),\n UNIQUE KEY `id` (`id`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;\n\nCREATE TABLE IF NOT EXISTS `openfuego_tweets_cache` (\n `link_id` mediumint(8) unsigned NOT NULL,\n `id_str` bigint(20) unsigned NOT NULL,\n `screen_name` varchar(15) COLLATE utf8_unicode_ci NOT NULL DEFAULT '',\n `text` varchar(140) COLLATE utf8_unicode_ci NOT NULL DEFAULT '',\n `profile_image_url` varchar(255) COLLATE utf8_unicode_ci NOT NULL DEFAULT '',\n `modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,\n PRIMARY KEY (`link_id`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;\n\t\nALTER TABLE `openfuego_tweets_cache`\n ADD CONSTRAINT `FK.openfuego_tweets_cache.link_id` FOREIGN KEY (`link_id`) REFERENCES `openfuego_links` (`link_id`) ON DELETE CASCADE ON UPDATE CASCADE;\n";
try {
$sth = $dbh->prepare($sql);
$sth->execute();
} catch (\PDOException $e) {
die($e);
}
$dbh = NULL;
$consumer = new Consumer();
$consumer->process();
exit;
public static function getInfluence($user_id_str)
{
try {
$dbh = self::getDbh();
$sql = "SELECT influence FROM openfuego_citizens WHERE user_id = :user_id LIMIT 1;";
$sth = self::$dbh->prepare($sql);
$sth->bindParam('user_id', $user_id_str);
$sth->execute();
$influence = $sth->fetchColumn(0);
return $influence;
} catch (\PDOException $e) {
Logger::error($e);
return FALSE;
}
}
if (php_sapi_name() != 'cli') {
die('This script must be invoked from the command line.');
}
if (!defined('OPENFUEGO') && function_exists('pcntl_fork')) {
$error_message = "\n" . 'Do not run this script directly. Run fetch.php to start.' . "\n\n";
die($error_message);
}
require_once __DIR__ . '/init.php';
register_shutdown_function(function () {
Logger::fatal(__NAMESPACE__ . " collector was terminated.");
});
$twitter = new TwitterHandle();
$twitter->get("account/verify_credentials", array("include_entities" => 0, "skip_status" => 1));
if ($twitter->http_code !== 200) {
$error_message = "Cannot continue. Your Twitter credentials appear to be invalid. Error code {$twitter->http_code}";
Logger::info($error_message);
die($error_message);
}
unset($twitter_handle);
$authorities = unserialize(\OpenFuego\AUTHORITIES);
$universe = new Universe();
/** The next line is commented out by default.
* Uncomment it to repopulate the universe on each fetch. */
// $universe->populate($authorities, 1);
$citizens = $universe->getCitizens(1);
if (!$citizens) {
$universe->populate($authorities, 1);
$citizens = $universe->getCitizens(1);
}
$citizens = array_slice($citizens, 0, TWITTER_PREDICATE_LIMIT);
// Start streaming/collecting
/**
* Processes a queue file and does something with it (example only)
* @param string $queueFile The queue file
*/
protected function processQueueFile($queueFile)
{
Logger::debug('Processing file: ' . $queueFile);
// Open file
$fp = fopen($queueFile, 'r');
// Check if something has gone wrong, or perhaps the file is just locked by another process
if (!is_resource($fp)) {
Logger::error('WARN: Unable to open file or file already open: ' . $queueFile . ' - Skipping.');
return FALSE;
}
// Lock file
flock($fp, LOCK_EX);
// Loop over each line (1 line per status)
$statusCounter = 0;
while ($rawStatus = fgets($fp, 8192)) {
$statusCounter++;
$status = json_decode($rawStatus, TRUE);
// convert JSON data into PHP array
// if data is invalid (e.g., if a user has deleted a tweet; surprisingly frequent)
if (is_array($status) == FALSE || !isset($status['user']['id_str'])) {
Logger::debug('Status is invalid, continuing.');
continue;
// skip it
}
if (array_key_exists(0, $status['entities']['urls']) == FALSE) {
// if tweet does not contain link
continue;
// skip it
}
/* Weed out statuses created by undesired user. (The streaming API also returns _retweets of_
** statuses by desired user, which we don't want.) */
if (!\OpenFuego\app\Universe::isCitizen($status['user']['id_str'])) {
// if the tweeter is not a citizen
continue;
// skip it
}
$this->processUrls($status);
Logger::debug('Decoded tweet: ' . $status['user']['screen_name'] . ': ' . urldecode($status['text']));
set_time_limit(60);
unset($status, $entities);
}
// End while
// Release lock and close
flock($fp, LOCK_UN);
fclose($fp);
// All done with this file
Logger::debug('Successfully processed ' . $statusCounter . ' tweets from ' . $queueFile . ' - deleting.');
unset($rawStatus);
unlink($queueFile);
}
}
if (php_sapi_name() != 'cli') {
die('This script must be invoked from the command line.');
}
require_once __DIR__ . '/init.php';
if (!function_exists('pcntl_fork')) {
$error_message = "\n" . 'To start OpenFuego, run these commands:' . "\n\n" . "\tnohup " . \PHP_BINDIR . '/php ' . BASE_DIR . '/collect.php > /dev/null 2> /dev/null & echo $!' . "\n" . "\tnohup " . \PHP_BINDIR . '/php ' . BASE_DIR . '/consume.php > /dev/null 2> /dev/null & echo $!' . "\n\n";
die($error_message);
}
// Ignore hangup signal (when user exits shell)
pcntl_signal(SIGHUP, SIG_IGN);
// Handle shutdown tasks
pcntl_signal(SIGTERM, function () {
global $_should_stop;
$_should_stop = TRUE;
Logger::info("Received shutdown request, finishing up.");
return;
});
$pids = array();
$pids[0] = pcntl_fork();
if (!$pids[0]) {
include_once __DIR__ . '/collect.php';
}
$pids[1] = pcntl_fork();
if (!$pids[1]) {
include_once __DIR__ . '/consume.php';
}
echo __NAMESPACE__ . ' collector running as PID ' . $pids[0] . "\n";
echo __NAMESPACE__ . ' consumer running as PID ' . $pids[1] . "\n";
@file_put_contents(\OpenFuego\TMP_DIR . '/OpenFuego-collect.pid', $pids[0]);
@file_put_contents(\OpenFuego\TMP_DIR . '/OpenFuego-consume.pid', $pids[1]);
