function facebook_crawl()
{
//TODO Crawl Facebook posts and comments and insert them into the database
global $THINKTANK_CFG;
global $db;
global $conn;
$logger = new Logger($THINKTANK_CFG['log_location']);
$id = new InstanceDAO($db, $logger);
$oid = new OwnerInstanceDAO($db, $logger);
$instances = $id->getAllActiveInstancesStalestFirstByNetwork('facebook');
foreach ($instances as $i) {
$logger->setUsername($i->network_username);
$tokens = $oid->getOAuthTokens($i->id);
$session_key = $tokens['oauth_access_token'];
$fb = new Facebook($THINKTANK_CFG['facebook_api_key'], $THINKTANK_CFG['facebook_api_secret']);
$cfg = new Config($i->network_username, $i->network_user_id);
$id->updateLastRun($i->id);
$crawler = new FacebookCrawler($i, $logger, $fb, $db);
$crawler->fetchInstanceUserInfo($i->network_user_id, $session_key);
$crawler->fetchUserPostsAndReplies($i->network_user_id, $session_key);
$id->save($crawler->instance, $crawler->owner_object->post_count, $logger, $fb);
}
$logger->close();
# Close logging
}
function twitter_crawl()
{
global $THINKTANK_CFG;
global $db;
global $conn;
$logger = new Logger($THINKTANK_CFG['log_location']);
$id = new InstanceDAO($db, $logger);
$oid = new OwnerInstanceDAO($db, $logger);
$instances = $id->getAllActiveInstancesStalestFirstByNetwork('twitter');
foreach ($instances as $i) {
$logger->setUsername($i->network_username);
$tokens = $oid->getOAuthTokens($i->id);
$noauth = true;
if (isset($tokens['oauth_access_token']) && $tokens['oauth_access_token'] != '' && isset($tokens['oauth_access_token_secret']) && $tokens['oauth_access_token_secret'] != '') {
$noauth = false;
}
if ($noauth) {
$api = new CrawlerTwitterAPIAccessorOAuth('NOAUTH', 'NOAUTH', $THINKTANK_CFG['oauth_consumer_key'], $THINKTANK_CFG['oauth_consumer_secret'], $i, $THINKTANK_CFG['archive_limit']);
} else {
$api = new CrawlerTwitterAPIAccessorOAuth($tokens['oauth_access_token'], $tokens['oauth_access_token_secret'], $THINKTANK_CFG['oauth_consumer_key'], $THINKTANK_CFG['oauth_consumer_secret'], $i, $THINKTANK_CFG['archive_limit']);
}
$crawler = new TwitterCrawler($i, $logger, $api, $db);
$cfg = new Config($i->network_username, $i->network_user_id);
$api->init($logger);
if ($api->available_api_calls_for_crawler > 0) {
$id->updateLastRun($i->id);
// No auth req'd
$crawler->fetchInstanceUserInfo();
// No auth for public Twitter users
$crawler->fetchInstanceUserTweets();
if (!$noauth) {
// Auth req'd, for calling user only
$crawler->fetchInstanceUserRetweetsByMe();
// Auth req'd, for calling user only
$crawler->fetchInstanceUserMentions();
// Auth req'd, for calling user only
$crawler->fetchInstanceUserFriends();
// Auth req'd, for calling user only
$crawler->fetchInstanceUserFollowers();
}
$crawler->fetchStrayRepliedToTweets();
$crawler->fetchUnloadedFollowerDetails();
$crawler->fetchFriendTweetsAndFriends();
// TODO: Get direct messages
// TODO: Gather favorites data
if ($noauth) {
// No auth req'd
$crawler->fetchSearchResults($i->network_username);
}
$crawler->cleanUpFollows();
// Save instance
$id->save($crawler->instance, $crawler->owner_object->post_count, $logger, $api);
}
}
$logger->close();
# Close logging
}
function testUpdateTokens()
{
global $THINKTANK_CFG;
$dao = new OwnerInstanceDAO($this->db, $this->logger);
$result = $dao->insert(10, 20, 'aaa', 'bbb');
$this->assertTrue($result);
$result = $dao->updateTokens(10, 20, 'ccc', 'ddd');
$this->assertTrue($result);
$tokens = $dao->getOAuthTokens(20);
$this->assertEqual($tokens['oauth_access_token'], 'ccc');
$this->assertEqual($tokens['oauth_access_token_secret'], 'ddd');
}
public function crawl()
{
global $db;
//TODO Remove when PDO port is complete
global $conn;
$logger = Logger::getInstance();
$config = Config::getInstance();
$id = DAOFactory::getDAO('InstanceDAO');
$oid = new OwnerInstanceDAO($db, $logger);
//crawl Facebook user profiles
$instances = $id->getAllActiveInstancesStalestFirstByNetwork('facebook');
foreach ($instances as $instance) {
$logger->setUsername($instance->network_username);
$tokens = $oid->getOAuthTokens($instance->id);
$session_key = $tokens['oauth_access_token'];
$fb = new Facebook($config->getValue('facebook_api_key'), $config->getValue('facebook_api_secret'));
$id->updateLastRun($instance->id);
$crawler = new FacebookCrawler($instance, $fb);
$crawler->fetchInstanceUserInfo($instance->network_user_id, $session_key);
$crawler->fetchUserPostsAndReplies($instance->network_user_id, $session_key);
$id->save($crawler->instance, $crawler->owner_object->post_count, $logger, $fb);
}
//crawl Facebook pages
$instances = $id->getAllActiveInstancesStalestFirstByNetwork('facebook page');
foreach ($instances as $instance) {
$logger->setUsername($instance->network_username);
$tokens = $oid->getOAuthTokens($instance->id);
$session_key = $tokens['oauth_access_token'];
$fb = new Facebook($config->getValue('facebook_api_key'), $config->getValue('facebook_api_secret'));
$id->updateLastRun($instance->id);
$crawler = new FacebookCrawler($instance, $fb);
$crawler->fetchPagePostsAndReplies($instance->network_user_id, $instance->network_viewer_id, $session_key);
$id->save($crawler->instance, 0, $logger, $fb);
}
$logger->close();
# Close logging
}
function twitter_crawl()
{
global $THINKTANK_CFG;
global $db;
global $conn;
$logger = new Logger($THINKTANK_CFG['log_location']);
$id = new InstanceDAO($db, $logger);
$oid = new OwnerInstanceDAO($db, $logger);
$lurlapi = new LongUrlAPIAccessor($THINKTANK_CFG['app_title']);
$flickrapi = new FlickrAPIAccessor($THINKTANK_CFG['flickr_api_key']);
$instances = $id->getAllActiveInstancesStalestFirst();
foreach ($instances as $i) {
$logger->setUsername($i->network_username);
$tokens = $oid->getOAuthTokens($i->id);
$api = new CrawlerTwitterAPIAccessorOAuth($tokens['oauth_access_token'], $tokens['oauth_access_token_secret'], $THINKTANK_CFG['oauth_consumer_key'], $THINKTANK_CFG['oauth_consumer_secret'], $i, $THINKTANK_CFG['archive_limit']);
$crawler = new TwitterCrawler($i, $logger, $api, $db);
$cfg = new Config($i->network_username, $i->network_user_id);
$api->init($logger);
if ($api->available_api_calls_for_crawler > 0) {
$id->updateLastRun($i->id);
$crawler->fetchInstanceUserInfo();
$crawler->fetchInstanceUserTweets($lurlapi, $flickrapi);
$crawler->fetchInstanceUserRetweetsByMe($lurlapi, $flickrapi);
$crawler->fetchInstanceUserMentions($lurlapi, $flickrapi);
$crawler->fetchInstanceUserFriends();
$crawler->fetchInstanceUserFollowers();
$crawler->fetchStrayRepliedToTweets($lurlapi, $flickrapi);
$crawler->fetchUnloadedFollowerDetails();
$crawler->fetchFriendTweetsAndFriends($lurlapi, $flickrapi);
// TODO: Get direct messages
// TODO: Gather favorites data
$crawler->cleanUpFollows();
// Save instance
$id->save($crawler->instance, $crawler->owner_object->post_count, $logger, $api);
}
}
$logger->close();
# Close logging
if (isset($conn)) {
$db->closeConnection($conn);
}
// Clean up
}
<?php
require_once 'config.crawler.inc.php';
ini_set("include_path", ini_get("include_path") . PATH_SEPARATOR . $INCLUDE_PATH);
require_once "init.php";
$db = new Database($THINKTANK_CFG);
$conn = $db->getConnection();
$logger = new Logger($THINKTANK_CFG['log_location']);
$id = new InstanceDAO($db, $logger);
$oid = new OwnerInstanceDAO($db, $logger);
$lurlapi = new LongUrlAPIAccessor($THINKTANK_CFG['app_title']);
$flickrapi = new FlickrAPIAccessor($THINKTANK_CFG['flickr_api_key']);
$instances = $id->getAllInstancesStalestFirst();
foreach ($instances as $i) {
$logger->setUsername($i->twitter_username);
$tokens = $oid->getOAuthTokens($i->id);
$api = new CrawlerTwitterAPIAccessorOAuth($tokens['oauth_access_token'], $tokens['oauth_access_token_secret'], $THINKTANK_CFG['oauth_consumer_key'], $THINKTANK_CFG['oauth_consumer_secret'], $i, $THINKTANK_CFG['archive_limit']);
$crawler = new Crawler($i, $logger, $api, $db);
$cfg = new Config($i->twitter_username, $i->twitter_user_id);
$api->init($logger);
if ($api->available_api_calls_for_crawler > 0) {
$id->updateLastRun($i->id);
$crawler->fetchInstanceUserInfo();
$crawler->fetchInstanceUserTweets($lurlapi, $flickrapi);
$crawler->fetchInstanceUserRetweetsByMe($lurlapi, $flickrapi);
$crawler->fetchInstanceUserMentions($lurlapi, $flickrapi);
$crawler->fetchInstanceUserFriends();
$crawler->fetchInstanceUserFollowers();
$crawler->fetchStrayRepliedToTweets($lurlapi, $flickrapi);
$crawler->fetchUnloadedFollowerDetails();
$crawler->fetchFriendTweetsAndFriends($lurlapi, $flickrapi);
