public function crawl()
{
$config = Config::getInstance();
$logger = Logger::getInstance();
$instance_dao = DAOFactory::getDAO('TwitterInstanceDAO');
$owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
$owner_dao = DAOFactory::getDAO('OwnerDAO');
$instance_hashtag_dao = DAOFactory::getDAO('InstanceHashtagDAO');
// get oauth values
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('twitter', true);
$current_owner = $owner_dao->getByEmail(Session::getLoggedInUser());
$instances = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'twitter');
foreach ($instances as $instance) {
$logger->setUsername($instance->network_username);
$logger->logUserSuccess("Starting to collect data for " . $instance->network_username . " on Twitter.", __METHOD__ . ',' . __LINE__);
$tokens = $owner_instance_dao->getOAuthTokens($instance->id);
$num_twitter_errors = isset($options['num_twitter_errors']) ? $options['num_twitter_errors']->option_value : null;
$dashboard_module_cacher = new DashboardModuleCacher($instance);
try {
if (isset($tokens['oauth_access_token']) && $tokens['oauth_access_token'] != '' && isset($tokens['oauth_access_token_secret']) && $tokens['oauth_access_token_secret'] != '') {
$archive_limit = isset($options['archive_limit']->option_value) ? $options['archive_limit']->option_value : 3200;
$api = new CrawlerTwitterAPIAccessorOAuth($tokens['oauth_access_token'], $tokens['oauth_access_token_secret'], $options['oauth_consumer_key']->option_value, $options['oauth_consumer_secret']->option_value, $archive_limit, $num_twitter_errors);
$twitter_crawler = new TwitterCrawler($instance, $api);
$instance_dao->updateLastRun($instance->id);
$twitter_crawler->fetchInstanceUserTweets();
$twitter_crawler->fetchInstanceUserMentions();
$twitter_crawler->fetchInstanceUserFriends();
$twitter_crawler->fetchInstanceUserFollowers();
$twitter_crawler->fetchInstanceUserGroups();
$twitter_crawler->fetchRetweetsOfInstanceUser();
$twitter_crawler->fetchInstanceUserFavorites();
$twitter_crawler->updateStaleGroupMemberships();
$twitter_crawler->fetchStrayRepliedToTweets();
$twitter_crawler->fetchUserFriendsByIDs();
$twitter_crawler->fetchUnloadedFriendDetails();
$twitter_crawler->fetchUnloadedFollowerDetails();
$twitter_crawler->cleanUpFollows();
$twitter_crawler->updateFriendsProfiles();
//Retrieve search results for saved keyword/hashtags
$instances_hashtags = $instance_hashtag_dao->getByInstance($instance->id);
foreach ($instances_hashtags as $instance_hashtag) {
$twitter_crawler->fetchInstanceHashtagTweets($instance_hashtag);
}
} else {
throw new Exception('Missing Twitter OAuth tokens.');
}
} catch (Exception $e) {
$logger->logUserError(get_class($e) . " while crawling " . $instance->network_username . " on Twitter: " . $e->getMessage(), __METHOD__ . ',' . __LINE__);
}
$dashboard_module_cacher->cacheDashboardModules();
// Save instance
if (isset($twitter_crawler->user)) {
$instance_dao->save($instance, $twitter_crawler->user->post_count, $logger);
}
Reporter::reportVersion($instance);
$logger->logUserSuccess("Finished collecting data for " . $instance->network_username . " on Twitter.", __METHOD__ . ',' . __LINE__);
}
}
function twitter_crawl()
{
global $THINKTANK_CFG;
global $db;
global $conn;
$logger = new Logger($THINKTANK_CFG['log_location']);
$id = new InstanceDAO($db, $logger);
$oid = new OwnerInstanceDAO($db, $logger);
$instances = $id->getAllActiveInstancesStalestFirstByNetwork('twitter');
foreach ($instances as $i) {
$logger->setUsername($i->network_username);
$tokens = $oid->getOAuthTokens($i->id);
$noauth = true;
if (isset($tokens['oauth_access_token']) && $tokens['oauth_access_token'] != '' && isset($tokens['oauth_access_token_secret']) && $tokens['oauth_access_token_secret'] != '') {
$noauth = false;
}
if ($noauth) {
$api = new CrawlerTwitterAPIAccessorOAuth('NOAUTH', 'NOAUTH', $THINKTANK_CFG['oauth_consumer_key'], $THINKTANK_CFG['oauth_consumer_secret'], $i, $THINKTANK_CFG['archive_limit']);
} else {
$api = new CrawlerTwitterAPIAccessorOAuth($tokens['oauth_access_token'], $tokens['oauth_access_token_secret'], $THINKTANK_CFG['oauth_consumer_key'], $THINKTANK_CFG['oauth_consumer_secret'], $i, $THINKTANK_CFG['archive_limit']);
}
$crawler = new TwitterCrawler($i, $logger, $api, $db);
$cfg = new Config($i->network_username, $i->network_user_id);
$api->init($logger);
if ($api->available_api_calls_for_crawler > 0) {
$id->updateLastRun($i->id);
// No auth req'd
$crawler->fetchInstanceUserInfo();
// No auth for public Twitter users
$crawler->fetchInstanceUserTweets();
if (!$noauth) {
// Auth req'd, for calling user only
$crawler->fetchInstanceUserRetweetsByMe();
// Auth req'd, for calling user only
$crawler->fetchInstanceUserMentions();
// Auth req'd, for calling user only
$crawler->fetchInstanceUserFriends();
// Auth req'd, for calling user only
$crawler->fetchInstanceUserFollowers();
}
$crawler->fetchStrayRepliedToTweets();
$crawler->fetchUnloadedFollowerDetails();
$crawler->fetchFriendTweetsAndFriends();
// TODO: Get direct messages
// TODO: Gather favorites data
if ($noauth) {
// No auth req'd
$crawler->fetchSearchResults($i->network_username);
}
$crawler->cleanUpFollows();
// Save instance
$id->save($crawler->instance, $crawler->owner_object->post_count, $logger, $api);
}
}
$logger->close();
# Close logging
}
function twitter_crawl()
{
global $THINKTANK_CFG;
global $db;
global $conn;
$logger = new Logger($THINKTANK_CFG['log_location']);
$id = new InstanceDAO($db, $logger);
$oid = new OwnerInstanceDAO($db, $logger);
$lurlapi = new LongUrlAPIAccessor($THINKTANK_CFG['app_title']);
$flickrapi = new FlickrAPIAccessor($THINKTANK_CFG['flickr_api_key']);
$instances = $id->getAllActiveInstancesStalestFirst();
foreach ($instances as $i) {
$logger->setUsername($i->network_username);
$tokens = $oid->getOAuthTokens($i->id);
$api = new CrawlerTwitterAPIAccessorOAuth($tokens['oauth_access_token'], $tokens['oauth_access_token_secret'], $THINKTANK_CFG['oauth_consumer_key'], $THINKTANK_CFG['oauth_consumer_secret'], $i, $THINKTANK_CFG['archive_limit']);
$crawler = new TwitterCrawler($i, $logger, $api, $db);
$cfg = new Config($i->network_username, $i->network_user_id);
$api->init($logger);
if ($api->available_api_calls_for_crawler > 0) {
$id->updateLastRun($i->id);
$crawler->fetchInstanceUserInfo();
$crawler->fetchInstanceUserTweets($lurlapi, $flickrapi);
$crawler->fetchInstanceUserRetweetsByMe($lurlapi, $flickrapi);
$crawler->fetchInstanceUserMentions($lurlapi, $flickrapi);
$crawler->fetchInstanceUserFriends();
$crawler->fetchInstanceUserFollowers();
$crawler->fetchStrayRepliedToTweets($lurlapi, $flickrapi);
$crawler->fetchUnloadedFollowerDetails();
$crawler->fetchFriendTweetsAndFriends($lurlapi, $flickrapi);
// TODO: Get direct messages
// TODO: Gather favorites data
$crawler->cleanUpFollows();
// Save instance
$id->save($crawler->instance, $crawler->owner_object->post_count, $logger, $api);
}
}
$logger->close();
# Close logging
if (isset($conn)) {
$db->closeConnection($conn);
}
// Clean up
}
public function testFetchStrayRepliedToTweetsBudget()
{
self::setUpInstanceUserAnilDash();
// set up crawl limit budget
$crawl_limit = array('fetchAndAddTweetRepliedTo' => array('count' => 2, 'remaining' => 0));
$this->api->setCallerLimits($crawl_limit);
$this->api->available_api_calls_for_crawler = 4;
$twitter_crawler = new TwitterCrawler($this->instance, $this->api);
$twitter_crawler->fetchInstanceUserInfo();
$twitter_crawler->fetchInstanceUserTweets();
$pdao = DAOFactory::getDAO('PostDAO');
$tweets = $pdao->getAllPostsByUsername('anildash', 'twitter');
$twitter_crawler->fetchStrayRepliedToTweets();
$post = $pdao->getPost(15752814831, 'twitter');
$this->assertTrue(isset($post));
$this->assertEqual($post->reply_count_cache, 0);
}
public function testFetchStrayRepliedToTweets() {
self::setUpInstanceUserAnilDash();
$this->api->available_api_calls_for_crawler = 4;
$tc = new TwitterCrawler($this->instance, $this->api);
$tc->fetchInstanceUserInfo();
$tc->fetchInstanceUserTweets();
$pdao = DAOFactory::getDAO('PostDAO');
$tweets = $pdao->getAllPostsByUsername('anildash', 'twitter');
$tc->fetchStrayRepliedToTweets();
$post = $pdao->getPost(15752814831, 'twitter');
$this->assertTrue(isset($post));
$this->assertEqual($post->reply_count_cache, 1);
}
public function crawl()
{
$config = Config::getInstance();
$logger = Logger::getInstance();
$instance_dao = DAOFactory::getDAO('TwitterInstanceDAO');
$owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
$owner_dao = DAOFactory::getDAO('OwnerDAO');
// get oauth values
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('twitter', true);
$current_owner = $owner_dao->getByEmail(Session::getLoggedInUser());
$instances = $instance_dao->getAllActiveInstancesStalestFirstByNetwork('twitter');
foreach ($instances as $instance) {
if (!$owner_instance_dao->doesOwnerHaveAccess($current_owner, $instance)) {
// Owner doesn't have access to this instance; let's not crawl it.
continue;
}
$logger->setUsername($instance->network_username);
$logger->logUserSuccess("Starting to collect data for " . $instance->network_username . " on Twitter.", __METHOD__ . ',' . __LINE__);
$tokens = $owner_instance_dao->getOAuthTokens($instance->id);
$noauth = true;
$num_twitter_errors = isset($options['num_twitter_errors']) ? $options['num_twitter_errors']->option_value : null;
$max_api_calls_per_crawl = isset($options['max_api_calls_per_crawl']) ? $options['max_api_calls_per_crawl']->option_value : 350;
if (isset($tokens['oauth_access_token']) && $tokens['oauth_access_token'] != '' && isset($tokens['oauth_access_token_secret']) && $tokens['oauth_access_token_secret'] != '') {
$noauth = false;
}
$api_calls_to_leave_unmade_per_minute = isset($options['api_calls_to_leave_unmade_per_minute']) ? $options['api_calls_to_leave_unmade_per_minute']->option_value : 2.0;
if ($noauth) {
$api = new CrawlerTwitterAPIAccessorOAuth('NOAUTH', 'NOAUTH', $options['oauth_consumer_key']->option_value, $options['oauth_consumer_secret']->option_value, $api_calls_to_leave_unmade_per_minute, $options['archive_limit']->option_value, $num_twitter_errors, $max_api_calls_per_crawl);
} else {
$api = new CrawlerTwitterAPIAccessorOAuth($tokens['oauth_access_token'], $tokens['oauth_access_token_secret'], $options['oauth_consumer_key']->option_value, $options['oauth_consumer_secret']->option_value, $api_calls_to_leave_unmade_per_minute, $options['archive_limit']->option_value, $num_twitter_errors, $max_api_calls_per_crawl);
}
$crawler = new TwitterCrawler($instance, $api);
$api->init();
if ($api->available_api_calls_for_crawler > 0) {
$instance_dao->updateLastRun($instance->id);
// No auth req'd
//$crawler->fetchInstanceUserInfo();
// No auth for public Twitter users
$crawler->fetchInstanceUserTweets();
if (!$noauth) {
// Auth req'd, for calling user only
$crawler->fetchInstanceUserMentions();
$crawler->fetchInstanceUserFriends();
$crawler->fetchInstanceFavorites();
$crawler->fetchInstanceUserFollowers();
$crawler->fetchRetweetsOfInstanceUser();
$crawler->cleanUpMissedFavsUnFavs();
}
$crawler->fetchStrayRepliedToTweets();
$crawler->fetchUnloadedFollowerDetails();
$crawler->fetchFriendTweetsAndFriends();
if ($noauth) {
// No auth req'd
$crawler->fetchSearchResults($instance->network_username);
}
$crawler->cleanUpFollows();
// Save instance
if (isset($crawler->user)) {
$instance_dao->save($instance, $crawler->user->post_count, $logger);
}
$logger->logUserSuccess("Finished collecting data for " . $instance->network_username . " on Twitter.", __METHOD__ . ',' . __LINE__);
}
}
}
function testFetchInstanceUserTweets()
{
self::setUpInstanceUserAnilDash();
$tc = new TwitterCrawler($this->instance, $this->api, $this->db);
$tc->fetchInstanceUserInfo();
$tc->fetchInstanceUserTweets();
//Test post with location has location set
$pdao = DAOFactory::getDAO('PostDAO');
$this->assertTrue($pdao->isPostInDB(15680112737.0));
$post = $pdao->getPost(15680112737.0);
$this->assertEqual($post->location, "NYC: 40.739069,-73.987082");
$this->assertEqual($post->place, "Stuyvesant Town, New York");
$this->assertEqual($post->geo, "40.73410845 -73.97885982");
//Test post without location doesn't have it set
$post = $pdao->getPost(15660552927.0);
$this->assertEqual($post->location, "NYC: 40.739069,-73.987082");
$this->assertEqual($post->place, "");
$this->assertEqual($post->geo, "");
}
public function testMediaHandling()
{
$r = array('id' => 1, 'network_username' => 'ecucurella', 'network_user_id' => '13771532', 'network_viewer_id' => '13771532', 'last_post_id' => '0', 'last_reply_id' => '10001', 'total_posts_in_system' => '0', 'total_replies_in_system' => '0', 'total_follows_in_system' => '0', 'is_archive_loaded_replies' => '0', 'is_archive_loaded_follows' => '0', 'crawler_last_run' => '', 'earliest_reply_in_system' => '', 'avg_replies_per_day' => '0', 'is_public' => '0', 'is_active' => '1', 'network' => 'twitter', 'last_favorite_id' => '0', 'favorites_profile' => '0', 'owner_favs_in_system' => '0', 'total_posts_by_owner' => 0, 'posts_per_day' => 1, 'posts_per_week' => 1, 'percentage_replies' => 50, 'percentage_links' => 50, 'earliest_post_in_system' => '2009-01-01 13:48:05');
$this->instance = new TwitterInstance($r);
$this->api = new CrawlerTwitterAPIAccessorOAuth($oauth_token = '111', $oauth_token_secret = '222', $oauth_consumer_key = 'fake_key', $oauth_consumer_secret = 'fake_secret', $archive_limit = 3200, $num_twitter_errors = 2);
$this->instance->is_archive_loaded_follows = true;
$post_dao = DAOFactory::getDAO('PostDAO');
$link_dao = DAOFactory::getDAO('LinkDAO');
$twitter_crawler = new TwitterCrawler($this->instance, $this->api);
$twitter_crawler->api->to->setDataPathFolder('testoftwittercrawler/cdmoyer/');
$twitter_crawler->fetchInstanceUserTweets();
$post_dao = DAOFactory::getDAO('PostDAO');
// NOrmal photo
$post = $post_dao->getPost("471310898695794688", 'twitter');
$this->assertNotNull($post);
$this->assertEqual("Just hanging out under the couch, like you do. http://t.co/1z8GGl5Zrv", $post->post_text);
$this->assertEqual(1, count($post->links));
$this->assertEqual('http://pbs.twimg.com/media/Bopuw9BIEAAAYVN.jpg', $post->links[0]->image_src);
$this->assertEqual('http://t.co/1z8GGl5Zrv', $post->links[0]->url);
$this->assertEqual('http://twitter.com/CDMoyer/status/471310898695794688/photo/1', $post->links[0]->expanded_url);
// Normal Photo
$post = $post_dao->getPost("458015480960532480", 'twitter');
$this->assertEqual('http://pbs.twimg.com/media/BlsypxzIIAAwgnw.jpg', $post->links[0]->image_src);
$this->assertEqual('http://t.co/gELt0NIzCx', $post->links[0]->url);
$this->assertEqual('http://twitter.com/CDMoyer/status/458015480960532480/photo/1', $post->links[0]->expanded_url);
// Photo and link in retweet
$post = $post_dao->getPost("462748167357091840", 'twitter');
$this->assertEqual(2, count($post->links));
$this->assertEqual('http://t.co/uGLsKU8Qkc', $post->links[0]->url);
$this->assertEqual('', $post->links[0]->expanded_url);
$this->assertEqual('http://t.co/wpdAD4iQzB', $post->links[1]->url);
$this->assertEqual('http://twitter.com/pourmecoffee/status/462748167357091840/photo/1', $post->links[1]->expanded_url);
$this->assertEqual('http://pbs.twimg.com/media/BmwDAUoCIAAM_H4.jpg', $post->links[1]->image_src);
// Nothing
$post = $post_dao->getPost("454606554801524736", 'twitter');
$this->assertEqual(0, count($post->links));
}
public function crawl()
{
$config = Config::getInstance();
$logger = Logger::getInstance();
$id = DAOFactory::getDAO('InstanceDAO');
$oid = DAOFactory::getDAO('OwnerInstanceDAO');
$od = DAOFactory::getDAO('OwnerDAO');
// get oauth values
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('twitter', true);
$current_owner = $od->getByEmail(Session::getLoggedInUser());
$instances = $id->getAllActiveInstancesStalestFirstByNetwork('twitter');
foreach ($instances as $instance) {
if (!$oid->doesOwnerHaveAccess($current_owner, $instance)) {
// Owner doesn't have access to this instance; let's not crawl it.
continue;
}
$logger->setUsername($instance->network_username);
$tokens = $oid->getOAuthTokens($instance->id);
$noauth = true;
$num_twitter_errors = isset($options['num_twitter_errors']) ? $options['num_twitter_errors']->option_value : null;
if (isset($tokens['oauth_access_token']) && $tokens['oauth_access_token'] != '' && isset($tokens['oauth_access_token_secret']) && $tokens['oauth_access_token_secret'] != '') {
$noauth = false;
}
if ($noauth) {
$api = new CrawlerTwitterAPIAccessorOAuth('NOAUTH', 'NOAUTH', $options['oauth_consumer_key']->option_value, $options['oauth_consumer_secret']->option_value, $instance, $options['archive_limit']->option_value, $num_twitter_errors);
} else {
$api = new CrawlerTwitterAPIAccessorOAuth($tokens['oauth_access_token'], $tokens['oauth_access_token_secret'], $options['oauth_consumer_key']->option_value, $options['oauth_consumer_secret']->option_value, $instance, $options['archive_limit']->option_value, $num_twitter_errors);
}
$crawler = new TwitterCrawler($instance, $api);
$api->init();
if ($api->available_api_calls_for_crawler > 0) {
$id->updateLastRun($instance->id);
// No auth req'd
//$crawler->fetchInstanceUserInfo();
// No auth for public Twitter users
$crawler->fetchInstanceUserTweets();
if (!$noauth) {
// Auth req'd, for calling user only
$crawler->fetchInstanceUserMentions();
$crawler->fetchRetweetsOfInstanceUser();
$crawler->fetchInstanceUserFriends();
$crawler->fetchInstanceUserFollowers();
}
$crawler->fetchStrayRepliedToTweets();
$crawler->fetchUnloadedFollowerDetails();
$crawler->fetchFriendTweetsAndFriends();
//@TODO Gather favorites data
if ($noauth) {
// No auth req'd
$crawler->fetchSearchResults($instance->network_username);
}
$crawler->cleanUpFollows();
// Save instance
if (isset($crawler->owner_object)) {
$id->save($instance, $crawler->owner_object->post_count, $logger);
}
}
}
$logger->close();
# Close logging
}
public function testFetchStrayRepliedToTweets()
{
$this->debug(__METHOD__);
self::setUpInstanceUserAnilDash();
$twitter_crawler = new TwitterCrawler($this->instance, $this->api);
$twitter_crawler->api->to->setDataPathFolder('testoftwittercrawler/anildash/');
$twitter_crawler->fetchInstanceUserTweets();
$post_dao = DAOFactory::getDAO('PostDAO');
$tweets = $post_dao->getAllPostsByUsername('anildash', 'twitter');
$this->debug(Utils::varDumpToString($tweets));
$twitter_crawler->fetchStrayRepliedToTweets();
$post = $post_dao->getPost('300389504274022400', 'twitter');
$this->assertTrue(isset($post));
$this->debug(Utils::varDumpToString($post));
$this->assertEqual($post->reply_count_cache, 2);
}