public function crawl() { $config = Config::getInstance(); $logger = Logger::getInstance(); $instance_dao = DAOFactory::getDAO('TwitterInstanceDAO'); $owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO'); $owner_dao = DAOFactory::getDAO('OwnerDAO'); $instance_hashtag_dao = DAOFactory::getDAO('InstanceHashtagDAO'); // get oauth values $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('twitter', true); $current_owner = $owner_dao->getByEmail(Session::getLoggedInUser()); $instances = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'twitter'); foreach ($instances as $instance) { $logger->setUsername($instance->network_username); $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . " on Twitter.", __METHOD__ . ',' . __LINE__); $tokens = $owner_instance_dao->getOAuthTokens($instance->id); $num_twitter_errors = isset($options['num_twitter_errors']) ? $options['num_twitter_errors']->option_value : null; $dashboard_module_cacher = new DashboardModuleCacher($instance); try { if (isset($tokens['oauth_access_token']) && $tokens['oauth_access_token'] != '' && isset($tokens['oauth_access_token_secret']) && $tokens['oauth_access_token_secret'] != '') { $archive_limit = isset($options['archive_limit']->option_value) ? $options['archive_limit']->option_value : 3200; $api = new CrawlerTwitterAPIAccessorOAuth($tokens['oauth_access_token'], $tokens['oauth_access_token_secret'], $options['oauth_consumer_key']->option_value, $options['oauth_consumer_secret']->option_value, $archive_limit, $num_twitter_errors); $twitter_crawler = new TwitterCrawler($instance, $api); $instance_dao->updateLastRun($instance->id); $twitter_crawler->fetchInstanceUserTweets(); $twitter_crawler->fetchInstanceUserMentions(); $twitter_crawler->fetchInstanceUserFriends(); $twitter_crawler->fetchInstanceUserFollowers(); $twitter_crawler->fetchInstanceUserGroups(); $twitter_crawler->fetchRetweetsOfInstanceUser(); $twitter_crawler->fetchInstanceUserFavorites(); $twitter_crawler->updateStaleGroupMemberships(); $twitter_crawler->fetchStrayRepliedToTweets(); $twitter_crawler->fetchUserFriendsByIDs(); $twitter_crawler->fetchUnloadedFriendDetails(); $twitter_crawler->fetchUnloadedFollowerDetails(); $twitter_crawler->cleanUpFollows(); $twitter_crawler->updateFriendsProfiles(); //Retrieve search results for saved keyword/hashtags $instances_hashtags = $instance_hashtag_dao->getByInstance($instance->id); foreach ($instances_hashtags as $instance_hashtag) { $twitter_crawler->fetchInstanceHashtagTweets($instance_hashtag); } } else { throw new Exception('Missing Twitter OAuth tokens.'); } } catch (Exception $e) { $logger->logUserError(get_class($e) . " while crawling " . $instance->network_username . " on Twitter: " . $e->getMessage(), __METHOD__ . ',' . __LINE__); } $dashboard_module_cacher->cacheDashboardModules(); // Save instance if (isset($twitter_crawler->user)) { $instance_dao->save($instance, $twitter_crawler->user->post_count, $logger); } Reporter::reportVersion($instance); $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . " on Twitter.", __METHOD__ . ',' . __LINE__); } }
function twitter_crawl() { global $THINKTANK_CFG; global $db; global $conn; $logger = new Logger($THINKTANK_CFG['log_location']); $id = new InstanceDAO($db, $logger); $oid = new OwnerInstanceDAO($db, $logger); $instances = $id->getAllActiveInstancesStalestFirstByNetwork('twitter'); foreach ($instances as $i) { $logger->setUsername($i->network_username); $tokens = $oid->getOAuthTokens($i->id); $noauth = true; if (isset($tokens['oauth_access_token']) && $tokens['oauth_access_token'] != '' && isset($tokens['oauth_access_token_secret']) && $tokens['oauth_access_token_secret'] != '') { $noauth = false; } if ($noauth) { $api = new CrawlerTwitterAPIAccessorOAuth('NOAUTH', 'NOAUTH', $THINKTANK_CFG['oauth_consumer_key'], $THINKTANK_CFG['oauth_consumer_secret'], $i, $THINKTANK_CFG['archive_limit']); } else { $api = new CrawlerTwitterAPIAccessorOAuth($tokens['oauth_access_token'], $tokens['oauth_access_token_secret'], $THINKTANK_CFG['oauth_consumer_key'], $THINKTANK_CFG['oauth_consumer_secret'], $i, $THINKTANK_CFG['archive_limit']); } $crawler = new TwitterCrawler($i, $logger, $api, $db); $cfg = new Config($i->network_username, $i->network_user_id); $api->init($logger); if ($api->available_api_calls_for_crawler > 0) { $id->updateLastRun($i->id); // No auth req'd $crawler->fetchInstanceUserInfo(); // No auth for public Twitter users $crawler->fetchInstanceUserTweets(); if (!$noauth) { // Auth req'd, for calling user only $crawler->fetchInstanceUserRetweetsByMe(); // Auth req'd, for calling user only $crawler->fetchInstanceUserMentions(); // Auth req'd, for calling user only $crawler->fetchInstanceUserFriends(); // Auth req'd, for calling user only $crawler->fetchInstanceUserFollowers(); } $crawler->fetchStrayRepliedToTweets(); $crawler->fetchUnloadedFollowerDetails(); $crawler->fetchFriendTweetsAndFriends(); // TODO: Get direct messages // TODO: Gather favorites data if ($noauth) { // No auth req'd $crawler->fetchSearchResults($i->network_username); } $crawler->cleanUpFollows(); // Save instance $id->save($crawler->instance, $crawler->owner_object->post_count, $logger, $api); } } $logger->close(); # Close logging }
function twitter_crawl() { global $THINKTANK_CFG; global $db; global $conn; $logger = new Logger($THINKTANK_CFG['log_location']); $id = new InstanceDAO($db, $logger); $oid = new OwnerInstanceDAO($db, $logger); $lurlapi = new LongUrlAPIAccessor($THINKTANK_CFG['app_title']); $flickrapi = new FlickrAPIAccessor($THINKTANK_CFG['flickr_api_key']); $instances = $id->getAllActiveInstancesStalestFirst(); foreach ($instances as $i) { $logger->setUsername($i->network_username); $tokens = $oid->getOAuthTokens($i->id); $api = new CrawlerTwitterAPIAccessorOAuth($tokens['oauth_access_token'], $tokens['oauth_access_token_secret'], $THINKTANK_CFG['oauth_consumer_key'], $THINKTANK_CFG['oauth_consumer_secret'], $i, $THINKTANK_CFG['archive_limit']); $crawler = new TwitterCrawler($i, $logger, $api, $db); $cfg = new Config($i->network_username, $i->network_user_id); $api->init($logger); if ($api->available_api_calls_for_crawler > 0) { $id->updateLastRun($i->id); $crawler->fetchInstanceUserInfo(); $crawler->fetchInstanceUserTweets($lurlapi, $flickrapi); $crawler->fetchInstanceUserRetweetsByMe($lurlapi, $flickrapi); $crawler->fetchInstanceUserMentions($lurlapi, $flickrapi); $crawler->fetchInstanceUserFriends(); $crawler->fetchInstanceUserFollowers(); $crawler->fetchStrayRepliedToTweets($lurlapi, $flickrapi); $crawler->fetchUnloadedFollowerDetails(); $crawler->fetchFriendTweetsAndFriends($lurlapi, $flickrapi); // TODO: Get direct messages // TODO: Gather favorites data $crawler->cleanUpFollows(); // Save instance $id->save($crawler->instance, $crawler->owner_object->post_count, $logger, $api); } } $logger->close(); # Close logging if (isset($conn)) { $db->closeConnection($conn); } // Clean up }
public function testFetchInstanceUserTweetsRetweetsBudget() { self::setUpInstanceUserAmygdala(); // set up crawl limit budget $crawl_limit = array('fetchInstanceUserMentions' => array('count' => 2, 'remaining' => 0)); $this->api->setCallerLimits($crawl_limit); $this->instance->last_page_fetched_tweets = 17; $twitter_crawler = new TwitterCrawler($this->instance, $this->api); $twitter_crawler->fetchInstanceUserInfo(); $twitter_crawler->fetchInstanceUserTweets(); $pdao = DAOFactory::getDAO('PostDAO'); $twitter_crawler->fetchInstanceUserMentions(); // old-style RT $post = $pdao->getPost('8957053141778432', 'twitter'); $this->assertNull($post); }
public function testFetchInstanceUserTweetsRetweets() { self::setUpInstanceUserAmygdala(); $this->instance->last_page_fetched_tweets = 17; $tc = new TwitterCrawler($this->instance, $this->api); $tc->fetchInstanceUserInfo(); $tc->fetchInstanceUserTweets(); $pdao = DAOFactory::getDAO('PostDAO'); $post = $pdao->getPost(13708601491193856, 'twitter'); $retweets = $pdao->getRetweetsOfPost(13708601491193856, 'twitter', true); $this->assertEqual(sizeof($retweets), 1); $this->assertEqual($post->link->url, "http://is.gd/izUl5"); $post = $pdao->getPost(13960125416996864, 'twitter'); $this->assertEqual($post->in_retweet_of_post_id, 13708601491193856); $this->assertEqual($post->in_rt_of_user_id, 20542737); $this->assertEqual($post->link->url, "http://is.gd/izUl5"); $tc->fetchInstanceUserMentions(); $post = $pdao->getPost(8957053141778432, 'twitter'); $this->assertEqual($post->in_rt_of_user_id, 2768241); $this->assertEqual($post->in_retweet_of_post_id, 8927196122972160); $post_orig = $pdao->getPost(8927196122972160, 'twitter'); $this->assertEqual($post_orig->old_retweet_count_cache, 1); $this->assertEqual($post_orig->retweet_count_cache, 0); }
public function crawl() { $config = Config::getInstance(); $logger = Logger::getInstance(); $instance_dao = DAOFactory::getDAO('TwitterInstanceDAO'); $owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO'); $owner_dao = DAOFactory::getDAO('OwnerDAO'); // get oauth values $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('twitter', true); $current_owner = $owner_dao->getByEmail(Session::getLoggedInUser()); $instances = $instance_dao->getAllActiveInstancesStalestFirstByNetwork('twitter'); foreach ($instances as $instance) { if (!$owner_instance_dao->doesOwnerHaveAccess($current_owner, $instance)) { // Owner doesn't have access to this instance; let's not crawl it. continue; } $logger->setUsername($instance->network_username); $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . " on Twitter.", __METHOD__ . ',' . __LINE__); $tokens = $owner_instance_dao->getOAuthTokens($instance->id); $noauth = true; $num_twitter_errors = isset($options['num_twitter_errors']) ? $options['num_twitter_errors']->option_value : null; $max_api_calls_per_crawl = isset($options['max_api_calls_per_crawl']) ? $options['max_api_calls_per_crawl']->option_value : 350; if (isset($tokens['oauth_access_token']) && $tokens['oauth_access_token'] != '' && isset($tokens['oauth_access_token_secret']) && $tokens['oauth_access_token_secret'] != '') { $noauth = false; } $api_calls_to_leave_unmade_per_minute = isset($options['api_calls_to_leave_unmade_per_minute']) ? $options['api_calls_to_leave_unmade_per_minute']->option_value : 2.0; if ($noauth) { $api = new CrawlerTwitterAPIAccessorOAuth('NOAUTH', 'NOAUTH', $options['oauth_consumer_key']->option_value, $options['oauth_consumer_secret']->option_value, $api_calls_to_leave_unmade_per_minute, $options['archive_limit']->option_value, $num_twitter_errors, $max_api_calls_per_crawl); } else { $api = new CrawlerTwitterAPIAccessorOAuth($tokens['oauth_access_token'], $tokens['oauth_access_token_secret'], $options['oauth_consumer_key']->option_value, $options['oauth_consumer_secret']->option_value, $api_calls_to_leave_unmade_per_minute, $options['archive_limit']->option_value, $num_twitter_errors, $max_api_calls_per_crawl); } $crawler = new TwitterCrawler($instance, $api); $api->init(); if ($api->available_api_calls_for_crawler > 0) { $instance_dao->updateLastRun($instance->id); // No auth req'd //$crawler->fetchInstanceUserInfo(); // No auth for public Twitter users $crawler->fetchInstanceUserTweets(); if (!$noauth) { // Auth req'd, for calling user only $crawler->fetchInstanceUserMentions(); $crawler->fetchInstanceUserFriends(); $crawler->fetchInstanceFavorites(); $crawler->fetchInstanceUserFollowers(); $crawler->fetchRetweetsOfInstanceUser(); $crawler->cleanUpMissedFavsUnFavs(); } $crawler->fetchStrayRepliedToTweets(); $crawler->fetchUnloadedFollowerDetails(); $crawler->fetchFriendTweetsAndFriends(); if ($noauth) { // No auth req'd $crawler->fetchSearchResults($instance->network_username); } $crawler->cleanUpFollows(); // Save instance if (isset($crawler->user)) { $instance_dao->save($instance, $crawler->user->post_count, $logger); } $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . " on Twitter.", __METHOD__ . ',' . __LINE__); } } }
public function testFetchInstanceUserTweetsRetweets() { self::setUpInstanceUserAmygdala(); $this->instance->last_page_fetched_tweets = 17; $tc = new TwitterCrawler($this->instance, $this->api); $tc->fetchInstanceUserInfo(); $tc->fetchInstanceUserTweets(); $pdao = DAOFactory::getDAO('PostDAO'); $post = $pdao->getPost('13708601491193856', 'twitter'); // the xml <retweeted_status> for the original post (that 'amygdala' retweeted) includes a native // <retweet_count> of 8. In our database we only have 1 of those RTs stored/processed. $this->assertEqual($post->retweet_count_api, 8); $this->assertEqual($post->retweet_count_cache, 1); $this->assertEqual($post->old_retweet_count_cache, 0); $retweets = $pdao->getRetweetsOfPost('13708601491193856', 'twitter', true); $this->assertEqual(sizeof($retweets), 1); $this->assertEqual($post->link->url, "http://is.gd/izUl5"); $post = $pdao->getPost('13960125416996864', 'twitter'); $this->assertEqual($post->in_retweet_of_post_id, '13708601491193856'); $this->assertEqual($post->in_rt_of_user_id, 20542737); $this->assertEqual($post->link->url, "http://is.gd/izUl5"); $tc->fetchInstanceUserMentions(); // old-style RT $post = $pdao->getPost('8957053141778432', 'twitter'); $this->assertEqual($post->in_rt_of_user_id, 2768241); $this->assertEqual($post->in_retweet_of_post_id, '8927196122972160'); $post_orig = $pdao->getPost('8927196122972160', 'twitter'); $this->assertEqual($post_orig->old_retweet_count_cache, 1); $this->assertEqual($post_orig->retweet_count_cache, 0); $this->assertEqual($post_orig->retweet_count_api, 0); }
public function testFetchInstanceUserTweetsRetweets() { $this->debug(__METHOD__); self::setUpInstanceUserAmygdala(); $user_builder = FixtureBuilder::build('users', array('user_id' => '2768241', 'network' => 'twitter')); $twitter_crawler = new TwitterCrawler($this->instance, $this->api); $twitter_crawler->api->to->setDataPathFolder('testoftwittercrawler/retweets/'); $twitter_crawler->fetchInstanceUserTweets(); $post_dao = DAOFactory::getDAO('PostDAO'); $post = $post_dao->getPost('297179577304875011', 'twitter'); // the JSON retweeted_status for the original post (that 'amygdala' retweeted) includes a native // retweet_count of 114. In our database we only have 1 of those RTs stored/processed. $this->assertNotNull($post); $this->assertIsA($post, 'Post'); $this->assertEqual($post->retweet_count_api, 114); $this->assertEqual($post->retweet_count_cache, 1); $this->assertEqual($post->old_retweet_count_cache, 0); $retweets = $post_dao->getRetweetsOfPost('297179577304875011', 'twitter', true); $this->assertEqual(sizeof($retweets), 1); $post = $post_dao->getPost('300464193944055808', 'twitter'); $this->assertEqual($post->in_retweet_of_post_id, '297179577304875011'); $this->assertEqual($post->in_rt_of_user_id, '14248315'); $twitter_crawler->fetchInstanceUserMentions(); // old-style RT $post = $post_dao->getPost('298872594713673728', 'twitter'); $this->assertEqual($post->in_rt_of_user_id, '2768241'); $this->assertEqual($post->in_retweet_of_post_id, '298865318707752960'); $post_orig = $post_dao->getPost('298865318707752960', 'twitter'); $this->assertEqual($post_orig->old_retweet_count_cache, 1); $this->assertEqual($post_orig->retweet_count_cache, 0); $this->assertEqual($post_orig->retweet_count_api, 1); $mention_dao = DAOFactory::getDAO('MentionDAO'); $mentions = $mention_dao->getMentionInfoUserName('amygdala', 'twitter'); $this->assertEqual($mentions['count_cache'], 10); }
public function crawl() { $config = Config::getInstance(); $logger = Logger::getInstance(); $id = DAOFactory::getDAO('InstanceDAO'); $oid = DAOFactory::getDAO('OwnerInstanceDAO'); $od = DAOFactory::getDAO('OwnerDAO'); // get oauth values $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('twitter', true); $current_owner = $od->getByEmail(Session::getLoggedInUser()); $instances = $id->getAllActiveInstancesStalestFirstByNetwork('twitter'); foreach ($instances as $instance) { if (!$oid->doesOwnerHaveAccess($current_owner, $instance)) { // Owner doesn't have access to this instance; let's not crawl it. continue; } $logger->setUsername($instance->network_username); $tokens = $oid->getOAuthTokens($instance->id); $noauth = true; $num_twitter_errors = isset($options['num_twitter_errors']) ? $options['num_twitter_errors']->option_value : null; if (isset($tokens['oauth_access_token']) && $tokens['oauth_access_token'] != '' && isset($tokens['oauth_access_token_secret']) && $tokens['oauth_access_token_secret'] != '') { $noauth = false; } if ($noauth) { $api = new CrawlerTwitterAPIAccessorOAuth('NOAUTH', 'NOAUTH', $options['oauth_consumer_key']->option_value, $options['oauth_consumer_secret']->option_value, $instance, $options['archive_limit']->option_value, $num_twitter_errors); } else { $api = new CrawlerTwitterAPIAccessorOAuth($tokens['oauth_access_token'], $tokens['oauth_access_token_secret'], $options['oauth_consumer_key']->option_value, $options['oauth_consumer_secret']->option_value, $instance, $options['archive_limit']->option_value, $num_twitter_errors); } $crawler = new TwitterCrawler($instance, $api); $api->init(); if ($api->available_api_calls_for_crawler > 0) { $id->updateLastRun($instance->id); // No auth req'd //$crawler->fetchInstanceUserInfo(); // No auth for public Twitter users $crawler->fetchInstanceUserTweets(); if (!$noauth) { // Auth req'd, for calling user only $crawler->fetchInstanceUserMentions(); $crawler->fetchRetweetsOfInstanceUser(); $crawler->fetchInstanceUserFriends(); $crawler->fetchInstanceUserFollowers(); } $crawler->fetchStrayRepliedToTweets(); $crawler->fetchUnloadedFollowerDetails(); $crawler->fetchFriendTweetsAndFriends(); //@TODO Gather favorites data if ($noauth) { // No auth req'd $crawler->fetchSearchResults($instance->network_username); } $crawler->cleanUpFollows(); // Save instance if (isset($crawler->owner_object)) { $id->save($instance, $crawler->owner_object->post_count, $logger); } } } $logger->close(); # Close logging }