public function crawl()
 {
     $config = Config::getInstance();
     $logger = Logger::getInstance();
     $instance_dao = DAOFactory::getDAO('TwitterInstanceDAO');
     $owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
     $owner_dao = DAOFactory::getDAO('OwnerDAO');
     $instance_hashtag_dao = DAOFactory::getDAO('InstanceHashtagDAO');
     // get oauth values
     $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
     $options = $plugin_option_dao->getOptionsHash('twitter', true);
     $current_owner = $owner_dao->getByEmail(Session::getLoggedInUser());
     $instances = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'twitter');
     foreach ($instances as $instance) {
         $logger->setUsername($instance->network_username);
         $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . " on Twitter.", __METHOD__ . ',' . __LINE__);
         $tokens = $owner_instance_dao->getOAuthTokens($instance->id);
         $num_twitter_errors = isset($options['num_twitter_errors']) ? $options['num_twitter_errors']->option_value : null;
         $dashboard_module_cacher = new DashboardModuleCacher($instance);
         try {
             if (isset($tokens['oauth_access_token']) && $tokens['oauth_access_token'] != '' && isset($tokens['oauth_access_token_secret']) && $tokens['oauth_access_token_secret'] != '') {
                 $archive_limit = isset($options['archive_limit']->option_value) ? $options['archive_limit']->option_value : 3200;
                 $api = new CrawlerTwitterAPIAccessorOAuth($tokens['oauth_access_token'], $tokens['oauth_access_token_secret'], $options['oauth_consumer_key']->option_value, $options['oauth_consumer_secret']->option_value, $archive_limit, $num_twitter_errors);
                 $twitter_crawler = new TwitterCrawler($instance, $api);
                 $instance_dao->updateLastRun($instance->id);
                 $twitter_crawler->fetchInstanceUserTweets();
                 $twitter_crawler->fetchInstanceUserMentions();
                 $twitter_crawler->fetchInstanceUserFriends();
                 $twitter_crawler->fetchInstanceUserFollowers();
                 $twitter_crawler->fetchInstanceUserGroups();
                 $twitter_crawler->fetchRetweetsOfInstanceUser();
                 $twitter_crawler->fetchInstanceUserFavorites();
                 $twitter_crawler->updateStaleGroupMemberships();
                 $twitter_crawler->fetchStrayRepliedToTweets();
                 $twitter_crawler->fetchUserFriendsByIDs();
                 $twitter_crawler->fetchUnloadedFriendDetails();
                 $twitter_crawler->fetchUnloadedFollowerDetails();
                 $twitter_crawler->cleanUpFollows();
                 $twitter_crawler->updateFriendsProfiles();
                 //Retrieve search results for saved keyword/hashtags
                 $instances_hashtags = $instance_hashtag_dao->getByInstance($instance->id);
                 foreach ($instances_hashtags as $instance_hashtag) {
                     $twitter_crawler->fetchInstanceHashtagTweets($instance_hashtag);
                 }
             } else {
                 throw new Exception('Missing Twitter OAuth tokens.');
             }
         } catch (Exception $e) {
             $logger->logUserError(get_class($e) . " while crawling " . $instance->network_username . " on Twitter: " . $e->getMessage(), __METHOD__ . ',' . __LINE__);
         }
         $dashboard_module_cacher->cacheDashboardModules();
         // Save instance
         if (isset($twitter_crawler->user)) {
             $instance_dao->save($instance, $twitter_crawler->user->post_count, $logger);
         }
         Reporter::reportVersion($instance);
         $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . " on Twitter.", __METHOD__ . ',' . __LINE__);
     }
 }
Example #2
0
function twitter_crawl()
{
    global $THINKTANK_CFG;
    global $db;
    global $conn;
    $logger = new Logger($THINKTANK_CFG['log_location']);
    $id = new InstanceDAO($db, $logger);
    $oid = new OwnerInstanceDAO($db, $logger);
    $instances = $id->getAllActiveInstancesStalestFirstByNetwork('twitter');
    foreach ($instances as $i) {
        $logger->setUsername($i->network_username);
        $tokens = $oid->getOAuthTokens($i->id);
        $noauth = true;
        if (isset($tokens['oauth_access_token']) && $tokens['oauth_access_token'] != '' && isset($tokens['oauth_access_token_secret']) && $tokens['oauth_access_token_secret'] != '') {
            $noauth = false;
        }
        if ($noauth) {
            $api = new CrawlerTwitterAPIAccessorOAuth('NOAUTH', 'NOAUTH', $THINKTANK_CFG['oauth_consumer_key'], $THINKTANK_CFG['oauth_consumer_secret'], $i, $THINKTANK_CFG['archive_limit']);
        } else {
            $api = new CrawlerTwitterAPIAccessorOAuth($tokens['oauth_access_token'], $tokens['oauth_access_token_secret'], $THINKTANK_CFG['oauth_consumer_key'], $THINKTANK_CFG['oauth_consumer_secret'], $i, $THINKTANK_CFG['archive_limit']);
        }
        $crawler = new TwitterCrawler($i, $logger, $api, $db);
        $cfg = new Config($i->network_username, $i->network_user_id);
        $api->init($logger);
        if ($api->available_api_calls_for_crawler > 0) {
            $id->updateLastRun($i->id);
            // No auth req'd
            $crawler->fetchInstanceUserInfo();
            // No auth for public Twitter users
            $crawler->fetchInstanceUserTweets();
            if (!$noauth) {
                // Auth req'd, for calling user only
                $crawler->fetchInstanceUserRetweetsByMe();
                // Auth req'd, for calling user only
                $crawler->fetchInstanceUserMentions();
                // Auth req'd, for calling user only
                $crawler->fetchInstanceUserFriends();
                // Auth req'd, for calling user only
                $crawler->fetchInstanceUserFollowers();
            }
            $crawler->fetchStrayRepliedToTweets();
            $crawler->fetchUnloadedFollowerDetails();
            $crawler->fetchFriendTweetsAndFriends();
            // TODO: Get direct messages
            // TODO: Gather favorites data
            if ($noauth) {
                // No auth req'd
                $crawler->fetchSearchResults($i->network_username);
            }
            $crawler->cleanUpFollows();
            // Save instance
            $id->save($crawler->instance, $crawler->owner_object->post_count, $logger, $api);
        }
    }
    $logger->close();
    # Close logging
}
Example #3
0
function twitter_crawl()
{
    global $THINKTANK_CFG;
    global $db;
    global $conn;
    $logger = new Logger($THINKTANK_CFG['log_location']);
    $id = new InstanceDAO($db, $logger);
    $oid = new OwnerInstanceDAO($db, $logger);
    $lurlapi = new LongUrlAPIAccessor($THINKTANK_CFG['app_title']);
    $flickrapi = new FlickrAPIAccessor($THINKTANK_CFG['flickr_api_key']);
    $instances = $id->getAllActiveInstancesStalestFirst();
    foreach ($instances as $i) {
        $logger->setUsername($i->network_username);
        $tokens = $oid->getOAuthTokens($i->id);
        $api = new CrawlerTwitterAPIAccessorOAuth($tokens['oauth_access_token'], $tokens['oauth_access_token_secret'], $THINKTANK_CFG['oauth_consumer_key'], $THINKTANK_CFG['oauth_consumer_secret'], $i, $THINKTANK_CFG['archive_limit']);
        $crawler = new TwitterCrawler($i, $logger, $api, $db);
        $cfg = new Config($i->network_username, $i->network_user_id);
        $api->init($logger);
        if ($api->available_api_calls_for_crawler > 0) {
            $id->updateLastRun($i->id);
            $crawler->fetchInstanceUserInfo();
            $crawler->fetchInstanceUserTweets($lurlapi, $flickrapi);
            $crawler->fetchInstanceUserRetweetsByMe($lurlapi, $flickrapi);
            $crawler->fetchInstanceUserMentions($lurlapi, $flickrapi);
            $crawler->fetchInstanceUserFriends();
            $crawler->fetchInstanceUserFollowers();
            $crawler->fetchStrayRepliedToTweets($lurlapi, $flickrapi);
            $crawler->fetchUnloadedFollowerDetails();
            $crawler->fetchFriendTweetsAndFriends($lurlapi, $flickrapi);
            // TODO: Get direct messages
            // TODO: Gather favorites data
            $crawler->cleanUpFollows();
            // Save instance
            $id->save($crawler->instance, $crawler->owner_object->post_count, $logger, $api);
        }
    }
    $logger->close();
    # Close logging
    if (isset($conn)) {
        $db->closeConnection($conn);
    }
    // Clean up
}
 public function testFetchStrayRepliedToTweetsBudget()
 {
     self::setUpInstanceUserAnilDash();
     // set up crawl limit budget
     $crawl_limit = array('fetchAndAddTweetRepliedTo' => array('count' => 2, 'remaining' => 0));
     $this->api->setCallerLimits($crawl_limit);
     $this->api->available_api_calls_for_crawler = 4;
     $twitter_crawler = new TwitterCrawler($this->instance, $this->api);
     $twitter_crawler->fetchInstanceUserInfo();
     $twitter_crawler->fetchInstanceUserTweets();
     $pdao = DAOFactory::getDAO('PostDAO');
     $tweets = $pdao->getAllPostsByUsername('anildash', 'twitter');
     $twitter_crawler->fetchStrayRepliedToTweets();
     $post = $pdao->getPost(15752814831, 'twitter');
     $this->assertTrue(isset($post));
     $this->assertEqual($post->reply_count_cache, 0);
 }
Example #5
0
    public function testFetchStrayRepliedToTweets() {
        self::setUpInstanceUserAnilDash();
        $this->api->available_api_calls_for_crawler = 4;
        $tc = new TwitterCrawler($this->instance, $this->api);
        $tc->fetchInstanceUserInfo();
        $tc->fetchInstanceUserTweets();
        $pdao = DAOFactory::getDAO('PostDAO');
        $tweets = $pdao->getAllPostsByUsername('anildash', 'twitter');

        $tc->fetchStrayRepliedToTweets();
        $post = $pdao->getPost(15752814831, 'twitter');
        $this->assertTrue(isset($post));
        $this->assertEqual($post->reply_count_cache, 1);
    }
 public function crawl()
 {
     $config = Config::getInstance();
     $logger = Logger::getInstance();
     $instance_dao = DAOFactory::getDAO('TwitterInstanceDAO');
     $owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
     $owner_dao = DAOFactory::getDAO('OwnerDAO');
     // get oauth values
     $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
     $options = $plugin_option_dao->getOptionsHash('twitter', true);
     $current_owner = $owner_dao->getByEmail(Session::getLoggedInUser());
     $instances = $instance_dao->getAllActiveInstancesStalestFirstByNetwork('twitter');
     foreach ($instances as $instance) {
         if (!$owner_instance_dao->doesOwnerHaveAccess($current_owner, $instance)) {
             // Owner doesn't have access to this instance; let's not crawl it.
             continue;
         }
         $logger->setUsername($instance->network_username);
         $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . " on Twitter.", __METHOD__ . ',' . __LINE__);
         $tokens = $owner_instance_dao->getOAuthTokens($instance->id);
         $noauth = true;
         $num_twitter_errors = isset($options['num_twitter_errors']) ? $options['num_twitter_errors']->option_value : null;
         $max_api_calls_per_crawl = isset($options['max_api_calls_per_crawl']) ? $options['max_api_calls_per_crawl']->option_value : 350;
         if (isset($tokens['oauth_access_token']) && $tokens['oauth_access_token'] != '' && isset($tokens['oauth_access_token_secret']) && $tokens['oauth_access_token_secret'] != '') {
             $noauth = false;
         }
         $api_calls_to_leave_unmade_per_minute = isset($options['api_calls_to_leave_unmade_per_minute']) ? $options['api_calls_to_leave_unmade_per_minute']->option_value : 2.0;
         if ($noauth) {
             $api = new CrawlerTwitterAPIAccessorOAuth('NOAUTH', 'NOAUTH', $options['oauth_consumer_key']->option_value, $options['oauth_consumer_secret']->option_value, $api_calls_to_leave_unmade_per_minute, $options['archive_limit']->option_value, $num_twitter_errors, $max_api_calls_per_crawl);
         } else {
             $api = new CrawlerTwitterAPIAccessorOAuth($tokens['oauth_access_token'], $tokens['oauth_access_token_secret'], $options['oauth_consumer_key']->option_value, $options['oauth_consumer_secret']->option_value, $api_calls_to_leave_unmade_per_minute, $options['archive_limit']->option_value, $num_twitter_errors, $max_api_calls_per_crawl);
         }
         $crawler = new TwitterCrawler($instance, $api);
         $api->init();
         if ($api->available_api_calls_for_crawler > 0) {
             $instance_dao->updateLastRun($instance->id);
             // No auth req'd
             //$crawler->fetchInstanceUserInfo();
             // No auth for public Twitter users
             $crawler->fetchInstanceUserTweets();
             if (!$noauth) {
                 // Auth req'd, for calling user only
                 $crawler->fetchInstanceUserMentions();
                 $crawler->fetchInstanceUserFriends();
                 $crawler->fetchInstanceFavorites();
                 $crawler->fetchInstanceUserFollowers();
                 $crawler->fetchRetweetsOfInstanceUser();
                 $crawler->cleanUpMissedFavsUnFavs();
             }
             $crawler->fetchStrayRepliedToTweets();
             $crawler->fetchUnloadedFollowerDetails();
             $crawler->fetchFriendTweetsAndFriends();
             if ($noauth) {
                 // No auth req'd
                 $crawler->fetchSearchResults($instance->network_username);
             }
             $crawler->cleanUpFollows();
             // Save instance
             if (isset($crawler->user)) {
                 $instance_dao->save($instance, $crawler->user->post_count, $logger);
             }
             $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . " on Twitter.", __METHOD__ . ',' . __LINE__);
         }
     }
 }
Example #7
0
 function testFetchInstanceUserTweets()
 {
     self::setUpInstanceUserAnilDash();
     $tc = new TwitterCrawler($this->instance, $this->api, $this->db);
     $tc->fetchInstanceUserInfo();
     $tc->fetchInstanceUserTweets();
     //Test post with location has location set
     $pdao = DAOFactory::getDAO('PostDAO');
     $this->assertTrue($pdao->isPostInDB(15680112737.0));
     $post = $pdao->getPost(15680112737.0);
     $this->assertEqual($post->location, "NYC: 40.739069,-73.987082");
     $this->assertEqual($post->place, "Stuyvesant Town, New York");
     $this->assertEqual($post->geo, "40.73410845 -73.97885982");
     //Test post without location doesn't have it set
     $post = $pdao->getPost(15660552927.0);
     $this->assertEqual($post->location, "NYC: 40.739069,-73.987082");
     $this->assertEqual($post->place, "");
     $this->assertEqual($post->geo, "");
 }
 public function testMediaHandling()
 {
     $r = array('id' => 1, 'network_username' => 'ecucurella', 'network_user_id' => '13771532', 'network_viewer_id' => '13771532', 'last_post_id' => '0', 'last_reply_id' => '10001', 'total_posts_in_system' => '0', 'total_replies_in_system' => '0', 'total_follows_in_system' => '0', 'is_archive_loaded_replies' => '0', 'is_archive_loaded_follows' => '0', 'crawler_last_run' => '', 'earliest_reply_in_system' => '', 'avg_replies_per_day' => '0', 'is_public' => '0', 'is_active' => '1', 'network' => 'twitter', 'last_favorite_id' => '0', 'favorites_profile' => '0', 'owner_favs_in_system' => '0', 'total_posts_by_owner' => 0, 'posts_per_day' => 1, 'posts_per_week' => 1, 'percentage_replies' => 50, 'percentage_links' => 50, 'earliest_post_in_system' => '2009-01-01 13:48:05');
     $this->instance = new TwitterInstance($r);
     $this->api = new CrawlerTwitterAPIAccessorOAuth($oauth_token = '111', $oauth_token_secret = '222', $oauth_consumer_key = 'fake_key', $oauth_consumer_secret = 'fake_secret', $archive_limit = 3200, $num_twitter_errors = 2);
     $this->instance->is_archive_loaded_follows = true;
     $post_dao = DAOFactory::getDAO('PostDAO');
     $link_dao = DAOFactory::getDAO('LinkDAO');
     $twitter_crawler = new TwitterCrawler($this->instance, $this->api);
     $twitter_crawler->api->to->setDataPathFolder('testoftwittercrawler/cdmoyer/');
     $twitter_crawler->fetchInstanceUserTweets();
     $post_dao = DAOFactory::getDAO('PostDAO');
     // NOrmal photo
     $post = $post_dao->getPost("471310898695794688", 'twitter');
     $this->assertNotNull($post);
     $this->assertEqual("Just hanging out under the couch, like you do. http://t.co/1z8GGl5Zrv", $post->post_text);
     $this->assertEqual(1, count($post->links));
     $this->assertEqual('http://pbs.twimg.com/media/Bopuw9BIEAAAYVN.jpg', $post->links[0]->image_src);
     $this->assertEqual('http://t.co/1z8GGl5Zrv', $post->links[0]->url);
     $this->assertEqual('http://twitter.com/CDMoyer/status/471310898695794688/photo/1', $post->links[0]->expanded_url);
     // Normal Photo
     $post = $post_dao->getPost("458015480960532480", 'twitter');
     $this->assertEqual('http://pbs.twimg.com/media/BlsypxzIIAAwgnw.jpg', $post->links[0]->image_src);
     $this->assertEqual('http://t.co/gELt0NIzCx', $post->links[0]->url);
     $this->assertEqual('http://twitter.com/CDMoyer/status/458015480960532480/photo/1', $post->links[0]->expanded_url);
     // Photo and link in retweet
     $post = $post_dao->getPost("462748167357091840", 'twitter');
     $this->assertEqual(2, count($post->links));
     $this->assertEqual('http://t.co/uGLsKU8Qkc', $post->links[0]->url);
     $this->assertEqual('', $post->links[0]->expanded_url);
     $this->assertEqual('http://t.co/wpdAD4iQzB', $post->links[1]->url);
     $this->assertEqual('http://twitter.com/pourmecoffee/status/462748167357091840/photo/1', $post->links[1]->expanded_url);
     $this->assertEqual('http://pbs.twimg.com/media/BmwDAUoCIAAM_H4.jpg', $post->links[1]->image_src);
     // Nothing
     $post = $post_dao->getPost("454606554801524736", 'twitter');
     $this->assertEqual(0, count($post->links));
 }
 public function crawl()
 {
     $config = Config::getInstance();
     $logger = Logger::getInstance();
     $id = DAOFactory::getDAO('InstanceDAO');
     $oid = DAOFactory::getDAO('OwnerInstanceDAO');
     $od = DAOFactory::getDAO('OwnerDAO');
     // get oauth values
     $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
     $options = $plugin_option_dao->getOptionsHash('twitter', true);
     $current_owner = $od->getByEmail(Session::getLoggedInUser());
     $instances = $id->getAllActiveInstancesStalestFirstByNetwork('twitter');
     foreach ($instances as $instance) {
         if (!$oid->doesOwnerHaveAccess($current_owner, $instance)) {
             // Owner doesn't have access to this instance; let's not crawl it.
             continue;
         }
         $logger->setUsername($instance->network_username);
         $tokens = $oid->getOAuthTokens($instance->id);
         $noauth = true;
         $num_twitter_errors = isset($options['num_twitter_errors']) ? $options['num_twitter_errors']->option_value : null;
         if (isset($tokens['oauth_access_token']) && $tokens['oauth_access_token'] != '' && isset($tokens['oauth_access_token_secret']) && $tokens['oauth_access_token_secret'] != '') {
             $noauth = false;
         }
         if ($noauth) {
             $api = new CrawlerTwitterAPIAccessorOAuth('NOAUTH', 'NOAUTH', $options['oauth_consumer_key']->option_value, $options['oauth_consumer_secret']->option_value, $instance, $options['archive_limit']->option_value, $num_twitter_errors);
         } else {
             $api = new CrawlerTwitterAPIAccessorOAuth($tokens['oauth_access_token'], $tokens['oauth_access_token_secret'], $options['oauth_consumer_key']->option_value, $options['oauth_consumer_secret']->option_value, $instance, $options['archive_limit']->option_value, $num_twitter_errors);
         }
         $crawler = new TwitterCrawler($instance, $api);
         $api->init();
         if ($api->available_api_calls_for_crawler > 0) {
             $id->updateLastRun($instance->id);
             // No auth req'd
             //$crawler->fetchInstanceUserInfo();
             // No auth for public Twitter users
             $crawler->fetchInstanceUserTweets();
             if (!$noauth) {
                 // Auth req'd, for calling user only
                 $crawler->fetchInstanceUserMentions();
                 $crawler->fetchRetweetsOfInstanceUser();
                 $crawler->fetchInstanceUserFriends();
                 $crawler->fetchInstanceUserFollowers();
             }
             $crawler->fetchStrayRepliedToTweets();
             $crawler->fetchUnloadedFollowerDetails();
             $crawler->fetchFriendTweetsAndFriends();
             //@TODO Gather favorites data
             if ($noauth) {
                 // No auth req'd
                 $crawler->fetchSearchResults($instance->network_username);
             }
             $crawler->cleanUpFollows();
             // Save instance
             if (isset($crawler->owner_object)) {
                 $id->save($instance, $crawler->owner_object->post_count, $logger);
             }
         }
     }
     $logger->close();
     # Close logging
 }
Example #10
0
 public function testFetchStrayRepliedToTweets()
 {
     $this->debug(__METHOD__);
     self::setUpInstanceUserAnilDash();
     $twitter_crawler = new TwitterCrawler($this->instance, $this->api);
     $twitter_crawler->api->to->setDataPathFolder('testoftwittercrawler/anildash/');
     $twitter_crawler->fetchInstanceUserTweets();
     $post_dao = DAOFactory::getDAO('PostDAO');
     $tweets = $post_dao->getAllPostsByUsername('anildash', 'twitter');
     $this->debug(Utils::varDumpToString($tweets));
     $twitter_crawler->fetchStrayRepliedToTweets();
     $post = $post_dao->getPost('300389504274022400', 'twitter');
     $this->assertTrue(isset($post));
     $this->debug(Utils::varDumpToString($post));
     $this->assertEqual($post->reply_count_cache, 2);
 }