Ejemplo n.º 1
0
 public function control()
 {
     $this->view_name = isset($_GET['v']) ? $_GET['v'] : 'default';
     $post_dao = DAOFactory::getDAO('PostDAO');
     $this->setPageTitle('Post Details');
     $this->setViewTemplate('post.index.tpl');
     $network = isset($_GET['n']) ? $_GET['n'] : 'twitter';
     if ($this->shouldRefreshCache()) {
         if (isset($_GET['t']) && is_numeric($_GET['t'])) {
             $post_id = $_GET['t'];
             $post = $post_dao->getPost($post_id, $network);
             if (isset($post)) {
                 $config = Config::getInstance();
                 $this->addToView('disable_embed_code', $config->getValue('is_embed_disabled') || $post->is_protected);
                 if (isset($_GET['search'])) {
                     $this->addToView('search_on', true);
                 }
                 if (!$post->is_protected || $this->isLoggedIn()) {
                     $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
                     $options = $plugin_option_dao->getOptionsHash('geoencoder', true);
                     if (isset($options['distance_unit']->option_value)) {
                         $distance_unit = $options['distance_unit']->option_value;
                     } else {
                         $distance_unit = 'km';
                     }
                     $this->addToView('post', $post);
                     $this->addToView('unit', $distance_unit);
                     $replies = $post_dao->getRepliesToPost($post_id, $network, 'default', $distance_unit);
                     $public_replies = array();
                     foreach ($replies as $reply) {
                         if (!$reply->author->is_protected) {
                             $public_replies[] = $reply;
                         }
                     }
                     $public_replies_count = count($public_replies);
                     $this->addToView('public_reply_count', $public_replies_count);
                     if ($this->isLoggedIn()) {
                         $this->addToView('replies', $replies);
                     } else {
                         $this->addToView('replies', $public_replies);
                     }
                     $all_replies_count = count($replies);
                     $private_reply_count = $all_replies_count - $public_replies_count;
                     $this->addToView('private_reply_count', $private_reply_count);
                     $webapp = Webapp::getInstance();
                     $sidebar_menu = $webapp->getPostDetailMenu($post);
                     $this->addToView('sidebar_menu', $sidebar_menu);
                     $this->loadView($post);
                 } else {
                     $this->addErrorMessage('Insufficient privileges');
                 }
             } else {
                 $this->addErrorMessage('Post not found');
             }
         } else {
             $this->addErrorMessage('Post not specified');
         }
     }
     return $this->generateView();
 }
Ejemplo n.º 2
0
 public function generateInsight(Instance $instance, $last_week_of_posts, $number_days)
 {
     parent::generateInsight($instance, $last_week_of_posts, $number_days);
     $this->logger->logInfo("Begin generating insight", __METHOD__ . ',' . __LINE__);
     $filename = basename(__FILE__, ".php");
     foreach ($last_week_of_posts as $post) {
         $simplified_post_date = date('Y-m-d', strtotime($post->pub_date));
         // Map insight: If not a reply or retweet and geoencoded, show the map in the stream
         if (!isset($post->in_reply_to_user_id) && !isset($post->in_reply_to_post_id) && !isset($post->in_retweet_of_post_id) && $post->reply_count_cache > 5) {
             $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
             $options = $plugin_option_dao->getOptionsHash('geoencoder', true);
             if (isset($options['gmaps_api_key']->option_value) && $post->is_geo_encoded == 1) {
                 //Get post's replies and loop through them to make sure at least 5 are indeed geoencoded
                 $post_dao = DAOFactory::getDAO('PostDAO');
                 $post_replies = $post_dao->getRepliesToPost($post->post_id, $post->network);
                 $total_geoencoded_replies = 0;
                 foreach ($post_replies as $reply) {
                     if ($reply->is_geo_encoded == 1) {
                         $total_geoencoded_replies++;
                     }
                 }
                 if ($total_geoencoded_replies > 4) {
                     $this->insight_dao->insertInsight('geoencoded_replies', $instance->id, $simplified_post_date, "Going global!", "{$this->username}'s post got responses from locations all over the map.", $filename, Insight::EMPHASIS_LOW, serialize($post));
                 }
             }
         }
     }
     $this->logger->logInfo("Done generating insight", __METHOD__ . ',' . __LINE__);
 }
Ejemplo n.º 3
0
    /**
     * Main control method
     * @return Page markup
     */
    public function control() {
        if ($this->shouldRefreshCache()) {
            $this->setViewTemplate(Utils::getPluginViewDirectory('geoencoder').'geoencoder.map.iframe.tpl');
            $this->setPageTitle('Locate Post on Map');
            $post_dao = DAOFactory::getDAO('PostDAO');

            $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
            $options = $plugin_option_dao->getOptionsHash('geoencoder', true);
            if (isset($options['gmaps_api_key']->option_value)) {
                $api_key = $options['gmaps_api_key']->option_value;
            } else {
                $api_key = NULL;
            }
            $network = (isset($_GET['n']))?$_GET['n']:'twitter';
            $type = (isset($_GET['t']))?$_GET['t']:'post';
            $post_id = (isset($_GET['pid']))?$_GET['pid']:'post_id';
            $post = $post_dao->getPost($post_id, $network);

            if ($type == 'post' && isset($post) && $post->is_geo_encoded == 1) {
                $this->addToView('post', $post);

                $this->addHeaderJavaScript('plugins/geoencoder/assets/js/generatemap.js');
                $this->addToView('gmaps_api', $api_key);

                $post_rows =  $post_dao->getRelatedPostsArray($post_id, $network, !$this->isLoggedIn());

                $posts_json = $this->processLocations($post_rows, $post_id);
                $this->addToView('posts_data', $posts_json);
            } else {
                $this->addErrorMessage('This post has not been geoencoded yet; cannot display map.');
            }
        }
        return $this->generateView();
    }
 /**
  * Load view with individual post and replies and retweets
  * @param int $post_id
  * @param str $network
  */
 private function loadSinglePostThread($post_id, $network)
 {
     $this->setPageTitle('Public Post Replies');
     $this->post_dao->isPostByPublicInstance($_GET['t'], $network);
     $post = $this->post_dao->getPost($post_id, $network);
     if (!isset($post)) {
         $this->addErrorMessage("Post " . $post_id . " on " . ucwords($network) . " is not in ThinkUp.");
     } else {
         $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
         $options = $plugin_option_dao->getOptionsHash('geoencoder', true);
         if (isset($options['distance_unit']->option_value)) {
             $distance_unit = $options['distance_unit']->option_value;
         } else {
             $distance_unit = 'km';
         }
         $public_tweet_replies = $this->post_dao->getPublicRepliesToPost($post->post_id, $network, 'default', $distance_unit);
         $public_retweets = $this->post_dao->getRetweetsOfPost($post->post_id, $network, 'default', $distance_unit, true);
         $this->addToView('post', $post);
         $this->addToView('replies', $public_tweet_replies);
         $this->addToView('retweets', $public_retweets);
         $this->addToView('unit', $distance_unit);
         $rtreach = 0;
         foreach ($public_retweets as $t) {
             $rtreach += $t->author->follower_count;
         }
         $this->addToView('rtreach', $rtreach);
     }
 }
Ejemplo n.º 5
0
 public function crawl()
 {
     $logger = Logger::getInstance();
     $config = Config::getInstance();
     $instance_dao = DAOFactory::getDAO('InstanceDAO');
     $owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
     $owner_dao = DAOFactory::getDAO('OwnerDAO');
     $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
     $options = $plugin_option_dao->getOptionsHash('googleplus', true);
     //get cached
     $current_owner = $owner_dao->getByEmail(Session::getLoggedInUser());
     //crawl Google+ users
     $instances = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'google+');
     if (isset($options['google_plus_client_id']->option_value) && isset($options['google_plus_client_secret']->option_value)) {
         foreach ($instances as $instance) {
             $logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username);
             $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
             $tokens = $owner_instance_dao->getOAuthTokens($instance->id);
             $access_token = $tokens['oauth_access_token'];
             $refresh_token = $tokens['oauth_access_token_secret'];
             $instance_dao->updateLastRun($instance->id);
             $google_plus_crawler = new GooglePlusCrawler($instance, $access_token);
             $dashboard_module_cacher = new DashboardModuleCacher($instance);
             try {
                 $google_plus_crawler->initializeInstanceUser($options['google_plus_client_id']->option_value, $options['google_plus_client_secret']->option_value, $access_token, $refresh_token, $current_owner->id);
                 $google_plus_crawler->fetchInstanceUserPosts();
             } catch (Exception $e) {
                 $logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__);
             }
             $dashboard_module_cacher->cacheDashboardModules();
             $instance_dao->save($google_plus_crawler->instance, 0, $logger);
             $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
         }
     }
 }
Ejemplo n.º 6
0
 /**
  * Run when the crawler does
  */
 public function crawl()
 {
     $this->logger->setUsername(null);
     $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
     $options = $plugin_option_dao->getOptionsHash('expandurls', true);
     //Limit the number of links expanded each crawl
     $this->link_limit = isset($options['links_to_expand']->option_value) ? (int) $options['links_to_expand']->option_value : 1500;
     if ($this->link_limit != 0) {
         //set Flickr API key
         if (isset($options['flickr_api_key']->option_value)) {
             $flickr_api_key = $options['flickr_api_key']->option_value;
         } else {
             $flickr_api_key = null;
         }
         //Get short links into table via initial expansion
         self::expandOriginalURLs($flickr_api_key);
         //Capture click counts for bit.ly URLs
         if (isset($options['bitly_api_key']->option_value, $options['bitly_login']->option_value)) {
             self::acquireBitlyClickStats($options['bitly_api_key']->option_value, $options['bitly_login']->option_value);
         }
         //@TODO Capture click counts for goo.gl URLs
         //self::acquireGooglClickStats
     } else {
         $this->logger->logUserInfo("Limit of links to expand reached.", __METHOD__ . ',' . __LINE__);
     }
 }
Ejemplo n.º 7
0
 /**
  * Main control method
  * @return Page markup
  */
 public function control()
 {
     if ($this->shouldRefreshCache()) {
         $post_dao = DAOFactory::getDAO('PostDAO');
         $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
         $options = $plugin_option_dao->getOptionsHash('geoencoder', true);
         if (isset($options['gmaps_api_key']->option_value)) {
             $api_key = $options['gmaps_api_key']->option_value;
         } else {
             $api_key = NULL;
         }
         $network = isset($_GET['n']) ? $_GET['n'] : 'twitter';
         $type = isset($_GET['t']) ? $_GET['t'] : 'post';
         $post_id = isset($_GET['pid']) ? $_GET['pid'] : 'post_id';
         if ($type == 'post' && $post_dao->isPostInDB($post_id, $network)) {
             $this->addHeaderJavaScript('plugins/geoencoder/assets/js/generatemap.js');
             $this->addToView('gmaps_api', $api_key);
             $post = $post_dao->getPost($post_id, $network);
             $this->addToView('post', $post);
             $post_rows = $post_dao->getRelatedPosts($post_id, $network);
             $posts_json = $this->processLocations($post_rows, $post_id);
             $this->addToView('posts_data', $posts_json);
         } else {
             $this->addErrorMessage('No visualization data found for this post');
         }
     }
     return $this->generateView();
 }
 public function crawl()
 {
     $config = Config::getInstance();
     $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
     $options = $plugin_option_dao->getOptionsHash('flickrthumbnails', true);
     $api_key = $options['flickr_api_key']->option_value;
     if (isset($api_key) && $api_key != '') {
         $logger = Logger::getInstance();
         $fa = new FlickrAPIAccessor($api_key);
         $ldao = DAOFactory::getDAO('LinkDAO');
         $flickrlinkstoexpand = $ldao->getLinksToExpandByURL('http://flic.kr/');
         if (count($flickrlinkstoexpand > 0)) {
             $logger->logStatus(count($flickrlinkstoexpand) . " Flickr links to expand", "Flickr Plugin");
         } else {
             $logger->logStatus("No Flickr links to expand", "Flickr Plugin");
         }
         foreach ($flickrlinkstoexpand as $fl) {
             $eurl = $fa->getFlickrPhotoSource($fl);
             if ($eurl["expanded_url"] != '') {
                 $ldao->saveExpandedUrl($fl, $eurl["expanded_url"], '', 1);
             } elseif ($eurl["error"] != '') {
                 $ldao->saveExpansionError($fl, $eurl["error"]);
             }
         }
         $logger->close();
         # Close logging
     }
 }
Ejemplo n.º 9
0
 public function crawl()
 {
     $config = Config::getInstance();
     $logger = Logger::getInstance();
     $instance_dao = DAOFactory::getDAO('TwitterInstanceDAO');
     $owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
     $owner_dao = DAOFactory::getDAO('OwnerDAO');
     $instance_hashtag_dao = DAOFactory::getDAO('InstanceHashtagDAO');
     // get oauth values
     $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
     $options = $plugin_option_dao->getOptionsHash('twitter', true);
     $current_owner = $owner_dao->getByEmail(Session::getLoggedInUser());
     $instances = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'twitter');
     foreach ($instances as $instance) {
         $logger->setUsername($instance->network_username);
         $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . " on Twitter.", __METHOD__ . ',' . __LINE__);
         $tokens = $owner_instance_dao->getOAuthTokens($instance->id);
         $num_twitter_errors = isset($options['num_twitter_errors']) ? $options['num_twitter_errors']->option_value : null;
         $dashboard_module_cacher = new DashboardModuleCacher($instance);
         try {
             if (isset($tokens['oauth_access_token']) && $tokens['oauth_access_token'] != '' && isset($tokens['oauth_access_token_secret']) && $tokens['oauth_access_token_secret'] != '') {
                 $archive_limit = isset($options['archive_limit']->option_value) ? $options['archive_limit']->option_value : 3200;
                 $api = new CrawlerTwitterAPIAccessorOAuth($tokens['oauth_access_token'], $tokens['oauth_access_token_secret'], $options['oauth_consumer_key']->option_value, $options['oauth_consumer_secret']->option_value, $archive_limit, $num_twitter_errors);
                 $twitter_crawler = new TwitterCrawler($instance, $api);
                 $instance_dao->updateLastRun($instance->id);
                 $twitter_crawler->fetchInstanceUserTweets();
                 $twitter_crawler->fetchInstanceUserMentions();
                 $twitter_crawler->fetchInstanceUserFriends();
                 $twitter_crawler->fetchInstanceUserFollowers();
                 $twitter_crawler->fetchInstanceUserGroups();
                 $twitter_crawler->fetchRetweetsOfInstanceUser();
                 $twitter_crawler->fetchInstanceUserFavorites();
                 $twitter_crawler->updateStaleGroupMemberships();
                 $twitter_crawler->fetchStrayRepliedToTweets();
                 $twitter_crawler->fetchUserFriendsByIDs();
                 $twitter_crawler->fetchUnloadedFriendDetails();
                 $twitter_crawler->fetchUnloadedFollowerDetails();
                 $twitter_crawler->cleanUpFollows();
                 $twitter_crawler->updateFriendsProfiles();
                 //Retrieve search results for saved keyword/hashtags
                 $instances_hashtags = $instance_hashtag_dao->getByInstance($instance->id);
                 foreach ($instances_hashtags as $instance_hashtag) {
                     $twitter_crawler->fetchInstanceHashtagTweets($instance_hashtag);
                 }
             } else {
                 throw new Exception('Missing Twitter OAuth tokens.');
             }
         } catch (Exception $e) {
             $logger->logUserError(get_class($e) . " while crawling " . $instance->network_username . " on Twitter: " . $e->getMessage(), __METHOD__ . ',' . __LINE__);
         }
         $dashboard_module_cacher->cacheDashboardModules();
         // Save instance
         if (isset($twitter_crawler->user)) {
             $instance_dao->save($instance, $twitter_crawler->user->post_count, $logger);
         }
         Reporter::reportVersion($instance);
         $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . " on Twitter.", __METHOD__ . ',' . __LINE__);
     }
 }
Ejemplo n.º 10
0
 /**
  * Constructor
  * @param Instance $instance
  * @param CrawlerTwitterAPIAccessorOAuth $api
  * @return TwitterCrawler
  */
 public function __construct($instance, $api) {
     $this->instance = $instance;
     $this->api = $api;
     $this->logger = Logger::getInstance();
     $this->logger->setUsername($instance->network_username);
     $this->user_dao = DAOFactory::getDAO('UserDAO');
     $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
     $this->twitter_options = $plugin_option_dao->getOptionsHash('twitter');
 }
Ejemplo n.º 11
0
 public function crawl()
 {
     $logger = Logger::getInstance();
     $config = Config::getInstance();
     $instance_dao = DAOFactory::getDAO('InstagramInstanceDAO');
     $owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
     $owner_dao = DAOFactory::getDAO('OwnerDAO');
     $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
     $options = $plugin_option_dao->getOptionsHash('instagram', true);
     //get cached
     $max_api_calls = isset($options['max_api_calls']) ? $options['max_api_calls']->option_value : 2500;
     $current_owner = $owner_dao->getByEmail(Session::getLoggedInUser());
     //crawl instagram user profiles and pages
     $instances = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'instagram');
     foreach ($instances as $instance) {
         $logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username);
         $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
         $tokens = $owner_instance_dao->getOAuthTokens($instance->id);
         $access_token = $tokens['oauth_access_token'];
         $instance_dao->updateLastRun($instance->id);
         $dashboard_module_cacher = new DashboardModuleCacher($instance);
         try {
             /**
              * 1. Fetch user info, media + its likes and comments.
              * 2. Fetch user's likes.
              * 3. Fetch user's friends, and update stale relationships.
              * 4. Fetch user's followers, and update stale relationships.
              * 5. Update stale friends' profiles.
              */
             $instagram_crawler = new InstagramCrawler($instance, $access_token, $max_api_calls);
             $instagram_crawler->fetchPostsAndReplies();
             $instagram_crawler->fetchLikes();
             $instagram_crawler->fetchFriends();
             $instagram_crawler->fetchFollowers();
             $instagram_crawler->updateStaleFriendsProfiles();
         } catch (Instagram\Core\ApiAuthException $e) {
             //The access token is invalid, save in owner_instances table
             $owner_instance_dao->setAuthErrorByTokens($instance->id, $access_token, '', $e->getMessage());
             //Send email alert
             //Get owner by auth tokens first, then send to that person
             $owner_email_to_notify = $owner_instance_dao->getOwnerEmailByInstanceTokens($instance->id, $access_token, '');
             $email_attempt = $this->sendInvalidOAuthEmailAlert($owner_email_to_notify, $instance->network_username);
             if ($email_attempt) {
                 $logger->logUserInfo('Sent reauth email to ' . $owner_email_to_notify, __METHOD__ . ',' . __LINE__);
             } else {
                 $logger->logInfo('Didn\'t send reauth email to ' . $owner_email_to_notify, __METHOD__ . ',' . __LINE__);
             }
             $logger->logUserError(get_class($e) . ' ' . $e->getMessage(), __METHOD__ . ',' . __LINE__);
         } catch (Exception $e) {
             $logger->logUserError(get_class($e) . ' ' . $e->getMessage(), __METHOD__ . ',' . __LINE__);
         }
         $dashboard_module_cacher->cacheDashboardModules();
         $instance_dao->save($instagram_crawler->instance, 0, $logger);
         Reporter::reportVersion($instance);
         $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
     }
 }
Ejemplo n.º 12
0
 public function crawl()
 {
     $logger = Logger::getInstance();
     $config = Config::getInstance();
     $instance_dao = DAOFactory::getDAO('InstanceDAO');
     $owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
     $owner_dao = DAOFactory::getDAO('OwnerDAO');
     $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
     $options = $plugin_option_dao->getOptionsHash('foursquare', true);
     $current_owner = $owner_dao->getByEmail(Session::getLoggedInUser());
     $instances = $instance_dao->getAllActiveInstancesStalestFirstByNetwork('foursquare');
     // Check the client id and secret are set or we can't crawl
     if (isset($options['foursquare_client_id']->option_value) && isset($options['foursquare_client_secret']->option_value)) {
         // For each instance of foursquare on this install
         foreach ($instances as $instance) {
             if (!$owner_instance_dao->doesOwnerHaveAccessToInstance($current_owner, $instance)) {
                 // Owner doesn't have access to this instance; let's not crawl it.
                 continue;
             }
             // Set the user name in the log
             $logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username);
             // Write to the log that we have started to collect data
             $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
             // Get the OAuth tokens for this user
             $tokens = $owner_instance_dao->getOAuthTokens($instance->id);
             // Set the access token
             $access_token = $tokens['oauth_access_token'];
             // Update the last time we crawled
             $instance_dao->updateLastRun($instance->id);
             // Create a new crawler
             $crawler = new FoursquareCrawler($instance, $access_token);
             // Check the OAuth tokens we have are valid
             try {
                 $logger->logInfo("About to initializeInstanceUser", __METHOD__ . ',' . __LINE__);
                 $user = $crawler->initializeInstanceUser($access_token, $current_owner->id);
                 if (isset($user) && $user instanceof User) {
                     $logger->logInfo("User initialized", __METHOD__ . ',' . __LINE__);
                 }
                 // Get the data we want and store it in the database
                 $logger->logInfo("About to fetchInstanceUserCheckins", __METHOD__ . ',' . __LINE__);
                 $crawler->fetchInstanceUserCheckins();
             } catch (Exception $e) {
                 // Catch any errors that happen when we check the validity of the OAuth tokens
                 $logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__);
             }
             $logger->logInfo("About to cache dashboard modules", __METHOD__ . ',' . __LINE__);
             // Cache the insights to improve the page load speed of the dashboard
             $dashboard_module_cacher = new DashboardModuleCacher($instance);
             $dashboard_module_cacher->cacheDashboardModules();
             $instance_dao->save($crawler->instance, 0, $logger);
             Reporter::reportVersion($instance);
             // Tell the user crawling was sucessful
             $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
         }
     }
 }
Ejemplo n.º 13
0
 public function crawl()
 {
     $logger = Logger::getInstance();
     $config = Config::getInstance();
     $instance_dao = DAOFactory::getDAO('FacebookInstanceDAO');
     $owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
     $owner_dao = DAOFactory::getDAO('OwnerDAO');
     $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
     $options = $plugin_option_dao->getOptionsHash('facebook', true);
     //get cached
     $max_crawl_time = isset($options['max_crawl_time']) ? $options['max_crawl_time']->option_value : 20;
     //convert to seconds
     $max_crawl_time = $max_crawl_time * 60;
     $current_owner = $owner_dao->getByEmail(Session::getLoggedInUser());
     //crawl Facebook user profiles and pages
     $profiles = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'facebook');
     $pages = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'facebook page');
     $instances = array_merge($profiles, $pages);
     foreach ($instances as $instance) {
         $logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username);
         $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
         $tokens = $owner_instance_dao->getOAuthTokens($instance->id);
         $access_token = $tokens['oauth_access_token'];
         $instance_dao->updateLastRun($instance->id);
         $facebook_crawler = new FacebookCrawler($instance, $access_token, $max_crawl_time);
         $dashboard_module_cacher = new DashboardModuleCacher($instance);
         try {
             $facebook_crawler->fetchPostsAndReplies();
         } catch (APIOAuthException $e) {
             $logger->logUserError(get_class($e) . ": " . $e->getMessage(), __METHOD__ . ',' . __LINE__);
             //Don't send reauth email if it's app-level API rate limting
             //https://developers.facebook.com/docs/reference/ads-api/api-rate-limiting/#applimit
             if (strpos($e->getMessage(), 'Application request limit reached') === false && strpos($e->getMessage(), 'Please retry your request later.') === false) {
                 //The access token is invalid, save in owner_instances table
                 $owner_instance_dao->setAuthErrorByTokens($instance->id, $access_token, '', $e->getMessage());
                 //Send email alert
                 //Get owner by auth tokens first, then send to that person
                 $owner_email_to_notify = $owner_instance_dao->getOwnerEmailByInstanceTokens($instance->id, $access_token, '');
                 $email_attempt = $this->sendInvalidOAuthEmailAlert($owner_email_to_notify, $instance->network_username);
                 if ($email_attempt) {
                     $logger->logUserInfo('Sent reauth email to ' . $owner_email_to_notify, __METHOD__ . ',' . __LINE__);
                 } else {
                     $logger->logInfo('Didn\'t send reauth email to ' . $owner_email_to_notify, __METHOD__ . ',' . __LINE__);
                 }
             } else {
                 $logger->logInfo('Facebook API returned an error: ' . $e->getMessage() . ' Do nothing now and try again later', __METHOD__ . ',' . __LINE__);
             }
         } catch (Exception $e) {
             $logger->logUserError(get_class($e) . ": " . $e->getMessage(), __METHOD__ . ',' . __LINE__);
         }
         $dashboard_module_cacher->cacheDashboardModules();
         $instance_dao->save($facebook_crawler->instance, 0, $logger);
         Reporter::reportVersion($instance);
         $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
     }
 }
 /**
  * @return str
  */
 public function authControl()
 {
     $config = Config::getInstance();
     Loader::definePathConstants();
     $this->setViewTemplate(THINKUP_WEBAPP_PATH . 'plugins/twitterrealtime/view/twitterrealtime.account.index.tpl');
     $this->view_mgr->addHelp('twitterrealtime', 'userguide/settings/plugins/twitterrealtime');
     $id = DAOFactory::getDAO('InstanceDAO');
     $od = DAOFactory::getDAO('OwnerDAO');
     // get plugin option values if defined...
     $plugin_options = $this->getPluginOptions();
     // get oauth option values from twitter plugin.
     // @TODO -- what is the right way to do this?
     $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
     $twitter_options = $plugin_option_dao->getOptionsHash('twitter', true);
     $oauth_consumer_key = null;
     if (isset($twitter_options['oauth_consumer_key'])) {
         $oauth_consumer_key = $twitter_options['oauth_consumer_key']->option_value;
     }
     $oauth_consumer_secret = null;
     if (isset($twitter_options['oauth_consumer_secret'])) {
         $oauth_consumer_secret = $twitter_options['oauth_consumer_secret']->option_value;
     }
     // @TODO - get any other option values as necessary
     // $archive_limit = $this->getPluginOption('archive_limit');
     $auth_from_twitter = '';
     if (isset($oauth_consumer_key) && isset($oauth_consumer_secret)) {
         $to = new TwitterOAuth($oauth_consumer_key, $oauth_consumer_secret);
         /* Request tokens from twitter */
         $tok = $to->getRequestToken();
         if (isset($tok['oauth_token'])) {
             $token = $tok['oauth_token'];
             $_SESSION['oauth_request_token_secret'] = $tok['oauth_token_secret'];
             /* Build the authorization URL */
             $oauthorize_link = $to->getAuthorizeURL($token);
             // create indication that auth from twitter plugin is okay
             $auth_from_twitter = "Using the Twitter Consumer key and secret as set in " . "the <a href=\"./?p=twitter\">Twitter plugin</a>.";
         } else {
             //set error message here
             $this->addErrorMessage("Unable to obtain OAuth token. Check your Twitter plugin consumer key and secret configuration.");
             $oauthorize_link = '';
         }
     } else {
         $this->addErrorMessage("Missing required settings! Please configure the Twitter plugin.");
         $oauthorize_link = '';
     }
     $owner_instances = $id->getByOwnerAndNetwork($this->owner, 'twitter');
     $this->addToView('owner_instances', $owner_instances);
     $this->addToView('oauthorize_link', $oauthorize_link);
     $this->addToView('auth_from_twitter', $auth_from_twitter);
     // add plugin options from
     $this->addOptionForm();
     $plugin = new TwitterRealtimePlugin();
     $this->addToView('is_configured', $plugin->isConfigured());
     return $this->generateView();
 }
Ejemplo n.º 15
0
 public function authControl()
 {
     if (!$this->is_missing_param) {
         $fb_user = null;
         $msg = '';
         $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
         $options = $plugin_option_dao->getOptionsHash('facebook', true);
         //get cached
         $facebook = new Facebook($options['facebook_api_key']->option_value, $options['facebook_api_secret']->option_value);
         $fb_user = $facebook->api_client->users_getLoggedInUser();
         $msg .= "Facebook user is logged in and user ID set<br />";
         $fb_username = $facebook->api_client->users_getInfo($fb_user, 'name');
         if (isset($_GET['sessionKey']) && isset($fb_user) && $fb_user > 0) {
             $fb_username = $fb_username[0]['name'];
             $session_key = $_GET['sessionKey'];
             $owner_dao = DAOFactory::getDAO('OwnerDAO');
             $instance_dao = DAOFactory::getDAO('InstanceDAO');
             $oid = DAOFactory::getDAO('OwnerInstanceDAO');
             $user_dao = DAOFactory::getDAO('UserDAO');
             $owner = $owner_dao->getByEmail($this->getLoggedInUser());
             $i = $instance_dao->getByUserIdOnNetwork($fb_user, 'facebook');
             if (isset($i)) {
                 $msg .= "Instance exists<br />";
                 $oi = $oid->get($owner->id, $i->id);
                 if ($oi == null) {
                     //Instance already exists, owner instance doesn't
                     $oid->insert($owner->id, $i->id, $session_key);
                     //Add owner instance with session key
                     $msg .= "Created owner instance.<br />";
                 }
             } else {
                 //Instance does not exist
                 $msg .= "Instance does not exist<br />";
                 $instance_dao->insert($fb_user, $fb_username, 'facebook');
                 $msg .= "Created instance";
                 $i = $instance_dao->getByUserIdOnNetwork($fb_user, 'facebook');
                 $oid->insert($owner->id, $i->id, $session_key);
                 $msg .= "Created owner instance.<br />";
             }
             if (!$user_dao->isUserInDB($fb_user, 'facebook')) {
                 $r = array('user_id' => $fb_user, 'user_name' => $fb_username, 'full_name' => $fb_username, 'avatar' => '', 'location' => '', 'description' => '', 'url' => '', 'is_protected' => '', 'follower_count' => 0, 'friend_count' => 0, 'post_count' => 0, 'last_updated' => '', 'last_post' => '', 'joined' => '', 'last_post_id' => '', 'network' => 'facebook');
                 $u = new User($r, 'Owner info');
                 $user_dao->updateUser($u);
             }
         } else {
             $msg .= "No session key or logged in Facebook user.";
         }
         $config = Config::getInstance();
         $msg .= '<br /> <a href="' . $config->getValue('site_root_path') . 'account/">Back to your account</a>.';
         $this->addInfoMessage($msg);
     }
     return $this->generateView();
 }
Ejemplo n.º 16
0
 public function authControl()
 {
     if (!$this->is_missing_param) {
         $request_token = $_GET['oauth_token'];
         $request_token_secret = SessionCache::get('oauth_request_token_secret');
         // get oauth values
         $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
         $options = $plugin_option_dao->getOptionsHash('twitter', true);
         //get cached
         $to = new TwitterOAuth($options['oauth_consumer_key']->option_value, $options['oauth_consumer_secret']->option_value, $request_token, $request_token_secret);
         $tok = $to->getAccessToken();
         if (isset($tok['oauth_token']) && isset($tok['oauth_token_secret'])) {
             $api = new TwitterAPIAccessorOAuth($tok['oauth_token'], $tok['oauth_token_secret'], $options['oauth_consumer_key']->option_value, $options['oauth_consumer_secret']->option_value, $options['num_twitter_errors']->option_value, $options['max_api_calls_per_crawl']->option_value, false);
             $authed_twitter_user = $api->verifyCredentials();
             //                echo "User ID: ". $authed_twitter_user['user_id'];
             //                echo "User name: ". $authed_twitter_user['user_name'];
             $owner_dao = DAOFactory::getDAO('OwnerDAO');
             $owner = $owner_dao->getByEmail($this->getLoggedInUser());
             if ((int) $authed_twitter_user['user_id'] > 0) {
                 $instance_dao = DAOFactory::getDAO('TwitterInstanceDAO');
                 $instance = $instance_dao->getByUsername($authed_twitter_user['user_name'], 'twitter');
                 $owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
                 if (isset($instance)) {
                     $owner_instance = $owner_instance_dao->get($owner->id, $instance->id);
                     if ($owner_instance != null) {
                         $owner_instance_dao->updateTokens($owner->id, $instance->id, $tok['oauth_token'], $tok['oauth_token_secret']);
                         $this->addSuccessMessage($authed_twitter_user['user_name'] . " on Twitter is already set up in ThinkUp! To add a different Twitter account, " . "log out of Twitter.com in your browser and authorize ThinkUp again.");
                     } else {
                         if ($owner_instance_dao->insert($owner->id, $instance->id, $tok['oauth_token'], $tok['oauth_token_secret'])) {
                             $this->addSuccessMessage("Success! " . $authed_twitter_user['user_name'] . " on Twitter has been added to ThinkUp!");
                         } else {
                             $this->addErrorMessage("Error: Could not create an owner instance.");
                         }
                     }
                 } else {
                     $instance_dao->insert($authed_twitter_user['user_id'], $authed_twitter_user['user_name']);
                     $instance = $instance_dao->getByUsername($authed_twitter_user['user_name']);
                     if ($owner_instance_dao->insert($owner->id, $instance->id, $tok['oauth_token'], $tok['oauth_token_secret'])) {
                         $this->addSuccessMessage("Success! " . $authed_twitter_user['user_name'] . " on Twitter has been added to ThinkUp!");
                     } else {
                         $this->addErrorMessage("Error: Could not create an owner instance.");
                     }
                 }
             }
         } else {
             $msg = "Error: Twitter authorization did not complete successfully. Check if your account already " . " exists. If not, please try again.";
             $this->addErrorMessage($msg);
         }
         $this->view_mgr->clear_all_cache();
     }
     return $this->generateView();
 }
Ejemplo n.º 17
0
 public function crawl()
 {
     $logger = Logger::getInstance();
     $config = Config::getInstance();
     $id = DAOFactory::getDAO('InstanceDAO');
     $oid = DAOFactory::getDAO('OwnerInstanceDAO');
     $od = DAOFactory::getDAO('OwnerDAO');
     $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
     $options = $plugin_option_dao->getOptionsHash('facebook', true);
     //get cached
     $current_owner = $od->getByEmail(Session::getLoggedInUser());
     //crawl Facebook user profiles
     $instances = $id->getAllActiveInstancesStalestFirstByNetwork('facebook');
     foreach ($instances as $instance) {
         if (!$oid->doesOwnerHaveAccess($current_owner, $instance)) {
             // Owner doesn't have access to this instance; let's not crawl it.
             continue;
         }
         $logger->setUsername($instance->network_username);
         $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . " on Facebook.", __METHOD__ . ',' . __LINE__);
         $tokens = $oid->getOAuthTokens($instance->id);
         $access_token = $tokens['oauth_access_token'];
         $id->updateLastRun($instance->id);
         $crawler = new FacebookCrawler($instance, $access_token);
         try {
             $crawler->fetchInstanceUserInfo();
             $crawler->fetchUserPostsAndReplies($instance->network_user_id);
         } catch (Exception $e) {
             $logger->logUserError('PROFILE EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__);
         }
         $id->save($crawler->instance, 0, $logger);
         $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . " on Facebook.", __METHOD__ . ',' . __LINE__);
     }
     //crawl Facebook pages
     $instances = $id->getAllActiveInstancesStalestFirstByNetwork('facebook page');
     foreach ($instances as $instance) {
         $logger->setUsername($instance->network_username);
         $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s Facebook Page.", __METHOD__ . ',' . __LINE__);
         $tokens = $oid->getOAuthTokens($instance->id);
         $access_token = $tokens['oauth_access_token'];
         $id->updateLastRun($instance->id);
         $crawler = new FacebookCrawler($instance, $access_token);
         try {
             $crawler->fetchPagePostsAndReplies($instance->network_user_id);
         } catch (Exception $e) {
             $logger->logUserError('PAGE EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__);
         }
         $id->save($crawler->instance, 0, $logger);
         $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s Facebook Page.", __METHOD__ . ',' . __LINE__);
     }
 }
Ejemplo n.º 18
0
 public function crawl()
 {
     $logger = Logger::getInstance();
     $config = Config::getInstance();
     $id = DAOFactory::getDAO('InstanceDAO');
     $oid = DAOFactory::getDAO('OwnerInstanceDAO');
     $od = DAOFactory::getDAO('OwnerDAO');
     $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
     $options = $plugin_option_dao->getOptionsHash('facebook', true);
     //get cached
     $current_owner = $od->getByEmail(Session::getLoggedInUser());
     //crawl Facebook user profiles
     $instances = $id->getAllActiveInstancesStalestFirstByNetwork('facebook');
     foreach ($instances as $instance) {
         if (!$oid->doesOwnerHaveAccess($current_owner, $instance)) {
             // Owner doesn't have access to this instance; let's not crawl it.
             continue;
         }
         $logger->setUsername($instance->network_username);
         $tokens = $oid->getOAuthTokens($instance->id);
         $session_key = $tokens['oauth_access_token'];
         $fb = new Facebook($options['facebook_api_key']->option_value, $options['facebook_api_secret']->option_value);
         $id->updateLastRun($instance->id);
         $crawler = new FacebookCrawler($instance, $fb);
         try {
             $crawler->fetchInstanceUserInfo($instance->network_user_id, $session_key);
             $crawler->fetchUserPostsAndReplies($instance->network_user_id, $session_key);
         } catch (Exception $e) {
             $logger->logStatus('PROFILE EXCEPTION: ' . $e->getMessage(), get_class($this));
         }
         $id->save($crawler->instance, $crawler->owner_object->post_count, $logger);
     }
     //crawl Facebook pages
     $instances = $id->getAllActiveInstancesStalestFirstByNetwork('facebook page');
     foreach ($instances as $instance) {
         $logger->setUsername($instance->network_username);
         $tokens = $oid->getOAuthTokens($instance->id);
         $session_key = $tokens['oauth_access_token'];
         $fb = new Facebook($options['facebook_api_key']->option_value, $options['facebook_api_secret']->option_value);
         $id->updateLastRun($instance->id);
         $crawler = new FacebookCrawler($instance, $fb);
         try {
             $crawler->fetchPagePostsAndReplies($instance->network_user_id, $instance->network_viewer_id, $session_key);
         } catch (Exception $e) {
             $logger->logStatus('PAGE EXCEPTION: ' . $e->getMessage(), get_class($this));
         }
         $id->save($crawler->instance, 0, $logger);
     }
     $logger->close();
     # Close logging
 }
Ejemplo n.º 19
0
 /**
  * Run when the crawler does
  */
 public function crawl()
 {
     $logger = Logger::getInstance();
     $logger->setUsername(null);
     $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
     $options = $plugin_option_dao->getOptionsHash('expandurls', true);
     //Flickr image thumbnails
     if (isset($options['flickr_api_key']->option_value)) {
         self::expandFlickrThumbnails($options['flickr_api_key']->option_value);
     }
     //@TODO: Bit.ly URLs
     //Remaining URLs
     $link_limit = isset($options['links_to_expand']->option_value) ? (int) $options['links_to_expand']->option_value : 1500;
     self::expandRemainingURLs($link_limit);
 }
 public static function getQueue()
 {
     // get stream plugin settings
     $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
     $options = $plugin_option_dao->getOptionsHash('twitterrealtime', true);
     if (is_null(self::$queue)) {
         if (isset($options['use_redis']) && $options['use_redis']->option_value == "true") {
             // we need a redis queue
             self::$queue = new StreamMessageQueueRedis();
         } else {
             // we need a mysql queue
             self::$queue = new StreamMessageQueueMySQL();
         }
     }
     return self::$queue;
 }
 /**
  * @static
  * @param  $oauth_access_token
  * @param  $oauth_access_token_secret
  * @return ConsumerUserStream|null
  */
 public static function getInstance($oauth_access_token, $oauth_access_token_secret)
 {
     $sc = null;
     $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
     $options = $plugin_option_dao->getOptionsHash('twitter', true);
     //get cached
     if (isset($options['oauth_consumer_key']) && isset($options['oauth_consumer_secret'])) {
         define('TWITTER_CONSUMER_KEY', $options['oauth_consumer_key']->option_value);
         define('TWITTER_CONSUMER_SECRET', $options['oauth_consumer_secret']->option_value);
         $sc = new ConsumerUserStream($oauth_access_token, $oauth_access_token_secret);
     } else {
         $logger = Logger::getInstance('stream_log_location');
         $logger->logError("Error: could not obtain Twitter app consumer key and secret from Twitter plugin " . "settings.", __METHOD__ . ',' . __LINE__);
     }
     return $sc;
 }
Ejemplo n.º 22
0
 /**
  * Outputs javascript callback string with json array/list of post as an argument
  */
 public function authControl()
 {
     if (!$this->is_missing_param) {
         $instance_dao = DAOFactory::getDAO('InstanceDAO');
         if ($instance_dao->isUserConfigured($_GET['u'], $_GET['n'])) {
             $username = $_GET['u'];
             $ownerinstance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
             $owner_dao = DAOFactory::getDAO('OwnerDAO');
             $owner = $owner_dao->getByEmail($this->getLoggedInUser());
             $instance = $instance_dao->getByUsername($username, $_GET['n']);
             if (!$ownerinstance_dao->doesOwnerHaveAccess($owner, $instance)) {
                 echo '{"status":"failed","message":"Insufficient privileges."}';
             } else {
                 echo "tu_grid_search.populate_grid(";
                 $post_dao = DAOFactory::GetDAO('PostDAO');
                 $posts_it = null;
                 if ($_GET['d'] == 'tweets-all') {
                     $posts_it = $post_dao->getAllPostsByUsernameIterator($_GET['u'], $_GET['n'], self::MAX_ROWS);
                 } else {
                     if ($_GET['d'] == 'tweets-mostreplies') {
                         $posts_it = $post_dao->getAllMentionsIterator($_GET['u'], self::MAX_ROWS, $_GET['n']);
                     } else {
                         if ($_GET['d'] == 'tweets-mostretweeted') {
                             $posts_it = $post_dao->getMostRetweetedPostsIterator($_GET['u'], $_GET['n'], self::MAX_ROWS, self::MAX_RT_DAYS);
                         }
                     }
                 }
                 echo '{"status":"success","posts": [' . "\n";
                 $cnt = 0;
                 foreach ($posts_it as $key => $value) {
                     $cnt++;
                     $data = array('id' => $cnt, 'text' => $value->post_text, 'post_id' => $value->post_id, 'author' => $value->author_username, 'date' => $value->adj_pub_date);
                     echo json_encode($data) . ",\n";
                     flush();
                 }
                 $data = array('id' => -1, 'text' => 'Last Post', 'author' => 'nobody');
                 echo json_encode($data);
                 echo ']});';
             }
         } else {
             echo '{"status":"failed","message":"' . $_GET['u'] . 'is not configured."}';
         }
     } else {
         echo '{"status":"failed","message":"Missing Parameters"}';
     }
 }
Ejemplo n.º 23
0
 /**
  * Main control method
  */
 public function authControl()
 {
     $this->setViewTemplate('post.index.tpl');
     $network = isset($_GET['n']) ? $_GET['n'] : 'twitter';
     $_GET['n'] = $network;
     if ($this->shouldRefreshCache()) {
         if (isset($_GET['t']) && is_numeric($_GET['t']) && $this->post_dao->isPostInDB($_GET['t'], $network)) {
             $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
             $options = $plugin_option_dao->getOptionsHash('geoencoder', true);
             if (isset($options['distance_unit']->option_value)) {
                 $distance_unit = $options['distance_unit']->option_value;
             } else {
                 $distance_unit = 'km';
             }
             $post_id = $_GET['t'];
             $post = $this->post_dao->getPost($post_id, $network);
             $this->addToView('post', $post);
             $this->addToView('unit', $distance_unit);
             // costly query
             //$this->addToView('likely_orphans', $this->post_dao->getLikelyOrphansForParent($post->pub_date,
             //$post->author_user_id,$post->author_username, 15) );
             //$this->addToView('all_tweets', $this->post_dao->getAllPosts($post->author_user_id, 15) );
             $all_replies = $this->post_dao->getRepliesToPost($post_id, $network, 'default', $distance_unit);
             $this->addToView('replies', $all_replies);
             $all_replies_by_location = $this->post_dao->getRepliesToPost($post_id, $network, 'location', $distance_unit);
             $this->addToView('replies_by_location', $all_replies_by_location);
             $all_replies_count = count($all_replies);
             $this->addToView('reply_count', $all_replies_count);
             $all_retweets = $this->post_dao->getRetweetsOfPost($post_id, $network, 'default', $distance_unit);
             $this->addToView('retweets', $all_retweets);
             $all_retweets_by_location = $this->post_dao->getRetweetsOfPost($post_id, $network, 'location', $distance_unit);
             $this->addToView('retweets_by_location', $all_retweets_by_location);
             $retweet_reach = $this->post_dao->getPostReachViaRetweets($post_id, $network);
             $this->addToView('retweet_reach', $retweet_reach);
             $public_replies = $this->post_dao->getPublicRepliesToPost($post_id, $network);
             $public_replies_count = count($public_replies);
             $this->addToView('public_reply_count', $public_replies_count);
             $private_replies_count = $all_replies_count - $public_replies_count;
             $this->addToView('private_reply_count', $private_replies_count);
         } else {
             $this->addErrorMessage('Post not found');
         }
     }
     return $this->generateView();
 }
Ejemplo n.º 24
0
 /**
  * Constructor
  * @return StreamMasterCollect
  */
 public function __construct()
 {
     $config = Config::getInstance();
     $this->streaming_dir = $config->getValue('source_root_path') . 'webapp/plugins/twitterrealtime/streaming';
     // @TODO -- get this from plugin information now
     $this->log_dir = $config->getValue('source_root_path') . 'logs';
     $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
     $options = $plugin_option_dao->getOptionsHash('twitterrealtime', true);
     if (isset($options['php_path'])) {
         $this->php_path = $options['php_path']->option_value;
     } else {
         $this->php_path = null;
         // path to the php interp on the user's system
     }
     $this->stream_proc_dao = DAOFactory::getDAO('StreamProcDAO');
     $this->instance_dao = DAOFactory::getDAO('InstanceDAO');
     $this->owner_dao = DAOFactory::getDAO('OwnerDAO');
 }
Ejemplo n.º 25
0
 public function crawl()
 {
     $logger = Logger::getInstance();
     $config = Config::getInstance();
     $instance_dao = DAOFactory::getDAO('InstanceDAO');
     $owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
     $owner_dao = DAOFactory::getDAO('OwnerDAO');
     $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
     $options = $plugin_option_dao->getOptionsHash('facebook', true);
     //get cached
     $max_crawl_time = isset($options['max_crawl_time']) ? $options['max_crawl_time']->option_value : 20;
     //convert to seconds
     $max_crawl_time = $max_crawl_time * 60;
     $current_owner = $owner_dao->getByEmail(Session::getLoggedInUser());
     //crawl Facebook user profiles and pages
     $profiles = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'facebook');
     $pages = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'facebook page');
     $instances = array_merge($profiles, $pages);
     foreach ($instances as $instance) {
         $logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username);
         $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
         $tokens = $owner_instance_dao->getOAuthTokens($instance->id);
         $access_token = $tokens['oauth_access_token'];
         $instance_dao->updateLastRun($instance->id);
         $facebook_crawler = new FacebookCrawler($instance, $access_token, $max_crawl_time);
         $dashboard_module_cacher = new DashboardModuleCacher($instance);
         try {
             $facebook_crawler->fetchPostsAndReplies();
         } catch (APIOAuthException $e) {
             //The access token is invalid, save in owner_instances table
             $owner_instance_dao->setAuthError($current_owner->id, $instance->id, $e->getMessage());
             //Send email alert
             $this->sendInvalidOAuthEmailAlert($current_owner->email, $instance->network_username);
             $logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__);
         } catch (Exception $e) {
             $logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__);
         }
         $dashboard_module_cacher->cacheDashboardModules();
         $instance_dao->save($facebook_crawler->instance, 0, $logger);
         Reporter::reportVersion($instance);
         $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
     }
 }
Ejemplo n.º 26
0
 public function crawl()
 {
     $logger = Logger::getInstance();
     $config = Config::getInstance();
     $instance_dao = DAOFactory::getDAO('InstanceDAO');
     $owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
     $owner_dao = DAOFactory::getDAO('OwnerDAO');
     $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
     $options = $plugin_option_dao->getOptionsHash('youtube', true);
     //get cached
     $max_crawl_time = isset($options['max_crawl_time']) ? $options['max_crawl_time']->option_value : 20;
     //convert to seconds
     $max_crawl_time = $max_crawl_time * 60;
     $developer_key = isset($options['developer_key']) ? $options['developer_key']->option_value : null;
     $max_comments = isset($options['max_comments']) ? $options['max_comments']->option_value : null;
     $current_owner = $owner_dao->getByEmail(Session::getLoggedInUser());
     //crawl youtube users
     $instances = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'youtube');
     if (isset($options['youtube_client_id']->option_value) && isset($options['youtube_client_secret']->option_value)) {
         foreach ($instances as $instance) {
             $logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username);
             $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
             $tokens = $owner_instance_dao->getOAuthTokens($instance->id);
             $access_token = $tokens['oauth_access_token'];
             $refresh_token = $tokens['oauth_access_token_secret'];
             $instance_dao->updateLastRun($instance->id);
             $youtube_crawler = new YouTubeCrawler($instance, $access_token, $max_crawl_time, $developer_key, $max_comments);
             $dashboard_module_cacher = new DashboardModuleCacher($instance);
             try {
                 $youtube_crawler->initializeInstanceUser($options['youtube_client_id']->option_value, $options['youtube_client_secret']->option_value, $access_token, $refresh_token, $current_owner->id);
                 $youtube_crawler->fetchInstanceUserVideos();
             } catch (Exception $e) {
                 $logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__);
             }
             $dashboard_module_cacher->cacheDashboardModules();
             $instance_dao->save($youtube_crawler->instance, 0, $logger);
             Reporter::reportVersion($instance);
             $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
         }
     }
 }
Ejemplo n.º 27
0
    public function crawl() {
        $config = Config::getInstance();

        $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
        $options = $plugin_option_dao->getOptionsHash('flickrthumbnails', true);
        $api_key = '';
        if (isset($options['flickr_api_key']->option_value)) {
            $api_key =  $options['flickr_api_key']->option_value;
        }

        if (isset($api_key) && $api_key != '') {
            $logger = Logger::getInstance();
            $logger->setUsername(null);
            $fa = new FlickrAPIAccessor($api_key);
            $ldao = DAOFactory::getDAO('LinkDAO');

            $flickrlinkstoexpand = $ldao->getLinksToExpandByURL('http://flic.kr/');
            if (count($flickrlinkstoexpand) > 0) {
                $logger->logUserInfo(count($flickrlinkstoexpand)." Flickr links to expand.",  __METHOD__.','.__LINE__);
            } else {
                $logger->logUserInfo("There are no Flickr thumbnails to expand.",  __METHOD__.','.__LINE__);
            }

            $total_thumbnails = 0;
            $total_errors = 0;
            foreach ($flickrlinkstoexpand as $fl) {
                $eurl = $fa->getFlickrPhotoSource($fl);
                if ($eurl["expanded_url"] != '') {
                    $ldao->saveExpandedUrl($fl, $eurl["expanded_url"], '', 1);
                    $total_thumbnails = $total_thumbnails + 1;
                } elseif ($eurl["error"] != '') {
                    $ldao->saveExpansionError($fl, $eurl["error"]);
                    $total_errors = $total_errors + 1;
                }
                $logger->logUserSuccess($total_thumbnails." Flickr thumbnails expanded (".$total_errors." errors)",
                __METHOD__.','.__LINE__);
            }
        }
    }
Ejemplo n.º 28
0
 public function crawl()
 {
     $logger = Logger::getInstance();
     $config = Config::getInstance();
     $instance_dao = DAOFactory::getDAO('InstanceDAO');
     $owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
     $owner_dao = DAOFactory::getDAO('OwnerDAO');
     $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
     $options = $plugin_option_dao->getOptionsHash('facebook', true);
     //get cached
     $max_crawl_time = isset($options['max_crawl_time']) ? $options['max_crawl_time']->option_value : 20;
     //convert to seconds
     $max_crawl_time = $max_crawl_time * 60;
     $current_owner = $owner_dao->getByEmail(Session::getLoggedInUser());
     //crawl Facebook user profiles and pages
     $profiles = $instance_dao->getAllActiveInstancesStalestFirstByNetwork('facebook');
     $pages = $instance_dao->getAllActiveInstancesStalestFirstByNetwork('facebook page');
     $instances = array_merge($profiles, $pages);
     foreach ($instances as $instance) {
         if (!$owner_instance_dao->doesOwnerHaveAccessToInstance($current_owner, $instance)) {
             // Owner doesn't have access to this instance; let's not crawl it.
             continue;
         }
         $logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username);
         $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
         $tokens = $owner_instance_dao->getOAuthTokens($instance->id);
         $access_token = $tokens['oauth_access_token'];
         $instance_dao->updateLastRun($instance->id);
         $crawler = new FacebookCrawler($instance, $access_token, $max_crawl_time);
         try {
             $crawler->fetchPostsAndReplies();
         } catch (Exception $e) {
             $logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__);
         }
         $instance_dao->save($crawler->instance, 0, $logger);
         $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
     }
 }
Ejemplo n.º 29
0
 /**
  * Run when the crawler does
  */
 public function crawl()
 {
     $logger = Logger::getInstance();
     $logger->setUsername(null);
     $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
     $options = $plugin_option_dao->getOptionsHash('expandurls', true);
     //Limit the number of links expanded each crawl
     $this->link_limit = isset($options['links_to_expand']->option_value) ? (int) $options['links_to_expand']->option_value : 1500;
     if ($this->link_limit != 0) {
         //Flickr image thumbnails
         if (isset($options['flickr_api_key']->option_value)) {
             self::expandFlickrThumbnails($options['flickr_api_key']->option_value);
         }
         //Bit.ly URLs
         if (isset($options['bitly_api_key']->option_value, $options['bitly_login']->option_value)) {
             self::expandBitlyLinks($options['bitly_api_key']->option_value, $options['bitly_login']->option_value);
         }
         //Remaining links
         self::expandRemainingURLs();
     } else {
         $logger->logUserInfo("Limit of links to expand reached.", __METHOD__ . ',' . __LINE__);
     }
 }
Ejemplo n.º 30
0
    /**
     * Run when the crawler does
     * @TODO Set limit on total number of links to expand per crawler run in the plugin settings, for now 1500
     */
    public function crawl() {
        $logger = Logger::getInstance();
        $logger->setUsername(null);
        $ldao = DAOFactory::getDAO('LinkDAO');

        $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
        $options = $plugin_option_dao->getOptionsHash('expandurls', true);

        $total_links_to_expand = isset($options['links_to_expand']->option_value) ? 
        (int)$options['links_to_expand']->option_value : 1500;
        $linkstoexpand = $ldao->getLinksToExpand($total_links_to_expand);

        $logger->logUserInfo(count($linkstoexpand)." links to expand. Please wait. Working...",
        __METHOD__.','.__LINE__);

        $total_expanded = 0;
        $total_errors = 0;
        foreach ($linkstoexpand as $l) {
            if (Utils::validateURL($l)) {
                $logger->logInfo("Expanding ".($total_expanded+1). " of ".count($linkstoexpand)." (".$l.")",
                __METHOD__.','.__LINE__);

                $eurl = self::untinyurl($l, $ldao);
                if ($eurl != '') {
                    $ldao->saveExpandedUrl($l, $eurl);
                    $total_expanded = $total_expanded + 1;
                } else {
                    $total_errors = $total_errors + 1;
                }
            } else {
                $total_errors = $total_errors + 1;
                $logger->logError($l." is not a valid URL; skipping expansion", __METHOD__.','.__LINE__);
            }
        }
        $logger->logUserSuccess($total_expanded." URLs successfully expanded (".$total_errors." errors).",
        __METHOD__.','.__LINE__);
    }