public function control() { $this->view_name = isset($_GET['v']) ? $_GET['v'] : 'default'; $post_dao = DAOFactory::getDAO('PostDAO'); $this->setPageTitle('Post Details'); $this->setViewTemplate('post.index.tpl'); $network = isset($_GET['n']) ? $_GET['n'] : 'twitter'; if ($this->shouldRefreshCache()) { if (isset($_GET['t']) && is_numeric($_GET['t'])) { $post_id = $_GET['t']; $post = $post_dao->getPost($post_id, $network); if (isset($post)) { $config = Config::getInstance(); $this->addToView('disable_embed_code', $config->getValue('is_embed_disabled') || $post->is_protected); if (isset($_GET['search'])) { $this->addToView('search_on', true); } if (!$post->is_protected || $this->isLoggedIn()) { $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('geoencoder', true); if (isset($options['distance_unit']->option_value)) { $distance_unit = $options['distance_unit']->option_value; } else { $distance_unit = 'km'; } $this->addToView('post', $post); $this->addToView('unit', $distance_unit); $replies = $post_dao->getRepliesToPost($post_id, $network, 'default', $distance_unit); $public_replies = array(); foreach ($replies as $reply) { if (!$reply->author->is_protected) { $public_replies[] = $reply; } } $public_replies_count = count($public_replies); $this->addToView('public_reply_count', $public_replies_count); if ($this->isLoggedIn()) { $this->addToView('replies', $replies); } else { $this->addToView('replies', $public_replies); } $all_replies_count = count($replies); $private_reply_count = $all_replies_count - $public_replies_count; $this->addToView('private_reply_count', $private_reply_count); $webapp = Webapp::getInstance(); $sidebar_menu = $webapp->getPostDetailMenu($post); $this->addToView('sidebar_menu', $sidebar_menu); $this->loadView($post); } else { $this->addErrorMessage('Insufficient privileges'); } } else { $this->addErrorMessage('Post not found'); } } else { $this->addErrorMessage('Post not specified'); } } return $this->generateView(); }
public function generateInsight(Instance $instance, $last_week_of_posts, $number_days) { parent::generateInsight($instance, $last_week_of_posts, $number_days); $this->logger->logInfo("Begin generating insight", __METHOD__ . ',' . __LINE__); $filename = basename(__FILE__, ".php"); foreach ($last_week_of_posts as $post) { $simplified_post_date = date('Y-m-d', strtotime($post->pub_date)); // Map insight: If not a reply or retweet and geoencoded, show the map in the stream if (!isset($post->in_reply_to_user_id) && !isset($post->in_reply_to_post_id) && !isset($post->in_retweet_of_post_id) && $post->reply_count_cache > 5) { $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('geoencoder', true); if (isset($options['gmaps_api_key']->option_value) && $post->is_geo_encoded == 1) { //Get post's replies and loop through them to make sure at least 5 are indeed geoencoded $post_dao = DAOFactory::getDAO('PostDAO'); $post_replies = $post_dao->getRepliesToPost($post->post_id, $post->network); $total_geoencoded_replies = 0; foreach ($post_replies as $reply) { if ($reply->is_geo_encoded == 1) { $total_geoencoded_replies++; } } if ($total_geoencoded_replies > 4) { $this->insight_dao->insertInsight('geoencoded_replies', $instance->id, $simplified_post_date, "Going global!", "{$this->username}'s post got responses from locations all over the map.", $filename, Insight::EMPHASIS_LOW, serialize($post)); } } } } $this->logger->logInfo("Done generating insight", __METHOD__ . ',' . __LINE__); }
/** * Main control method * @return Page markup */ public function control() { if ($this->shouldRefreshCache()) { $this->setViewTemplate(Utils::getPluginViewDirectory('geoencoder').'geoencoder.map.iframe.tpl'); $this->setPageTitle('Locate Post on Map'); $post_dao = DAOFactory::getDAO('PostDAO'); $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('geoencoder', true); if (isset($options['gmaps_api_key']->option_value)) { $api_key = $options['gmaps_api_key']->option_value; } else { $api_key = NULL; } $network = (isset($_GET['n']))?$_GET['n']:'twitter'; $type = (isset($_GET['t']))?$_GET['t']:'post'; $post_id = (isset($_GET['pid']))?$_GET['pid']:'post_id'; $post = $post_dao->getPost($post_id, $network); if ($type == 'post' && isset($post) && $post->is_geo_encoded == 1) { $this->addToView('post', $post); $this->addHeaderJavaScript('plugins/geoencoder/assets/js/generatemap.js'); $this->addToView('gmaps_api', $api_key); $post_rows = $post_dao->getRelatedPostsArray($post_id, $network, !$this->isLoggedIn()); $posts_json = $this->processLocations($post_rows, $post_id); $this->addToView('posts_data', $posts_json); } else { $this->addErrorMessage('This post has not been geoencoded yet; cannot display map.'); } } return $this->generateView(); }
/** * Load view with individual post and replies and retweets * @param int $post_id * @param str $network */ private function loadSinglePostThread($post_id, $network) { $this->setPageTitle('Public Post Replies'); $this->post_dao->isPostByPublicInstance($_GET['t'], $network); $post = $this->post_dao->getPost($post_id, $network); if (!isset($post)) { $this->addErrorMessage("Post " . $post_id . " on " . ucwords($network) . " is not in ThinkUp."); } else { $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('geoencoder', true); if (isset($options['distance_unit']->option_value)) { $distance_unit = $options['distance_unit']->option_value; } else { $distance_unit = 'km'; } $public_tweet_replies = $this->post_dao->getPublicRepliesToPost($post->post_id, $network, 'default', $distance_unit); $public_retweets = $this->post_dao->getRetweetsOfPost($post->post_id, $network, 'default', $distance_unit, true); $this->addToView('post', $post); $this->addToView('replies', $public_tweet_replies); $this->addToView('retweets', $public_retweets); $this->addToView('unit', $distance_unit); $rtreach = 0; foreach ($public_retweets as $t) { $rtreach += $t->author->follower_count; } $this->addToView('rtreach', $rtreach); } }
public function crawl() { $logger = Logger::getInstance(); $config = Config::getInstance(); $instance_dao = DAOFactory::getDAO('InstanceDAO'); $owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO'); $owner_dao = DAOFactory::getDAO('OwnerDAO'); $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('googleplus', true); //get cached $current_owner = $owner_dao->getByEmail(Session::getLoggedInUser()); //crawl Google+ users $instances = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'google+'); if (isset($options['google_plus_client_id']->option_value) && isset($options['google_plus_client_secret']->option_value)) { foreach ($instances as $instance) { $logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username); $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__); $tokens = $owner_instance_dao->getOAuthTokens($instance->id); $access_token = $tokens['oauth_access_token']; $refresh_token = $tokens['oauth_access_token_secret']; $instance_dao->updateLastRun($instance->id); $google_plus_crawler = new GooglePlusCrawler($instance, $access_token); $dashboard_module_cacher = new DashboardModuleCacher($instance); try { $google_plus_crawler->initializeInstanceUser($options['google_plus_client_id']->option_value, $options['google_plus_client_secret']->option_value, $access_token, $refresh_token, $current_owner->id); $google_plus_crawler->fetchInstanceUserPosts(); } catch (Exception $e) { $logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__); } $dashboard_module_cacher->cacheDashboardModules(); $instance_dao->save($google_plus_crawler->instance, 0, $logger); $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__); } } }
/** * Run when the crawler does */ public function crawl() { $this->logger->setUsername(null); $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('expandurls', true); //Limit the number of links expanded each crawl $this->link_limit = isset($options['links_to_expand']->option_value) ? (int) $options['links_to_expand']->option_value : 1500; if ($this->link_limit != 0) { //set Flickr API key if (isset($options['flickr_api_key']->option_value)) { $flickr_api_key = $options['flickr_api_key']->option_value; } else { $flickr_api_key = null; } //Get short links into table via initial expansion self::expandOriginalURLs($flickr_api_key); //Capture click counts for bit.ly URLs if (isset($options['bitly_api_key']->option_value, $options['bitly_login']->option_value)) { self::acquireBitlyClickStats($options['bitly_api_key']->option_value, $options['bitly_login']->option_value); } //@TODO Capture click counts for goo.gl URLs //self::acquireGooglClickStats } else { $this->logger->logUserInfo("Limit of links to expand reached.", __METHOD__ . ',' . __LINE__); } }
/** * Main control method * @return Page markup */ public function control() { if ($this->shouldRefreshCache()) { $post_dao = DAOFactory::getDAO('PostDAO'); $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('geoencoder', true); if (isset($options['gmaps_api_key']->option_value)) { $api_key = $options['gmaps_api_key']->option_value; } else { $api_key = NULL; } $network = isset($_GET['n']) ? $_GET['n'] : 'twitter'; $type = isset($_GET['t']) ? $_GET['t'] : 'post'; $post_id = isset($_GET['pid']) ? $_GET['pid'] : 'post_id'; if ($type == 'post' && $post_dao->isPostInDB($post_id, $network)) { $this->addHeaderJavaScript('plugins/geoencoder/assets/js/generatemap.js'); $this->addToView('gmaps_api', $api_key); $post = $post_dao->getPost($post_id, $network); $this->addToView('post', $post); $post_rows = $post_dao->getRelatedPosts($post_id, $network); $posts_json = $this->processLocations($post_rows, $post_id); $this->addToView('posts_data', $posts_json); } else { $this->addErrorMessage('No visualization data found for this post'); } } return $this->generateView(); }
public function crawl() { $config = Config::getInstance(); $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('flickrthumbnails', true); $api_key = $options['flickr_api_key']->option_value; if (isset($api_key) && $api_key != '') { $logger = Logger::getInstance(); $fa = new FlickrAPIAccessor($api_key); $ldao = DAOFactory::getDAO('LinkDAO'); $flickrlinkstoexpand = $ldao->getLinksToExpandByURL('http://flic.kr/'); if (count($flickrlinkstoexpand > 0)) { $logger->logStatus(count($flickrlinkstoexpand) . " Flickr links to expand", "Flickr Plugin"); } else { $logger->logStatus("No Flickr links to expand", "Flickr Plugin"); } foreach ($flickrlinkstoexpand as $fl) { $eurl = $fa->getFlickrPhotoSource($fl); if ($eurl["expanded_url"] != '') { $ldao->saveExpandedUrl($fl, $eurl["expanded_url"], '', 1); } elseif ($eurl["error"] != '') { $ldao->saveExpansionError($fl, $eurl["error"]); } } $logger->close(); # Close logging } }
public function crawl() { $config = Config::getInstance(); $logger = Logger::getInstance(); $instance_dao = DAOFactory::getDAO('TwitterInstanceDAO'); $owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO'); $owner_dao = DAOFactory::getDAO('OwnerDAO'); $instance_hashtag_dao = DAOFactory::getDAO('InstanceHashtagDAO'); // get oauth values $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('twitter', true); $current_owner = $owner_dao->getByEmail(Session::getLoggedInUser()); $instances = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'twitter'); foreach ($instances as $instance) { $logger->setUsername($instance->network_username); $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . " on Twitter.", __METHOD__ . ',' . __LINE__); $tokens = $owner_instance_dao->getOAuthTokens($instance->id); $num_twitter_errors = isset($options['num_twitter_errors']) ? $options['num_twitter_errors']->option_value : null; $dashboard_module_cacher = new DashboardModuleCacher($instance); try { if (isset($tokens['oauth_access_token']) && $tokens['oauth_access_token'] != '' && isset($tokens['oauth_access_token_secret']) && $tokens['oauth_access_token_secret'] != '') { $archive_limit = isset($options['archive_limit']->option_value) ? $options['archive_limit']->option_value : 3200; $api = new CrawlerTwitterAPIAccessorOAuth($tokens['oauth_access_token'], $tokens['oauth_access_token_secret'], $options['oauth_consumer_key']->option_value, $options['oauth_consumer_secret']->option_value, $archive_limit, $num_twitter_errors); $twitter_crawler = new TwitterCrawler($instance, $api); $instance_dao->updateLastRun($instance->id); $twitter_crawler->fetchInstanceUserTweets(); $twitter_crawler->fetchInstanceUserMentions(); $twitter_crawler->fetchInstanceUserFriends(); $twitter_crawler->fetchInstanceUserFollowers(); $twitter_crawler->fetchInstanceUserGroups(); $twitter_crawler->fetchRetweetsOfInstanceUser(); $twitter_crawler->fetchInstanceUserFavorites(); $twitter_crawler->updateStaleGroupMemberships(); $twitter_crawler->fetchStrayRepliedToTweets(); $twitter_crawler->fetchUserFriendsByIDs(); $twitter_crawler->fetchUnloadedFriendDetails(); $twitter_crawler->fetchUnloadedFollowerDetails(); $twitter_crawler->cleanUpFollows(); $twitter_crawler->updateFriendsProfiles(); //Retrieve search results for saved keyword/hashtags $instances_hashtags = $instance_hashtag_dao->getByInstance($instance->id); foreach ($instances_hashtags as $instance_hashtag) { $twitter_crawler->fetchInstanceHashtagTweets($instance_hashtag); } } else { throw new Exception('Missing Twitter OAuth tokens.'); } } catch (Exception $e) { $logger->logUserError(get_class($e) . " while crawling " . $instance->network_username . " on Twitter: " . $e->getMessage(), __METHOD__ . ',' . __LINE__); } $dashboard_module_cacher->cacheDashboardModules(); // Save instance if (isset($twitter_crawler->user)) { $instance_dao->save($instance, $twitter_crawler->user->post_count, $logger); } Reporter::reportVersion($instance); $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . " on Twitter.", __METHOD__ . ',' . __LINE__); } }
/** * Constructor * @param Instance $instance * @param CrawlerTwitterAPIAccessorOAuth $api * @return TwitterCrawler */ public function __construct($instance, $api) { $this->instance = $instance; $this->api = $api; $this->logger = Logger::getInstance(); $this->logger->setUsername($instance->network_username); $this->user_dao = DAOFactory::getDAO('UserDAO'); $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $this->twitter_options = $plugin_option_dao->getOptionsHash('twitter'); }
public function crawl() { $logger = Logger::getInstance(); $config = Config::getInstance(); $instance_dao = DAOFactory::getDAO('InstagramInstanceDAO'); $owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO'); $owner_dao = DAOFactory::getDAO('OwnerDAO'); $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('instagram', true); //get cached $max_api_calls = isset($options['max_api_calls']) ? $options['max_api_calls']->option_value : 2500; $current_owner = $owner_dao->getByEmail(Session::getLoggedInUser()); //crawl instagram user profiles and pages $instances = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'instagram'); foreach ($instances as $instance) { $logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username); $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__); $tokens = $owner_instance_dao->getOAuthTokens($instance->id); $access_token = $tokens['oauth_access_token']; $instance_dao->updateLastRun($instance->id); $dashboard_module_cacher = new DashboardModuleCacher($instance); try { /** * 1. Fetch user info, media + its likes and comments. * 2. Fetch user's likes. * 3. Fetch user's friends, and update stale relationships. * 4. Fetch user's followers, and update stale relationships. * 5. Update stale friends' profiles. */ $instagram_crawler = new InstagramCrawler($instance, $access_token, $max_api_calls); $instagram_crawler->fetchPostsAndReplies(); $instagram_crawler->fetchLikes(); $instagram_crawler->fetchFriends(); $instagram_crawler->fetchFollowers(); $instagram_crawler->updateStaleFriendsProfiles(); } catch (Instagram\Core\ApiAuthException $e) { //The access token is invalid, save in owner_instances table $owner_instance_dao->setAuthErrorByTokens($instance->id, $access_token, '', $e->getMessage()); //Send email alert //Get owner by auth tokens first, then send to that person $owner_email_to_notify = $owner_instance_dao->getOwnerEmailByInstanceTokens($instance->id, $access_token, ''); $email_attempt = $this->sendInvalidOAuthEmailAlert($owner_email_to_notify, $instance->network_username); if ($email_attempt) { $logger->logUserInfo('Sent reauth email to ' . $owner_email_to_notify, __METHOD__ . ',' . __LINE__); } else { $logger->logInfo('Didn\'t send reauth email to ' . $owner_email_to_notify, __METHOD__ . ',' . __LINE__); } $logger->logUserError(get_class($e) . ' ' . $e->getMessage(), __METHOD__ . ',' . __LINE__); } catch (Exception $e) { $logger->logUserError(get_class($e) . ' ' . $e->getMessage(), __METHOD__ . ',' . __LINE__); } $dashboard_module_cacher->cacheDashboardModules(); $instance_dao->save($instagram_crawler->instance, 0, $logger); Reporter::reportVersion($instance); $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__); } }
public function crawl() { $logger = Logger::getInstance(); $config = Config::getInstance(); $instance_dao = DAOFactory::getDAO('InstanceDAO'); $owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO'); $owner_dao = DAOFactory::getDAO('OwnerDAO'); $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('foursquare', true); $current_owner = $owner_dao->getByEmail(Session::getLoggedInUser()); $instances = $instance_dao->getAllActiveInstancesStalestFirstByNetwork('foursquare'); // Check the client id and secret are set or we can't crawl if (isset($options['foursquare_client_id']->option_value) && isset($options['foursquare_client_secret']->option_value)) { // For each instance of foursquare on this install foreach ($instances as $instance) { if (!$owner_instance_dao->doesOwnerHaveAccessToInstance($current_owner, $instance)) { // Owner doesn't have access to this instance; let's not crawl it. continue; } // Set the user name in the log $logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username); // Write to the log that we have started to collect data $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__); // Get the OAuth tokens for this user $tokens = $owner_instance_dao->getOAuthTokens($instance->id); // Set the access token $access_token = $tokens['oauth_access_token']; // Update the last time we crawled $instance_dao->updateLastRun($instance->id); // Create a new crawler $crawler = new FoursquareCrawler($instance, $access_token); // Check the OAuth tokens we have are valid try { $logger->logInfo("About to initializeInstanceUser", __METHOD__ . ',' . __LINE__); $user = $crawler->initializeInstanceUser($access_token, $current_owner->id); if (isset($user) && $user instanceof User) { $logger->logInfo("User initialized", __METHOD__ . ',' . __LINE__); } // Get the data we want and store it in the database $logger->logInfo("About to fetchInstanceUserCheckins", __METHOD__ . ',' . __LINE__); $crawler->fetchInstanceUserCheckins(); } catch (Exception $e) { // Catch any errors that happen when we check the validity of the OAuth tokens $logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__); } $logger->logInfo("About to cache dashboard modules", __METHOD__ . ',' . __LINE__); // Cache the insights to improve the page load speed of the dashboard $dashboard_module_cacher = new DashboardModuleCacher($instance); $dashboard_module_cacher->cacheDashboardModules(); $instance_dao->save($crawler->instance, 0, $logger); Reporter::reportVersion($instance); // Tell the user crawling was sucessful $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__); } } }
public function crawl() { $logger = Logger::getInstance(); $config = Config::getInstance(); $instance_dao = DAOFactory::getDAO('FacebookInstanceDAO'); $owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO'); $owner_dao = DAOFactory::getDAO('OwnerDAO'); $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('facebook', true); //get cached $max_crawl_time = isset($options['max_crawl_time']) ? $options['max_crawl_time']->option_value : 20; //convert to seconds $max_crawl_time = $max_crawl_time * 60; $current_owner = $owner_dao->getByEmail(Session::getLoggedInUser()); //crawl Facebook user profiles and pages $profiles = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'facebook'); $pages = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'facebook page'); $instances = array_merge($profiles, $pages); foreach ($instances as $instance) { $logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username); $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__); $tokens = $owner_instance_dao->getOAuthTokens($instance->id); $access_token = $tokens['oauth_access_token']; $instance_dao->updateLastRun($instance->id); $facebook_crawler = new FacebookCrawler($instance, $access_token, $max_crawl_time); $dashboard_module_cacher = new DashboardModuleCacher($instance); try { $facebook_crawler->fetchPostsAndReplies(); } catch (APIOAuthException $e) { $logger->logUserError(get_class($e) . ": " . $e->getMessage(), __METHOD__ . ',' . __LINE__); //Don't send reauth email if it's app-level API rate limting //https://developers.facebook.com/docs/reference/ads-api/api-rate-limiting/#applimit if (strpos($e->getMessage(), 'Application request limit reached') === false && strpos($e->getMessage(), 'Please retry your request later.') === false) { //The access token is invalid, save in owner_instances table $owner_instance_dao->setAuthErrorByTokens($instance->id, $access_token, '', $e->getMessage()); //Send email alert //Get owner by auth tokens first, then send to that person $owner_email_to_notify = $owner_instance_dao->getOwnerEmailByInstanceTokens($instance->id, $access_token, ''); $email_attempt = $this->sendInvalidOAuthEmailAlert($owner_email_to_notify, $instance->network_username); if ($email_attempt) { $logger->logUserInfo('Sent reauth email to ' . $owner_email_to_notify, __METHOD__ . ',' . __LINE__); } else { $logger->logInfo('Didn\'t send reauth email to ' . $owner_email_to_notify, __METHOD__ . ',' . __LINE__); } } else { $logger->logInfo('Facebook API returned an error: ' . $e->getMessage() . ' Do nothing now and try again later', __METHOD__ . ',' . __LINE__); } } catch (Exception $e) { $logger->logUserError(get_class($e) . ": " . $e->getMessage(), __METHOD__ . ',' . __LINE__); } $dashboard_module_cacher->cacheDashboardModules(); $instance_dao->save($facebook_crawler->instance, 0, $logger); Reporter::reportVersion($instance); $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__); } }
/** * @return str */ public function authControl() { $config = Config::getInstance(); Loader::definePathConstants(); $this->setViewTemplate(THINKUP_WEBAPP_PATH . 'plugins/twitterrealtime/view/twitterrealtime.account.index.tpl'); $this->view_mgr->addHelp('twitterrealtime', 'userguide/settings/plugins/twitterrealtime'); $id = DAOFactory::getDAO('InstanceDAO'); $od = DAOFactory::getDAO('OwnerDAO'); // get plugin option values if defined... $plugin_options = $this->getPluginOptions(); // get oauth option values from twitter plugin. // @TODO -- what is the right way to do this? $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $twitter_options = $plugin_option_dao->getOptionsHash('twitter', true); $oauth_consumer_key = null; if (isset($twitter_options['oauth_consumer_key'])) { $oauth_consumer_key = $twitter_options['oauth_consumer_key']->option_value; } $oauth_consumer_secret = null; if (isset($twitter_options['oauth_consumer_secret'])) { $oauth_consumer_secret = $twitter_options['oauth_consumer_secret']->option_value; } // @TODO - get any other option values as necessary // $archive_limit = $this->getPluginOption('archive_limit'); $auth_from_twitter = ''; if (isset($oauth_consumer_key) && isset($oauth_consumer_secret)) { $to = new TwitterOAuth($oauth_consumer_key, $oauth_consumer_secret); /* Request tokens from twitter */ $tok = $to->getRequestToken(); if (isset($tok['oauth_token'])) { $token = $tok['oauth_token']; $_SESSION['oauth_request_token_secret'] = $tok['oauth_token_secret']; /* Build the authorization URL */ $oauthorize_link = $to->getAuthorizeURL($token); // create indication that auth from twitter plugin is okay $auth_from_twitter = "Using the Twitter Consumer key and secret as set in " . "the <a href=\"./?p=twitter\">Twitter plugin</a>."; } else { //set error message here $this->addErrorMessage("Unable to obtain OAuth token. Check your Twitter plugin consumer key and secret configuration."); $oauthorize_link = ''; } } else { $this->addErrorMessage("Missing required settings! Please configure the Twitter plugin."); $oauthorize_link = ''; } $owner_instances = $id->getByOwnerAndNetwork($this->owner, 'twitter'); $this->addToView('owner_instances', $owner_instances); $this->addToView('oauthorize_link', $oauthorize_link); $this->addToView('auth_from_twitter', $auth_from_twitter); // add plugin options from $this->addOptionForm(); $plugin = new TwitterRealtimePlugin(); $this->addToView('is_configured', $plugin->isConfigured()); return $this->generateView(); }
public function authControl() { if (!$this->is_missing_param) { $fb_user = null; $msg = ''; $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('facebook', true); //get cached $facebook = new Facebook($options['facebook_api_key']->option_value, $options['facebook_api_secret']->option_value); $fb_user = $facebook->api_client->users_getLoggedInUser(); $msg .= "Facebook user is logged in and user ID set<br />"; $fb_username = $facebook->api_client->users_getInfo($fb_user, 'name'); if (isset($_GET['sessionKey']) && isset($fb_user) && $fb_user > 0) { $fb_username = $fb_username[0]['name']; $session_key = $_GET['sessionKey']; $owner_dao = DAOFactory::getDAO('OwnerDAO'); $instance_dao = DAOFactory::getDAO('InstanceDAO'); $oid = DAOFactory::getDAO('OwnerInstanceDAO'); $user_dao = DAOFactory::getDAO('UserDAO'); $owner = $owner_dao->getByEmail($this->getLoggedInUser()); $i = $instance_dao->getByUserIdOnNetwork($fb_user, 'facebook'); if (isset($i)) { $msg .= "Instance exists<br />"; $oi = $oid->get($owner->id, $i->id); if ($oi == null) { //Instance already exists, owner instance doesn't $oid->insert($owner->id, $i->id, $session_key); //Add owner instance with session key $msg .= "Created owner instance.<br />"; } } else { //Instance does not exist $msg .= "Instance does not exist<br />"; $instance_dao->insert($fb_user, $fb_username, 'facebook'); $msg .= "Created instance"; $i = $instance_dao->getByUserIdOnNetwork($fb_user, 'facebook'); $oid->insert($owner->id, $i->id, $session_key); $msg .= "Created owner instance.<br />"; } if (!$user_dao->isUserInDB($fb_user, 'facebook')) { $r = array('user_id' => $fb_user, 'user_name' => $fb_username, 'full_name' => $fb_username, 'avatar' => '', 'location' => '', 'description' => '', 'url' => '', 'is_protected' => '', 'follower_count' => 0, 'friend_count' => 0, 'post_count' => 0, 'last_updated' => '', 'last_post' => '', 'joined' => '', 'last_post_id' => '', 'network' => 'facebook'); $u = new User($r, 'Owner info'); $user_dao->updateUser($u); } } else { $msg .= "No session key or logged in Facebook user."; } $config = Config::getInstance(); $msg .= '<br /> <a href="' . $config->getValue('site_root_path') . 'account/">Back to your account</a>.'; $this->addInfoMessage($msg); } return $this->generateView(); }
public function authControl() { if (!$this->is_missing_param) { $request_token = $_GET['oauth_token']; $request_token_secret = SessionCache::get('oauth_request_token_secret'); // get oauth values $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('twitter', true); //get cached $to = new TwitterOAuth($options['oauth_consumer_key']->option_value, $options['oauth_consumer_secret']->option_value, $request_token, $request_token_secret); $tok = $to->getAccessToken(); if (isset($tok['oauth_token']) && isset($tok['oauth_token_secret'])) { $api = new TwitterAPIAccessorOAuth($tok['oauth_token'], $tok['oauth_token_secret'], $options['oauth_consumer_key']->option_value, $options['oauth_consumer_secret']->option_value, $options['num_twitter_errors']->option_value, $options['max_api_calls_per_crawl']->option_value, false); $authed_twitter_user = $api->verifyCredentials(); // echo "User ID: ". $authed_twitter_user['user_id']; // echo "User name: ". $authed_twitter_user['user_name']; $owner_dao = DAOFactory::getDAO('OwnerDAO'); $owner = $owner_dao->getByEmail($this->getLoggedInUser()); if ((int) $authed_twitter_user['user_id'] > 0) { $instance_dao = DAOFactory::getDAO('TwitterInstanceDAO'); $instance = $instance_dao->getByUsername($authed_twitter_user['user_name'], 'twitter'); $owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO'); if (isset($instance)) { $owner_instance = $owner_instance_dao->get($owner->id, $instance->id); if ($owner_instance != null) { $owner_instance_dao->updateTokens($owner->id, $instance->id, $tok['oauth_token'], $tok['oauth_token_secret']); $this->addSuccessMessage($authed_twitter_user['user_name'] . " on Twitter is already set up in ThinkUp! To add a different Twitter account, " . "log out of Twitter.com in your browser and authorize ThinkUp again."); } else { if ($owner_instance_dao->insert($owner->id, $instance->id, $tok['oauth_token'], $tok['oauth_token_secret'])) { $this->addSuccessMessage("Success! " . $authed_twitter_user['user_name'] . " on Twitter has been added to ThinkUp!"); } else { $this->addErrorMessage("Error: Could not create an owner instance."); } } } else { $instance_dao->insert($authed_twitter_user['user_id'], $authed_twitter_user['user_name']); $instance = $instance_dao->getByUsername($authed_twitter_user['user_name']); if ($owner_instance_dao->insert($owner->id, $instance->id, $tok['oauth_token'], $tok['oauth_token_secret'])) { $this->addSuccessMessage("Success! " . $authed_twitter_user['user_name'] . " on Twitter has been added to ThinkUp!"); } else { $this->addErrorMessage("Error: Could not create an owner instance."); } } } } else { $msg = "Error: Twitter authorization did not complete successfully. Check if your account already " . " exists. If not, please try again."; $this->addErrorMessage($msg); } $this->view_mgr->clear_all_cache(); } return $this->generateView(); }
public function crawl() { $logger = Logger::getInstance(); $config = Config::getInstance(); $id = DAOFactory::getDAO('InstanceDAO'); $oid = DAOFactory::getDAO('OwnerInstanceDAO'); $od = DAOFactory::getDAO('OwnerDAO'); $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('facebook', true); //get cached $current_owner = $od->getByEmail(Session::getLoggedInUser()); //crawl Facebook user profiles $instances = $id->getAllActiveInstancesStalestFirstByNetwork('facebook'); foreach ($instances as $instance) { if (!$oid->doesOwnerHaveAccess($current_owner, $instance)) { // Owner doesn't have access to this instance; let's not crawl it. continue; } $logger->setUsername($instance->network_username); $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . " on Facebook.", __METHOD__ . ',' . __LINE__); $tokens = $oid->getOAuthTokens($instance->id); $access_token = $tokens['oauth_access_token']; $id->updateLastRun($instance->id); $crawler = new FacebookCrawler($instance, $access_token); try { $crawler->fetchInstanceUserInfo(); $crawler->fetchUserPostsAndReplies($instance->network_user_id); } catch (Exception $e) { $logger->logUserError('PROFILE EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__); } $id->save($crawler->instance, 0, $logger); $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . " on Facebook.", __METHOD__ . ',' . __LINE__); } //crawl Facebook pages $instances = $id->getAllActiveInstancesStalestFirstByNetwork('facebook page'); foreach ($instances as $instance) { $logger->setUsername($instance->network_username); $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s Facebook Page.", __METHOD__ . ',' . __LINE__); $tokens = $oid->getOAuthTokens($instance->id); $access_token = $tokens['oauth_access_token']; $id->updateLastRun($instance->id); $crawler = new FacebookCrawler($instance, $access_token); try { $crawler->fetchPagePostsAndReplies($instance->network_user_id); } catch (Exception $e) { $logger->logUserError('PAGE EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__); } $id->save($crawler->instance, 0, $logger); $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s Facebook Page.", __METHOD__ . ',' . __LINE__); } }
public function crawl() { $logger = Logger::getInstance(); $config = Config::getInstance(); $id = DAOFactory::getDAO('InstanceDAO'); $oid = DAOFactory::getDAO('OwnerInstanceDAO'); $od = DAOFactory::getDAO('OwnerDAO'); $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('facebook', true); //get cached $current_owner = $od->getByEmail(Session::getLoggedInUser()); //crawl Facebook user profiles $instances = $id->getAllActiveInstancesStalestFirstByNetwork('facebook'); foreach ($instances as $instance) { if (!$oid->doesOwnerHaveAccess($current_owner, $instance)) { // Owner doesn't have access to this instance; let's not crawl it. continue; } $logger->setUsername($instance->network_username); $tokens = $oid->getOAuthTokens($instance->id); $session_key = $tokens['oauth_access_token']; $fb = new Facebook($options['facebook_api_key']->option_value, $options['facebook_api_secret']->option_value); $id->updateLastRun($instance->id); $crawler = new FacebookCrawler($instance, $fb); try { $crawler->fetchInstanceUserInfo($instance->network_user_id, $session_key); $crawler->fetchUserPostsAndReplies($instance->network_user_id, $session_key); } catch (Exception $e) { $logger->logStatus('PROFILE EXCEPTION: ' . $e->getMessage(), get_class($this)); } $id->save($crawler->instance, $crawler->owner_object->post_count, $logger); } //crawl Facebook pages $instances = $id->getAllActiveInstancesStalestFirstByNetwork('facebook page'); foreach ($instances as $instance) { $logger->setUsername($instance->network_username); $tokens = $oid->getOAuthTokens($instance->id); $session_key = $tokens['oauth_access_token']; $fb = new Facebook($options['facebook_api_key']->option_value, $options['facebook_api_secret']->option_value); $id->updateLastRun($instance->id); $crawler = new FacebookCrawler($instance, $fb); try { $crawler->fetchPagePostsAndReplies($instance->network_user_id, $instance->network_viewer_id, $session_key); } catch (Exception $e) { $logger->logStatus('PAGE EXCEPTION: ' . $e->getMessage(), get_class($this)); } $id->save($crawler->instance, 0, $logger); } $logger->close(); # Close logging }
/** * Run when the crawler does */ public function crawl() { $logger = Logger::getInstance(); $logger->setUsername(null); $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('expandurls', true); //Flickr image thumbnails if (isset($options['flickr_api_key']->option_value)) { self::expandFlickrThumbnails($options['flickr_api_key']->option_value); } //@TODO: Bit.ly URLs //Remaining URLs $link_limit = isset($options['links_to_expand']->option_value) ? (int) $options['links_to_expand']->option_value : 1500; self::expandRemainingURLs($link_limit); }
public static function getQueue() { // get stream plugin settings $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('twitterrealtime', true); if (is_null(self::$queue)) { if (isset($options['use_redis']) && $options['use_redis']->option_value == "true") { // we need a redis queue self::$queue = new StreamMessageQueueRedis(); } else { // we need a mysql queue self::$queue = new StreamMessageQueueMySQL(); } } return self::$queue; }
/** * @static * @param $oauth_access_token * @param $oauth_access_token_secret * @return ConsumerUserStream|null */ public static function getInstance($oauth_access_token, $oauth_access_token_secret) { $sc = null; $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('twitter', true); //get cached if (isset($options['oauth_consumer_key']) && isset($options['oauth_consumer_secret'])) { define('TWITTER_CONSUMER_KEY', $options['oauth_consumer_key']->option_value); define('TWITTER_CONSUMER_SECRET', $options['oauth_consumer_secret']->option_value); $sc = new ConsumerUserStream($oauth_access_token, $oauth_access_token_secret); } else { $logger = Logger::getInstance('stream_log_location'); $logger->logError("Error: could not obtain Twitter app consumer key and secret from Twitter plugin " . "settings.", __METHOD__ . ',' . __LINE__); } return $sc; }
/** * Outputs javascript callback string with json array/list of post as an argument */ public function authControl() { if (!$this->is_missing_param) { $instance_dao = DAOFactory::getDAO('InstanceDAO'); if ($instance_dao->isUserConfigured($_GET['u'], $_GET['n'])) { $username = $_GET['u']; $ownerinstance_dao = DAOFactory::getDAO('OwnerInstanceDAO'); $owner_dao = DAOFactory::getDAO('OwnerDAO'); $owner = $owner_dao->getByEmail($this->getLoggedInUser()); $instance = $instance_dao->getByUsername($username, $_GET['n']); if (!$ownerinstance_dao->doesOwnerHaveAccess($owner, $instance)) { echo '{"status":"failed","message":"Insufficient privileges."}'; } else { echo "tu_grid_search.populate_grid("; $post_dao = DAOFactory::GetDAO('PostDAO'); $posts_it = null; if ($_GET['d'] == 'tweets-all') { $posts_it = $post_dao->getAllPostsByUsernameIterator($_GET['u'], $_GET['n'], self::MAX_ROWS); } else { if ($_GET['d'] == 'tweets-mostreplies') { $posts_it = $post_dao->getAllMentionsIterator($_GET['u'], self::MAX_ROWS, $_GET['n']); } else { if ($_GET['d'] == 'tweets-mostretweeted') { $posts_it = $post_dao->getMostRetweetedPostsIterator($_GET['u'], $_GET['n'], self::MAX_ROWS, self::MAX_RT_DAYS); } } } echo '{"status":"success","posts": [' . "\n"; $cnt = 0; foreach ($posts_it as $key => $value) { $cnt++; $data = array('id' => $cnt, 'text' => $value->post_text, 'post_id' => $value->post_id, 'author' => $value->author_username, 'date' => $value->adj_pub_date); echo json_encode($data) . ",\n"; flush(); } $data = array('id' => -1, 'text' => 'Last Post', 'author' => 'nobody'); echo json_encode($data); echo ']});'; } } else { echo '{"status":"failed","message":"' . $_GET['u'] . 'is not configured."}'; } } else { echo '{"status":"failed","message":"Missing Parameters"}'; } }
/** * Main control method */ public function authControl() { $this->setViewTemplate('post.index.tpl'); $network = isset($_GET['n']) ? $_GET['n'] : 'twitter'; $_GET['n'] = $network; if ($this->shouldRefreshCache()) { if (isset($_GET['t']) && is_numeric($_GET['t']) && $this->post_dao->isPostInDB($_GET['t'], $network)) { $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('geoencoder', true); if (isset($options['distance_unit']->option_value)) { $distance_unit = $options['distance_unit']->option_value; } else { $distance_unit = 'km'; } $post_id = $_GET['t']; $post = $this->post_dao->getPost($post_id, $network); $this->addToView('post', $post); $this->addToView('unit', $distance_unit); // costly query //$this->addToView('likely_orphans', $this->post_dao->getLikelyOrphansForParent($post->pub_date, //$post->author_user_id,$post->author_username, 15) ); //$this->addToView('all_tweets', $this->post_dao->getAllPosts($post->author_user_id, 15) ); $all_replies = $this->post_dao->getRepliesToPost($post_id, $network, 'default', $distance_unit); $this->addToView('replies', $all_replies); $all_replies_by_location = $this->post_dao->getRepliesToPost($post_id, $network, 'location', $distance_unit); $this->addToView('replies_by_location', $all_replies_by_location); $all_replies_count = count($all_replies); $this->addToView('reply_count', $all_replies_count); $all_retweets = $this->post_dao->getRetweetsOfPost($post_id, $network, 'default', $distance_unit); $this->addToView('retweets', $all_retweets); $all_retweets_by_location = $this->post_dao->getRetweetsOfPost($post_id, $network, 'location', $distance_unit); $this->addToView('retweets_by_location', $all_retweets_by_location); $retweet_reach = $this->post_dao->getPostReachViaRetweets($post_id, $network); $this->addToView('retweet_reach', $retweet_reach); $public_replies = $this->post_dao->getPublicRepliesToPost($post_id, $network); $public_replies_count = count($public_replies); $this->addToView('public_reply_count', $public_replies_count); $private_replies_count = $all_replies_count - $public_replies_count; $this->addToView('private_reply_count', $private_replies_count); } else { $this->addErrorMessage('Post not found'); } } return $this->generateView(); }
/** * Constructor * @return StreamMasterCollect */ public function __construct() { $config = Config::getInstance(); $this->streaming_dir = $config->getValue('source_root_path') . 'webapp/plugins/twitterrealtime/streaming'; // @TODO -- get this from plugin information now $this->log_dir = $config->getValue('source_root_path') . 'logs'; $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('twitterrealtime', true); if (isset($options['php_path'])) { $this->php_path = $options['php_path']->option_value; } else { $this->php_path = null; // path to the php interp on the user's system } $this->stream_proc_dao = DAOFactory::getDAO('StreamProcDAO'); $this->instance_dao = DAOFactory::getDAO('InstanceDAO'); $this->owner_dao = DAOFactory::getDAO('OwnerDAO'); }
public function crawl() { $logger = Logger::getInstance(); $config = Config::getInstance(); $instance_dao = DAOFactory::getDAO('InstanceDAO'); $owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO'); $owner_dao = DAOFactory::getDAO('OwnerDAO'); $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('facebook', true); //get cached $max_crawl_time = isset($options['max_crawl_time']) ? $options['max_crawl_time']->option_value : 20; //convert to seconds $max_crawl_time = $max_crawl_time * 60; $current_owner = $owner_dao->getByEmail(Session::getLoggedInUser()); //crawl Facebook user profiles and pages $profiles = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'facebook'); $pages = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'facebook page'); $instances = array_merge($profiles, $pages); foreach ($instances as $instance) { $logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username); $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__); $tokens = $owner_instance_dao->getOAuthTokens($instance->id); $access_token = $tokens['oauth_access_token']; $instance_dao->updateLastRun($instance->id); $facebook_crawler = new FacebookCrawler($instance, $access_token, $max_crawl_time); $dashboard_module_cacher = new DashboardModuleCacher($instance); try { $facebook_crawler->fetchPostsAndReplies(); } catch (APIOAuthException $e) { //The access token is invalid, save in owner_instances table $owner_instance_dao->setAuthError($current_owner->id, $instance->id, $e->getMessage()); //Send email alert $this->sendInvalidOAuthEmailAlert($current_owner->email, $instance->network_username); $logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__); } catch (Exception $e) { $logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__); } $dashboard_module_cacher->cacheDashboardModules(); $instance_dao->save($facebook_crawler->instance, 0, $logger); Reporter::reportVersion($instance); $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__); } }
public function crawl() { $logger = Logger::getInstance(); $config = Config::getInstance(); $instance_dao = DAOFactory::getDAO('InstanceDAO'); $owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO'); $owner_dao = DAOFactory::getDAO('OwnerDAO'); $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('youtube', true); //get cached $max_crawl_time = isset($options['max_crawl_time']) ? $options['max_crawl_time']->option_value : 20; //convert to seconds $max_crawl_time = $max_crawl_time * 60; $developer_key = isset($options['developer_key']) ? $options['developer_key']->option_value : null; $max_comments = isset($options['max_comments']) ? $options['max_comments']->option_value : null; $current_owner = $owner_dao->getByEmail(Session::getLoggedInUser()); //crawl youtube users $instances = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'youtube'); if (isset($options['youtube_client_id']->option_value) && isset($options['youtube_client_secret']->option_value)) { foreach ($instances as $instance) { $logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username); $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__); $tokens = $owner_instance_dao->getOAuthTokens($instance->id); $access_token = $tokens['oauth_access_token']; $refresh_token = $tokens['oauth_access_token_secret']; $instance_dao->updateLastRun($instance->id); $youtube_crawler = new YouTubeCrawler($instance, $access_token, $max_crawl_time, $developer_key, $max_comments); $dashboard_module_cacher = new DashboardModuleCacher($instance); try { $youtube_crawler->initializeInstanceUser($options['youtube_client_id']->option_value, $options['youtube_client_secret']->option_value, $access_token, $refresh_token, $current_owner->id); $youtube_crawler->fetchInstanceUserVideos(); } catch (Exception $e) { $logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__); } $dashboard_module_cacher->cacheDashboardModules(); $instance_dao->save($youtube_crawler->instance, 0, $logger); Reporter::reportVersion($instance); $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__); } } }
public function crawl() { $config = Config::getInstance(); $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('flickrthumbnails', true); $api_key = ''; if (isset($options['flickr_api_key']->option_value)) { $api_key = $options['flickr_api_key']->option_value; } if (isset($api_key) && $api_key != '') { $logger = Logger::getInstance(); $logger->setUsername(null); $fa = new FlickrAPIAccessor($api_key); $ldao = DAOFactory::getDAO('LinkDAO'); $flickrlinkstoexpand = $ldao->getLinksToExpandByURL('http://flic.kr/'); if (count($flickrlinkstoexpand) > 0) { $logger->logUserInfo(count($flickrlinkstoexpand)." Flickr links to expand.", __METHOD__.','.__LINE__); } else { $logger->logUserInfo("There are no Flickr thumbnails to expand.", __METHOD__.','.__LINE__); } $total_thumbnails = 0; $total_errors = 0; foreach ($flickrlinkstoexpand as $fl) { $eurl = $fa->getFlickrPhotoSource($fl); if ($eurl["expanded_url"] != '') { $ldao->saveExpandedUrl($fl, $eurl["expanded_url"], '', 1); $total_thumbnails = $total_thumbnails + 1; } elseif ($eurl["error"] != '') { $ldao->saveExpansionError($fl, $eurl["error"]); $total_errors = $total_errors + 1; } $logger->logUserSuccess($total_thumbnails." Flickr thumbnails expanded (".$total_errors." errors)", __METHOD__.','.__LINE__); } } }
public function crawl() { $logger = Logger::getInstance(); $config = Config::getInstance(); $instance_dao = DAOFactory::getDAO('InstanceDAO'); $owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO'); $owner_dao = DAOFactory::getDAO('OwnerDAO'); $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('facebook', true); //get cached $max_crawl_time = isset($options['max_crawl_time']) ? $options['max_crawl_time']->option_value : 20; //convert to seconds $max_crawl_time = $max_crawl_time * 60; $current_owner = $owner_dao->getByEmail(Session::getLoggedInUser()); //crawl Facebook user profiles and pages $profiles = $instance_dao->getAllActiveInstancesStalestFirstByNetwork('facebook'); $pages = $instance_dao->getAllActiveInstancesStalestFirstByNetwork('facebook page'); $instances = array_merge($profiles, $pages); foreach ($instances as $instance) { if (!$owner_instance_dao->doesOwnerHaveAccessToInstance($current_owner, $instance)) { // Owner doesn't have access to this instance; let's not crawl it. continue; } $logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username); $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__); $tokens = $owner_instance_dao->getOAuthTokens($instance->id); $access_token = $tokens['oauth_access_token']; $instance_dao->updateLastRun($instance->id); $crawler = new FacebookCrawler($instance, $access_token, $max_crawl_time); try { $crawler->fetchPostsAndReplies(); } catch (Exception $e) { $logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__); } $instance_dao->save($crawler->instance, 0, $logger); $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__); } }
/** * Run when the crawler does */ public function crawl() { $logger = Logger::getInstance(); $logger->setUsername(null); $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('expandurls', true); //Limit the number of links expanded each crawl $this->link_limit = isset($options['links_to_expand']->option_value) ? (int) $options['links_to_expand']->option_value : 1500; if ($this->link_limit != 0) { //Flickr image thumbnails if (isset($options['flickr_api_key']->option_value)) { self::expandFlickrThumbnails($options['flickr_api_key']->option_value); } //Bit.ly URLs if (isset($options['bitly_api_key']->option_value, $options['bitly_login']->option_value)) { self::expandBitlyLinks($options['bitly_api_key']->option_value, $options['bitly_login']->option_value); } //Remaining links self::expandRemainingURLs(); } else { $logger->logUserInfo("Limit of links to expand reached.", __METHOD__ . ',' . __LINE__); } }
/** * Run when the crawler does * @TODO Set limit on total number of links to expand per crawler run in the plugin settings, for now 1500 */ public function crawl() { $logger = Logger::getInstance(); $logger->setUsername(null); $ldao = DAOFactory::getDAO('LinkDAO'); $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('expandurls', true); $total_links_to_expand = isset($options['links_to_expand']->option_value) ? (int)$options['links_to_expand']->option_value : 1500; $linkstoexpand = $ldao->getLinksToExpand($total_links_to_expand); $logger->logUserInfo(count($linkstoexpand)." links to expand. Please wait. Working...", __METHOD__.','.__LINE__); $total_expanded = 0; $total_errors = 0; foreach ($linkstoexpand as $l) { if (Utils::validateURL($l)) { $logger->logInfo("Expanding ".($total_expanded+1). " of ".count($linkstoexpand)." (".$l.")", __METHOD__.','.__LINE__); $eurl = self::untinyurl($l, $ldao); if ($eurl != '') { $ldao->saveExpandedUrl($l, $eurl); $total_expanded = $total_expanded + 1; } else { $total_errors = $total_errors + 1; } } else { $total_errors = $total_errors + 1; $logger->logError($l." is not a valid URL; skipping expansion", __METHOD__.','.__LINE__); } } $logger->logUserSuccess($total_expanded." URLs successfully expanded (".$total_errors." errors).", __METHOD__.','.__LINE__); }