public function control()
{
$this->view_name = isset($_GET['v']) ? $_GET['v'] : 'default';
$post_dao = DAOFactory::getDAO('PostDAO');
$this->setPageTitle('Post Details');
$this->setViewTemplate('post.index.tpl');
$network = isset($_GET['n']) ? $_GET['n'] : 'twitter';
if ($this->shouldRefreshCache()) {
if (isset($_GET['t']) && is_numeric($_GET['t'])) {
$post_id = $_GET['t'];
$post = $post_dao->getPost($post_id, $network);
if (isset($post)) {
$config = Config::getInstance();
$this->addToView('disable_embed_code', $config->getValue('is_embed_disabled') || $post->is_protected);
if (isset($_GET['search'])) {
$this->addToView('search_on', true);
}
if (!$post->is_protected || $this->isLoggedIn()) {
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('geoencoder', true);
if (isset($options['distance_unit']->option_value)) {
$distance_unit = $options['distance_unit']->option_value;
} else {
$distance_unit = 'km';
}
$this->addToView('post', $post);
$this->addToView('unit', $distance_unit);
$replies = $post_dao->getRepliesToPost($post_id, $network, 'default', $distance_unit);
$public_replies = array();
foreach ($replies as $reply) {
if (!$reply->author->is_protected) {
$public_replies[] = $reply;
}
}
$public_replies_count = count($public_replies);
$this->addToView('public_reply_count', $public_replies_count);
if ($this->isLoggedIn()) {
$this->addToView('replies', $replies);
} else {
$this->addToView('replies', $public_replies);
}
$all_replies_count = count($replies);
$private_reply_count = $all_replies_count - $public_replies_count;
$this->addToView('private_reply_count', $private_reply_count);
$webapp = Webapp::getInstance();
$sidebar_menu = $webapp->getPostDetailMenu($post);
$this->addToView('sidebar_menu', $sidebar_menu);
$this->loadView($post);
} else {
$this->addErrorMessage('Insufficient privileges');
}
} else {
$this->addErrorMessage('Post not found');
}
} else {
$this->addErrorMessage('Post not specified');
}
}
return $this->generateView();
}
public function generateInsight(Instance $instance, $last_week_of_posts, $number_days)
{
parent::generateInsight($instance, $last_week_of_posts, $number_days);
$this->logger->logInfo("Begin generating insight", __METHOD__ . ',' . __LINE__);
$filename = basename(__FILE__, ".php");
foreach ($last_week_of_posts as $post) {
$simplified_post_date = date('Y-m-d', strtotime($post->pub_date));
// Map insight: If not a reply or retweet and geoencoded, show the map in the stream
if (!isset($post->in_reply_to_user_id) && !isset($post->in_reply_to_post_id) && !isset($post->in_retweet_of_post_id) && $post->reply_count_cache > 5) {
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('geoencoder', true);
if (isset($options['gmaps_api_key']->option_value) && $post->is_geo_encoded == 1) {
//Get post's replies and loop through them to make sure at least 5 are indeed geoencoded
$post_dao = DAOFactory::getDAO('PostDAO');
$post_replies = $post_dao->getRepliesToPost($post->post_id, $post->network);
$total_geoencoded_replies = 0;
foreach ($post_replies as $reply) {
if ($reply->is_geo_encoded == 1) {
$total_geoencoded_replies++;
}
}
if ($total_geoencoded_replies > 4) {
$this->insight_dao->insertInsight('geoencoded_replies', $instance->id, $simplified_post_date, "Going global!", "{$this->username}'s post got responses from locations all over the map.", $filename, Insight::EMPHASIS_LOW, serialize($post));
}
}
}
}
$this->logger->logInfo("Done generating insight", __METHOD__ . ',' . __LINE__);
}
/**
* Main control method
* @return Page markup
*/
public function control() {
if ($this->shouldRefreshCache()) {
$this->setViewTemplate(Utils::getPluginViewDirectory('geoencoder').'geoencoder.map.iframe.tpl');
$this->setPageTitle('Locate Post on Map');
$post_dao = DAOFactory::getDAO('PostDAO');
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('geoencoder', true);
if (isset($options['gmaps_api_key']->option_value)) {
$api_key = $options['gmaps_api_key']->option_value;
} else {
$api_key = NULL;
}
$network = (isset($_GET['n']))?$_GET['n']:'twitter';
$type = (isset($_GET['t']))?$_GET['t']:'post';
$post_id = (isset($_GET['pid']))?$_GET['pid']:'post_id';
$post = $post_dao->getPost($post_id, $network);
if ($type == 'post' && isset($post) && $post->is_geo_encoded == 1) {
$this->addToView('post', $post);
$this->addHeaderJavaScript('plugins/geoencoder/assets/js/generatemap.js');
$this->addToView('gmaps_api', $api_key);
$post_rows = $post_dao->getRelatedPostsArray($post_id, $network, !$this->isLoggedIn());
$posts_json = $this->processLocations($post_rows, $post_id);
$this->addToView('posts_data', $posts_json);
} else {
$this->addErrorMessage('This post has not been geoencoded yet; cannot display map.');
}
}
return $this->generateView();
}
/**
* Load view with individual post and replies and retweets
* @param int $post_id
* @param str $network
*/
private function loadSinglePostThread($post_id, $network)
{
$this->setPageTitle('Public Post Replies');
$this->post_dao->isPostByPublicInstance($_GET['t'], $network);
$post = $this->post_dao->getPost($post_id, $network);
if (!isset($post)) {
$this->addErrorMessage("Post " . $post_id . " on " . ucwords($network) . " is not in ThinkUp.");
} else {
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('geoencoder', true);
if (isset($options['distance_unit']->option_value)) {
$distance_unit = $options['distance_unit']->option_value;
} else {
$distance_unit = 'km';
}
$public_tweet_replies = $this->post_dao->getPublicRepliesToPost($post->post_id, $network, 'default', $distance_unit);
$public_retweets = $this->post_dao->getRetweetsOfPost($post->post_id, $network, 'default', $distance_unit, true);
$this->addToView('post', $post);
$this->addToView('replies', $public_tweet_replies);
$this->addToView('retweets', $public_retweets);
$this->addToView('unit', $distance_unit);
$rtreach = 0;
foreach ($public_retweets as $t) {
$rtreach += $t->author->follower_count;
}
$this->addToView('rtreach', $rtreach);
}
}
public function crawl()
{
$logger = Logger::getInstance();
$config = Config::getInstance();
$instance_dao = DAOFactory::getDAO('InstanceDAO');
$owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
$owner_dao = DAOFactory::getDAO('OwnerDAO');
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('googleplus', true);
//get cached
$current_owner = $owner_dao->getByEmail(Session::getLoggedInUser());
//crawl Google+ users
$instances = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'google+');
if (isset($options['google_plus_client_id']->option_value) && isset($options['google_plus_client_secret']->option_value)) {
foreach ($instances as $instance) {
$logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username);
$logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
$tokens = $owner_instance_dao->getOAuthTokens($instance->id);
$access_token = $tokens['oauth_access_token'];
$refresh_token = $tokens['oauth_access_token_secret'];
$instance_dao->updateLastRun($instance->id);
$google_plus_crawler = new GooglePlusCrawler($instance, $access_token);
$dashboard_module_cacher = new DashboardModuleCacher($instance);
try {
$google_plus_crawler->initializeInstanceUser($options['google_plus_client_id']->option_value, $options['google_plus_client_secret']->option_value, $access_token, $refresh_token, $current_owner->id);
$google_plus_crawler->fetchInstanceUserPosts();
} catch (Exception $e) {
$logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__);
}
$dashboard_module_cacher->cacheDashboardModules();
$instance_dao->save($google_plus_crawler->instance, 0, $logger);
$logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
}
}
}
/**
* Run when the crawler does
*/
public function crawl()
{
$this->logger->setUsername(null);
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('expandurls', true);
//Limit the number of links expanded each crawl
$this->link_limit = isset($options['links_to_expand']->option_value) ? (int) $options['links_to_expand']->option_value : 1500;
if ($this->link_limit != 0) {
//set Flickr API key
if (isset($options['flickr_api_key']->option_value)) {
$flickr_api_key = $options['flickr_api_key']->option_value;
} else {
$flickr_api_key = null;
}
//Get short links into table via initial expansion
self::expandOriginalURLs($flickr_api_key);
//Capture click counts for bit.ly URLs
if (isset($options['bitly_api_key']->option_value, $options['bitly_login']->option_value)) {
self::acquireBitlyClickStats($options['bitly_api_key']->option_value, $options['bitly_login']->option_value);
}
//@TODO Capture click counts for goo.gl URLs
//self::acquireGooglClickStats
} else {
$this->logger->logUserInfo("Limit of links to expand reached.", __METHOD__ . ',' . __LINE__);
}
}
/**
* Main control method
* @return Page markup
*/
public function control()
{
if ($this->shouldRefreshCache()) {
$post_dao = DAOFactory::getDAO('PostDAO');
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('geoencoder', true);
if (isset($options['gmaps_api_key']->option_value)) {
$api_key = $options['gmaps_api_key']->option_value;
} else {
$api_key = NULL;
}
$network = isset($_GET['n']) ? $_GET['n'] : 'twitter';
$type = isset($_GET['t']) ? $_GET['t'] : 'post';
$post_id = isset($_GET['pid']) ? $_GET['pid'] : 'post_id';
if ($type == 'post' && $post_dao->isPostInDB($post_id, $network)) {
$this->addHeaderJavaScript('plugins/geoencoder/assets/js/generatemap.js');
$this->addToView('gmaps_api', $api_key);
$post = $post_dao->getPost($post_id, $network);
$this->addToView('post', $post);
$post_rows = $post_dao->getRelatedPosts($post_id, $network);
$posts_json = $this->processLocations($post_rows, $post_id);
$this->addToView('posts_data', $posts_json);
} else {
$this->addErrorMessage('No visualization data found for this post');
}
}
return $this->generateView();
}
public function crawl()
{
$config = Config::getInstance();
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('flickrthumbnails', true);
$api_key = $options['flickr_api_key']->option_value;
if (isset($api_key) && $api_key != '') {
$logger = Logger::getInstance();
$fa = new FlickrAPIAccessor($api_key);
$ldao = DAOFactory::getDAO('LinkDAO');
$flickrlinkstoexpand = $ldao->getLinksToExpandByURL('http://flic.kr/');
if (count($flickrlinkstoexpand > 0)) {
$logger->logStatus(count($flickrlinkstoexpand) . " Flickr links to expand", "Flickr Plugin");
} else {
$logger->logStatus("No Flickr links to expand", "Flickr Plugin");
}
foreach ($flickrlinkstoexpand as $fl) {
$eurl = $fa->getFlickrPhotoSource($fl);
if ($eurl["expanded_url"] != '') {
$ldao->saveExpandedUrl($fl, $eurl["expanded_url"], '', 1);
} elseif ($eurl["error"] != '') {
$ldao->saveExpansionError($fl, $eurl["error"]);
}
}
$logger->close();
# Close logging
}
}
public function crawl()
{
$config = Config::getInstance();
$logger = Logger::getInstance();
$instance_dao = DAOFactory::getDAO('TwitterInstanceDAO');
$owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
$owner_dao = DAOFactory::getDAO('OwnerDAO');
$instance_hashtag_dao = DAOFactory::getDAO('InstanceHashtagDAO');
// get oauth values
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('twitter', true);
$current_owner = $owner_dao->getByEmail(Session::getLoggedInUser());
$instances = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'twitter');
foreach ($instances as $instance) {
$logger->setUsername($instance->network_username);
$logger->logUserSuccess("Starting to collect data for " . $instance->network_username . " on Twitter.", __METHOD__ . ',' . __LINE__);
$tokens = $owner_instance_dao->getOAuthTokens($instance->id);
$num_twitter_errors = isset($options['num_twitter_errors']) ? $options['num_twitter_errors']->option_value : null;
$dashboard_module_cacher = new DashboardModuleCacher($instance);
try {
if (isset($tokens['oauth_access_token']) && $tokens['oauth_access_token'] != '' && isset($tokens['oauth_access_token_secret']) && $tokens['oauth_access_token_secret'] != '') {
$archive_limit = isset($options['archive_limit']->option_value) ? $options['archive_limit']->option_value : 3200;
$api = new CrawlerTwitterAPIAccessorOAuth($tokens['oauth_access_token'], $tokens['oauth_access_token_secret'], $options['oauth_consumer_key']->option_value, $options['oauth_consumer_secret']->option_value, $archive_limit, $num_twitter_errors);
$twitter_crawler = new TwitterCrawler($instance, $api);
$instance_dao->updateLastRun($instance->id);
$twitter_crawler->fetchInstanceUserTweets();
$twitter_crawler->fetchInstanceUserMentions();
$twitter_crawler->fetchInstanceUserFriends();
$twitter_crawler->fetchInstanceUserFollowers();
$twitter_crawler->fetchInstanceUserGroups();
$twitter_crawler->fetchRetweetsOfInstanceUser();
$twitter_crawler->fetchInstanceUserFavorites();
$twitter_crawler->updateStaleGroupMemberships();
$twitter_crawler->fetchStrayRepliedToTweets();
$twitter_crawler->fetchUserFriendsByIDs();
$twitter_crawler->fetchUnloadedFriendDetails();
$twitter_crawler->fetchUnloadedFollowerDetails();
$twitter_crawler->cleanUpFollows();
$twitter_crawler->updateFriendsProfiles();
//Retrieve search results for saved keyword/hashtags
$instances_hashtags = $instance_hashtag_dao->getByInstance($instance->id);
foreach ($instances_hashtags as $instance_hashtag) {
$twitter_crawler->fetchInstanceHashtagTweets($instance_hashtag);
}
} else {
throw new Exception('Missing Twitter OAuth tokens.');
}
} catch (Exception $e) {
$logger->logUserError(get_class($e) . " while crawling " . $instance->network_username . " on Twitter: " . $e->getMessage(), __METHOD__ . ',' . __LINE__);
}
$dashboard_module_cacher->cacheDashboardModules();
// Save instance
if (isset($twitter_crawler->user)) {
$instance_dao->save($instance, $twitter_crawler->user->post_count, $logger);
}
Reporter::reportVersion($instance);
$logger->logUserSuccess("Finished collecting data for " . $instance->network_username . " on Twitter.", __METHOD__ . ',' . __LINE__);
}
}
/**
* Constructor
* @param Instance $instance
* @param CrawlerTwitterAPIAccessorOAuth $api
* @return TwitterCrawler
*/
public function __construct($instance, $api) {
$this->instance = $instance;
$this->api = $api;
$this->logger = Logger::getInstance();
$this->logger->setUsername($instance->network_username);
$this->user_dao = DAOFactory::getDAO('UserDAO');
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$this->twitter_options = $plugin_option_dao->getOptionsHash('twitter');
}
public function crawl()
{
$logger = Logger::getInstance();
$config = Config::getInstance();
$instance_dao = DAOFactory::getDAO('InstagramInstanceDAO');
$owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
$owner_dao = DAOFactory::getDAO('OwnerDAO');
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('instagram', true);
//get cached
$max_api_calls = isset($options['max_api_calls']) ? $options['max_api_calls']->option_value : 2500;
$current_owner = $owner_dao->getByEmail(Session::getLoggedInUser());
//crawl instagram user profiles and pages
$instances = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'instagram');
foreach ($instances as $instance) {
$logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username);
$logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
$tokens = $owner_instance_dao->getOAuthTokens($instance->id);
$access_token = $tokens['oauth_access_token'];
$instance_dao->updateLastRun($instance->id);
$dashboard_module_cacher = new DashboardModuleCacher($instance);
try {
/**
* 1. Fetch user info, media + its likes and comments.
* 2. Fetch user's likes.
* 3. Fetch user's friends, and update stale relationships.
* 4. Fetch user's followers, and update stale relationships.
* 5. Update stale friends' profiles.
*/
$instagram_crawler = new InstagramCrawler($instance, $access_token, $max_api_calls);
$instagram_crawler->fetchPostsAndReplies();
$instagram_crawler->fetchLikes();
$instagram_crawler->fetchFriends();
$instagram_crawler->fetchFollowers();
$instagram_crawler->updateStaleFriendsProfiles();
} catch (Instagram\Core\ApiAuthException $e) {
//The access token is invalid, save in owner_instances table
$owner_instance_dao->setAuthErrorByTokens($instance->id, $access_token, '', $e->getMessage());
//Send email alert
//Get owner by auth tokens first, then send to that person
$owner_email_to_notify = $owner_instance_dao->getOwnerEmailByInstanceTokens($instance->id, $access_token, '');
$email_attempt = $this->sendInvalidOAuthEmailAlert($owner_email_to_notify, $instance->network_username);
if ($email_attempt) {
$logger->logUserInfo('Sent reauth email to ' . $owner_email_to_notify, __METHOD__ . ',' . __LINE__);
} else {
$logger->logInfo('Didn\'t send reauth email to ' . $owner_email_to_notify, __METHOD__ . ',' . __LINE__);
}
$logger->logUserError(get_class($e) . ' ' . $e->getMessage(), __METHOD__ . ',' . __LINE__);
} catch (Exception $e) {
$logger->logUserError(get_class($e) . ' ' . $e->getMessage(), __METHOD__ . ',' . __LINE__);
}
$dashboard_module_cacher->cacheDashboardModules();
$instance_dao->save($instagram_crawler->instance, 0, $logger);
Reporter::reportVersion($instance);
$logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
}
}
public function crawl()
{
$logger = Logger::getInstance();
$config = Config::getInstance();
$instance_dao = DAOFactory::getDAO('InstanceDAO');
$owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
$owner_dao = DAOFactory::getDAO('OwnerDAO');
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('foursquare', true);
$current_owner = $owner_dao->getByEmail(Session::getLoggedInUser());
$instances = $instance_dao->getAllActiveInstancesStalestFirstByNetwork('foursquare');
// Check the client id and secret are set or we can't crawl
if (isset($options['foursquare_client_id']->option_value) && isset($options['foursquare_client_secret']->option_value)) {
// For each instance of foursquare on this install
foreach ($instances as $instance) {
if (!$owner_instance_dao->doesOwnerHaveAccessToInstance($current_owner, $instance)) {
// Owner doesn't have access to this instance; let's not crawl it.
continue;
}
// Set the user name in the log
$logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username);
// Write to the log that we have started to collect data
$logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
// Get the OAuth tokens for this user
$tokens = $owner_instance_dao->getOAuthTokens($instance->id);
// Set the access token
$access_token = $tokens['oauth_access_token'];
// Update the last time we crawled
$instance_dao->updateLastRun($instance->id);
// Create a new crawler
$crawler = new FoursquareCrawler($instance, $access_token);
// Check the OAuth tokens we have are valid
try {
$logger->logInfo("About to initializeInstanceUser", __METHOD__ . ',' . __LINE__);
$user = $crawler->initializeInstanceUser($access_token, $current_owner->id);
if (isset($user) && $user instanceof User) {
$logger->logInfo("User initialized", __METHOD__ . ',' . __LINE__);
}
// Get the data we want and store it in the database
$logger->logInfo("About to fetchInstanceUserCheckins", __METHOD__ . ',' . __LINE__);
$crawler->fetchInstanceUserCheckins();
} catch (Exception $e) {
// Catch any errors that happen when we check the validity of the OAuth tokens
$logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__);
}
$logger->logInfo("About to cache dashboard modules", __METHOD__ . ',' . __LINE__);
// Cache the insights to improve the page load speed of the dashboard
$dashboard_module_cacher = new DashboardModuleCacher($instance);
$dashboard_module_cacher->cacheDashboardModules();
$instance_dao->save($crawler->instance, 0, $logger);
Reporter::reportVersion($instance);
// Tell the user crawling was sucessful
$logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
}
}
}
public function crawl()
{
$logger = Logger::getInstance();
$config = Config::getInstance();
$instance_dao = DAOFactory::getDAO('FacebookInstanceDAO');
$owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
$owner_dao = DAOFactory::getDAO('OwnerDAO');
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('facebook', true);
//get cached
$max_crawl_time = isset($options['max_crawl_time']) ? $options['max_crawl_time']->option_value : 20;
//convert to seconds
$max_crawl_time = $max_crawl_time * 60;
$current_owner = $owner_dao->getByEmail(Session::getLoggedInUser());
//crawl Facebook user profiles and pages
$profiles = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'facebook');
$pages = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'facebook page');
$instances = array_merge($profiles, $pages);
foreach ($instances as $instance) {
$logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username);
$logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
$tokens = $owner_instance_dao->getOAuthTokens($instance->id);
$access_token = $tokens['oauth_access_token'];
$instance_dao->updateLastRun($instance->id);
$facebook_crawler = new FacebookCrawler($instance, $access_token, $max_crawl_time);
$dashboard_module_cacher = new DashboardModuleCacher($instance);
try {
$facebook_crawler->fetchPostsAndReplies();
} catch (APIOAuthException $e) {
$logger->logUserError(get_class($e) . ": " . $e->getMessage(), __METHOD__ . ',' . __LINE__);
//Don't send reauth email if it's app-level API rate limting
//https://developers.facebook.com/docs/reference/ads-api/api-rate-limiting/#applimit
if (strpos($e->getMessage(), 'Application request limit reached') === false && strpos($e->getMessage(), 'Please retry your request later.') === false) {
//The access token is invalid, save in owner_instances table
$owner_instance_dao->setAuthErrorByTokens($instance->id, $access_token, '', $e->getMessage());
//Send email alert
//Get owner by auth tokens first, then send to that person
$owner_email_to_notify = $owner_instance_dao->getOwnerEmailByInstanceTokens($instance->id, $access_token, '');
$email_attempt = $this->sendInvalidOAuthEmailAlert($owner_email_to_notify, $instance->network_username);
if ($email_attempt) {
$logger->logUserInfo('Sent reauth email to ' . $owner_email_to_notify, __METHOD__ . ',' . __LINE__);
} else {
$logger->logInfo('Didn\'t send reauth email to ' . $owner_email_to_notify, __METHOD__ . ',' . __LINE__);
}
} else {
$logger->logInfo('Facebook API returned an error: ' . $e->getMessage() . ' Do nothing now and try again later', __METHOD__ . ',' . __LINE__);
}
} catch (Exception $e) {
$logger->logUserError(get_class($e) . ": " . $e->getMessage(), __METHOD__ . ',' . __LINE__);
}
$dashboard_module_cacher->cacheDashboardModules();
$instance_dao->save($facebook_crawler->instance, 0, $logger);
Reporter::reportVersion($instance);
$logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
}
}
/**
* @return str
*/
public function authControl()
{
$config = Config::getInstance();
Loader::definePathConstants();
$this->setViewTemplate(THINKUP_WEBAPP_PATH . 'plugins/twitterrealtime/view/twitterrealtime.account.index.tpl');
$this->view_mgr->addHelp('twitterrealtime', 'userguide/settings/plugins/twitterrealtime');
$id = DAOFactory::getDAO('InstanceDAO');
$od = DAOFactory::getDAO('OwnerDAO');
// get plugin option values if defined...
$plugin_options = $this->getPluginOptions();
// get oauth option values from twitter plugin.
// @TODO -- what is the right way to do this?
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$twitter_options = $plugin_option_dao->getOptionsHash('twitter', true);
$oauth_consumer_key = null;
if (isset($twitter_options['oauth_consumer_key'])) {
$oauth_consumer_key = $twitter_options['oauth_consumer_key']->option_value;
}
$oauth_consumer_secret = null;
if (isset($twitter_options['oauth_consumer_secret'])) {
$oauth_consumer_secret = $twitter_options['oauth_consumer_secret']->option_value;
}
// @TODO - get any other option values as necessary
// $archive_limit = $this->getPluginOption('archive_limit');
$auth_from_twitter = '';
if (isset($oauth_consumer_key) && isset($oauth_consumer_secret)) {
$to = new TwitterOAuth($oauth_consumer_key, $oauth_consumer_secret);
/* Request tokens from twitter */
$tok = $to->getRequestToken();
if (isset($tok['oauth_token'])) {
$token = $tok['oauth_token'];
$_SESSION['oauth_request_token_secret'] = $tok['oauth_token_secret'];
/* Build the authorization URL */
$oauthorize_link = $to->getAuthorizeURL($token);
// create indication that auth from twitter plugin is okay
$auth_from_twitter = "Using the Twitter Consumer key and secret as set in " . "the <a href=\"./?p=twitter\">Twitter plugin</a>.";
} else {
//set error message here
$this->addErrorMessage("Unable to obtain OAuth token. Check your Twitter plugin consumer key and secret configuration.");
$oauthorize_link = '';
}
} else {
$this->addErrorMessage("Missing required settings! Please configure the Twitter plugin.");
$oauthorize_link = '';
}
$owner_instances = $id->getByOwnerAndNetwork($this->owner, 'twitter');
$this->addToView('owner_instances', $owner_instances);
$this->addToView('oauthorize_link', $oauthorize_link);
$this->addToView('auth_from_twitter', $auth_from_twitter);
// add plugin options from
$this->addOptionForm();
$plugin = new TwitterRealtimePlugin();
$this->addToView('is_configured', $plugin->isConfigured());
return $this->generateView();
}
public function authControl()
{
if (!$this->is_missing_param) {
$fb_user = null;
$msg = '';
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('facebook', true);
//get cached
$facebook = new Facebook($options['facebook_api_key']->option_value, $options['facebook_api_secret']->option_value);
$fb_user = $facebook->api_client->users_getLoggedInUser();
$msg .= "Facebook user is logged in and user ID set<br />";
$fb_username = $facebook->api_client->users_getInfo($fb_user, 'name');
if (isset($_GET['sessionKey']) && isset($fb_user) && $fb_user > 0) {
$fb_username = $fb_username[0]['name'];
$session_key = $_GET['sessionKey'];
$owner_dao = DAOFactory::getDAO('OwnerDAO');
$instance_dao = DAOFactory::getDAO('InstanceDAO');
$oid = DAOFactory::getDAO('OwnerInstanceDAO');
$user_dao = DAOFactory::getDAO('UserDAO');
$owner = $owner_dao->getByEmail($this->getLoggedInUser());
$i = $instance_dao->getByUserIdOnNetwork($fb_user, 'facebook');
if (isset($i)) {
$msg .= "Instance exists<br />";
$oi = $oid->get($owner->id, $i->id);
if ($oi == null) {
//Instance already exists, owner instance doesn't
$oid->insert($owner->id, $i->id, $session_key);
//Add owner instance with session key
$msg .= "Created owner instance.<br />";
}
} else {
//Instance does not exist
$msg .= "Instance does not exist<br />";
$instance_dao->insert($fb_user, $fb_username, 'facebook');
$msg .= "Created instance";
$i = $instance_dao->getByUserIdOnNetwork($fb_user, 'facebook');
$oid->insert($owner->id, $i->id, $session_key);
$msg .= "Created owner instance.<br />";
}
if (!$user_dao->isUserInDB($fb_user, 'facebook')) {
$r = array('user_id' => $fb_user, 'user_name' => $fb_username, 'full_name' => $fb_username, 'avatar' => '', 'location' => '', 'description' => '', 'url' => '', 'is_protected' => '', 'follower_count' => 0, 'friend_count' => 0, 'post_count' => 0, 'last_updated' => '', 'last_post' => '', 'joined' => '', 'last_post_id' => '', 'network' => 'facebook');
$u = new User($r, 'Owner info');
$user_dao->updateUser($u);
}
} else {
$msg .= "No session key or logged in Facebook user.";
}
$config = Config::getInstance();
$msg .= '<br /> <a href="' . $config->getValue('site_root_path') . 'account/">Back to your account</a>.';
$this->addInfoMessage($msg);
}
return $this->generateView();
}
public function authControl()
{
if (!$this->is_missing_param) {
$request_token = $_GET['oauth_token'];
$request_token_secret = SessionCache::get('oauth_request_token_secret');
// get oauth values
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('twitter', true);
//get cached
$to = new TwitterOAuth($options['oauth_consumer_key']->option_value, $options['oauth_consumer_secret']->option_value, $request_token, $request_token_secret);
$tok = $to->getAccessToken();
if (isset($tok['oauth_token']) && isset($tok['oauth_token_secret'])) {
$api = new TwitterAPIAccessorOAuth($tok['oauth_token'], $tok['oauth_token_secret'], $options['oauth_consumer_key']->option_value, $options['oauth_consumer_secret']->option_value, $options['num_twitter_errors']->option_value, $options['max_api_calls_per_crawl']->option_value, false);
$authed_twitter_user = $api->verifyCredentials();
// echo "User ID: ". $authed_twitter_user['user_id'];
// echo "User name: ". $authed_twitter_user['user_name'];
$owner_dao = DAOFactory::getDAO('OwnerDAO');
$owner = $owner_dao->getByEmail($this->getLoggedInUser());
if ((int) $authed_twitter_user['user_id'] > 0) {
$instance_dao = DAOFactory::getDAO('TwitterInstanceDAO');
$instance = $instance_dao->getByUsername($authed_twitter_user['user_name'], 'twitter');
$owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
if (isset($instance)) {
$owner_instance = $owner_instance_dao->get($owner->id, $instance->id);
if ($owner_instance != null) {
$owner_instance_dao->updateTokens($owner->id, $instance->id, $tok['oauth_token'], $tok['oauth_token_secret']);
$this->addSuccessMessage($authed_twitter_user['user_name'] . " on Twitter is already set up in ThinkUp! To add a different Twitter account, " . "log out of Twitter.com in your browser and authorize ThinkUp again.");
} else {
if ($owner_instance_dao->insert($owner->id, $instance->id, $tok['oauth_token'], $tok['oauth_token_secret'])) {
$this->addSuccessMessage("Success! " . $authed_twitter_user['user_name'] . " on Twitter has been added to ThinkUp!");
} else {
$this->addErrorMessage("Error: Could not create an owner instance.");
}
}
} else {
$instance_dao->insert($authed_twitter_user['user_id'], $authed_twitter_user['user_name']);
$instance = $instance_dao->getByUsername($authed_twitter_user['user_name']);
if ($owner_instance_dao->insert($owner->id, $instance->id, $tok['oauth_token'], $tok['oauth_token_secret'])) {
$this->addSuccessMessage("Success! " . $authed_twitter_user['user_name'] . " on Twitter has been added to ThinkUp!");
} else {
$this->addErrorMessage("Error: Could not create an owner instance.");
}
}
}
} else {
$msg = "Error: Twitter authorization did not complete successfully. Check if your account already " . " exists. If not, please try again.";
$this->addErrorMessage($msg);
}
$this->view_mgr->clear_all_cache();
}
return $this->generateView();
}
public function crawl()
{
$logger = Logger::getInstance();
$config = Config::getInstance();
$id = DAOFactory::getDAO('InstanceDAO');
$oid = DAOFactory::getDAO('OwnerInstanceDAO');
$od = DAOFactory::getDAO('OwnerDAO');
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('facebook', true);
//get cached
$current_owner = $od->getByEmail(Session::getLoggedInUser());
//crawl Facebook user profiles
$instances = $id->getAllActiveInstancesStalestFirstByNetwork('facebook');
foreach ($instances as $instance) {
if (!$oid->doesOwnerHaveAccess($current_owner, $instance)) {
// Owner doesn't have access to this instance; let's not crawl it.
continue;
}
$logger->setUsername($instance->network_username);
$logger->logUserSuccess("Starting to collect data for " . $instance->network_username . " on Facebook.", __METHOD__ . ',' . __LINE__);
$tokens = $oid->getOAuthTokens($instance->id);
$access_token = $tokens['oauth_access_token'];
$id->updateLastRun($instance->id);
$crawler = new FacebookCrawler($instance, $access_token);
try {
$crawler->fetchInstanceUserInfo();
$crawler->fetchUserPostsAndReplies($instance->network_user_id);
} catch (Exception $e) {
$logger->logUserError('PROFILE EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__);
}
$id->save($crawler->instance, 0, $logger);
$logger->logUserSuccess("Finished collecting data for " . $instance->network_username . " on Facebook.", __METHOD__ . ',' . __LINE__);
}
//crawl Facebook pages
$instances = $id->getAllActiveInstancesStalestFirstByNetwork('facebook page');
foreach ($instances as $instance) {
$logger->setUsername($instance->network_username);
$logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s Facebook Page.", __METHOD__ . ',' . __LINE__);
$tokens = $oid->getOAuthTokens($instance->id);
$access_token = $tokens['oauth_access_token'];
$id->updateLastRun($instance->id);
$crawler = new FacebookCrawler($instance, $access_token);
try {
$crawler->fetchPagePostsAndReplies($instance->network_user_id);
} catch (Exception $e) {
$logger->logUserError('PAGE EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__);
}
$id->save($crawler->instance, 0, $logger);
$logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s Facebook Page.", __METHOD__ . ',' . __LINE__);
}
}
public function crawl()
{
$logger = Logger::getInstance();
$config = Config::getInstance();
$id = DAOFactory::getDAO('InstanceDAO');
$oid = DAOFactory::getDAO('OwnerInstanceDAO');
$od = DAOFactory::getDAO('OwnerDAO');
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('facebook', true);
//get cached
$current_owner = $od->getByEmail(Session::getLoggedInUser());
//crawl Facebook user profiles
$instances = $id->getAllActiveInstancesStalestFirstByNetwork('facebook');
foreach ($instances as $instance) {
if (!$oid->doesOwnerHaveAccess($current_owner, $instance)) {
// Owner doesn't have access to this instance; let's not crawl it.
continue;
}
$logger->setUsername($instance->network_username);
$tokens = $oid->getOAuthTokens($instance->id);
$session_key = $tokens['oauth_access_token'];
$fb = new Facebook($options['facebook_api_key']->option_value, $options['facebook_api_secret']->option_value);
$id->updateLastRun($instance->id);
$crawler = new FacebookCrawler($instance, $fb);
try {
$crawler->fetchInstanceUserInfo($instance->network_user_id, $session_key);
$crawler->fetchUserPostsAndReplies($instance->network_user_id, $session_key);
} catch (Exception $e) {
$logger->logStatus('PROFILE EXCEPTION: ' . $e->getMessage(), get_class($this));
}
$id->save($crawler->instance, $crawler->owner_object->post_count, $logger);
}
//crawl Facebook pages
$instances = $id->getAllActiveInstancesStalestFirstByNetwork('facebook page');
foreach ($instances as $instance) {
$logger->setUsername($instance->network_username);
$tokens = $oid->getOAuthTokens($instance->id);
$session_key = $tokens['oauth_access_token'];
$fb = new Facebook($options['facebook_api_key']->option_value, $options['facebook_api_secret']->option_value);
$id->updateLastRun($instance->id);
$crawler = new FacebookCrawler($instance, $fb);
try {
$crawler->fetchPagePostsAndReplies($instance->network_user_id, $instance->network_viewer_id, $session_key);
} catch (Exception $e) {
$logger->logStatus('PAGE EXCEPTION: ' . $e->getMessage(), get_class($this));
}
$id->save($crawler->instance, 0, $logger);
}
$logger->close();
# Close logging
}
/**
* Run when the crawler does
*/
public function crawl()
{
$logger = Logger::getInstance();
$logger->setUsername(null);
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('expandurls', true);
//Flickr image thumbnails
if (isset($options['flickr_api_key']->option_value)) {
self::expandFlickrThumbnails($options['flickr_api_key']->option_value);
}
//@TODO: Bit.ly URLs
//Remaining URLs
$link_limit = isset($options['links_to_expand']->option_value) ? (int) $options['links_to_expand']->option_value : 1500;
self::expandRemainingURLs($link_limit);
}
public static function getQueue()
{
// get stream plugin settings
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('twitterrealtime', true);
if (is_null(self::$queue)) {
if (isset($options['use_redis']) && $options['use_redis']->option_value == "true") {
// we need a redis queue
self::$queue = new StreamMessageQueueRedis();
} else {
// we need a mysql queue
self::$queue = new StreamMessageQueueMySQL();
}
}
return self::$queue;
}
/**
* @static
* @param $oauth_access_token
* @param $oauth_access_token_secret
* @return ConsumerUserStream|null
*/
public static function getInstance($oauth_access_token, $oauth_access_token_secret)
{
$sc = null;
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('twitter', true);
//get cached
if (isset($options['oauth_consumer_key']) && isset($options['oauth_consumer_secret'])) {
define('TWITTER_CONSUMER_KEY', $options['oauth_consumer_key']->option_value);
define('TWITTER_CONSUMER_SECRET', $options['oauth_consumer_secret']->option_value);
$sc = new ConsumerUserStream($oauth_access_token, $oauth_access_token_secret);
} else {
$logger = Logger::getInstance('stream_log_location');
$logger->logError("Error: could not obtain Twitter app consumer key and secret from Twitter plugin " . "settings.", __METHOD__ . ',' . __LINE__);
}
return $sc;
}
/**
* Outputs javascript callback string with json array/list of post as an argument
*/
public function authControl()
{
if (!$this->is_missing_param) {
$instance_dao = DAOFactory::getDAO('InstanceDAO');
if ($instance_dao->isUserConfigured($_GET['u'], $_GET['n'])) {
$username = $_GET['u'];
$ownerinstance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
$owner_dao = DAOFactory::getDAO('OwnerDAO');
$owner = $owner_dao->getByEmail($this->getLoggedInUser());
$instance = $instance_dao->getByUsername($username, $_GET['n']);
if (!$ownerinstance_dao->doesOwnerHaveAccess($owner, $instance)) {
echo '{"status":"failed","message":"Insufficient privileges."}';
} else {
echo "tu_grid_search.populate_grid(";
$post_dao = DAOFactory::GetDAO('PostDAO');
$posts_it = null;
if ($_GET['d'] == 'tweets-all') {
$posts_it = $post_dao->getAllPostsByUsernameIterator($_GET['u'], $_GET['n'], self::MAX_ROWS);
} else {
if ($_GET['d'] == 'tweets-mostreplies') {
$posts_it = $post_dao->getAllMentionsIterator($_GET['u'], self::MAX_ROWS, $_GET['n']);
} else {
if ($_GET['d'] == 'tweets-mostretweeted') {
$posts_it = $post_dao->getMostRetweetedPostsIterator($_GET['u'], $_GET['n'], self::MAX_ROWS, self::MAX_RT_DAYS);
}
}
}
echo '{"status":"success","posts": [' . "\n";
$cnt = 0;
foreach ($posts_it as $key => $value) {
$cnt++;
$data = array('id' => $cnt, 'text' => $value->post_text, 'post_id' => $value->post_id, 'author' => $value->author_username, 'date' => $value->adj_pub_date);
echo json_encode($data) . ",\n";
flush();
}
$data = array('id' => -1, 'text' => 'Last Post', 'author' => 'nobody');
echo json_encode($data);
echo ']});';
}
} else {
echo '{"status":"failed","message":"' . $_GET['u'] . 'is not configured."}';
}
} else {
echo '{"status":"failed","message":"Missing Parameters"}';
}
}
/**
* Main control method
*/
public function authControl()
{
$this->setViewTemplate('post.index.tpl');
$network = isset($_GET['n']) ? $_GET['n'] : 'twitter';
$_GET['n'] = $network;
if ($this->shouldRefreshCache()) {
if (isset($_GET['t']) && is_numeric($_GET['t']) && $this->post_dao->isPostInDB($_GET['t'], $network)) {
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('geoencoder', true);
if (isset($options['distance_unit']->option_value)) {
$distance_unit = $options['distance_unit']->option_value;
} else {
$distance_unit = 'km';
}
$post_id = $_GET['t'];
$post = $this->post_dao->getPost($post_id, $network);
$this->addToView('post', $post);
$this->addToView('unit', $distance_unit);
// costly query
//$this->addToView('likely_orphans', $this->post_dao->getLikelyOrphansForParent($post->pub_date,
//$post->author_user_id,$post->author_username, 15) );
//$this->addToView('all_tweets', $this->post_dao->getAllPosts($post->author_user_id, 15) );
$all_replies = $this->post_dao->getRepliesToPost($post_id, $network, 'default', $distance_unit);
$this->addToView('replies', $all_replies);
$all_replies_by_location = $this->post_dao->getRepliesToPost($post_id, $network, 'location', $distance_unit);
$this->addToView('replies_by_location', $all_replies_by_location);
$all_replies_count = count($all_replies);
$this->addToView('reply_count', $all_replies_count);
$all_retweets = $this->post_dao->getRetweetsOfPost($post_id, $network, 'default', $distance_unit);
$this->addToView('retweets', $all_retweets);
$all_retweets_by_location = $this->post_dao->getRetweetsOfPost($post_id, $network, 'location', $distance_unit);
$this->addToView('retweets_by_location', $all_retweets_by_location);
$retweet_reach = $this->post_dao->getPostReachViaRetweets($post_id, $network);
$this->addToView('retweet_reach', $retweet_reach);
$public_replies = $this->post_dao->getPublicRepliesToPost($post_id, $network);
$public_replies_count = count($public_replies);
$this->addToView('public_reply_count', $public_replies_count);
$private_replies_count = $all_replies_count - $public_replies_count;
$this->addToView('private_reply_count', $private_replies_count);
} else {
$this->addErrorMessage('Post not found');
}
}
return $this->generateView();
}
/**
* Constructor
* @return StreamMasterCollect
*/
public function __construct()
{
$config = Config::getInstance();
$this->streaming_dir = $config->getValue('source_root_path') . 'webapp/plugins/twitterrealtime/streaming';
// @TODO -- get this from plugin information now
$this->log_dir = $config->getValue('source_root_path') . 'logs';
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('twitterrealtime', true);
if (isset($options['php_path'])) {
$this->php_path = $options['php_path']->option_value;
} else {
$this->php_path = null;
// path to the php interp on the user's system
}
$this->stream_proc_dao = DAOFactory::getDAO('StreamProcDAO');
$this->instance_dao = DAOFactory::getDAO('InstanceDAO');
$this->owner_dao = DAOFactory::getDAO('OwnerDAO');
}
public function crawl()
{
$logger = Logger::getInstance();
$config = Config::getInstance();
$instance_dao = DAOFactory::getDAO('InstanceDAO');
$owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
$owner_dao = DAOFactory::getDAO('OwnerDAO');
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('facebook', true);
//get cached
$max_crawl_time = isset($options['max_crawl_time']) ? $options['max_crawl_time']->option_value : 20;
//convert to seconds
$max_crawl_time = $max_crawl_time * 60;
$current_owner = $owner_dao->getByEmail(Session::getLoggedInUser());
//crawl Facebook user profiles and pages
$profiles = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'facebook');
$pages = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'facebook page');
$instances = array_merge($profiles, $pages);
foreach ($instances as $instance) {
$logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username);
$logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
$tokens = $owner_instance_dao->getOAuthTokens($instance->id);
$access_token = $tokens['oauth_access_token'];
$instance_dao->updateLastRun($instance->id);
$facebook_crawler = new FacebookCrawler($instance, $access_token, $max_crawl_time);
$dashboard_module_cacher = new DashboardModuleCacher($instance);
try {
$facebook_crawler->fetchPostsAndReplies();
} catch (APIOAuthException $e) {
//The access token is invalid, save in owner_instances table
$owner_instance_dao->setAuthError($current_owner->id, $instance->id, $e->getMessage());
//Send email alert
$this->sendInvalidOAuthEmailAlert($current_owner->email, $instance->network_username);
$logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__);
} catch (Exception $e) {
$logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__);
}
$dashboard_module_cacher->cacheDashboardModules();
$instance_dao->save($facebook_crawler->instance, 0, $logger);
Reporter::reportVersion($instance);
$logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
}
}
public function crawl()
{
$logger = Logger::getInstance();
$config = Config::getInstance();
$instance_dao = DAOFactory::getDAO('InstanceDAO');
$owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
$owner_dao = DAOFactory::getDAO('OwnerDAO');
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('youtube', true);
//get cached
$max_crawl_time = isset($options['max_crawl_time']) ? $options['max_crawl_time']->option_value : 20;
//convert to seconds
$max_crawl_time = $max_crawl_time * 60;
$developer_key = isset($options['developer_key']) ? $options['developer_key']->option_value : null;
$max_comments = isset($options['max_comments']) ? $options['max_comments']->option_value : null;
$current_owner = $owner_dao->getByEmail(Session::getLoggedInUser());
//crawl youtube users
$instances = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'youtube');
if (isset($options['youtube_client_id']->option_value) && isset($options['youtube_client_secret']->option_value)) {
foreach ($instances as $instance) {
$logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username);
$logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
$tokens = $owner_instance_dao->getOAuthTokens($instance->id);
$access_token = $tokens['oauth_access_token'];
$refresh_token = $tokens['oauth_access_token_secret'];
$instance_dao->updateLastRun($instance->id);
$youtube_crawler = new YouTubeCrawler($instance, $access_token, $max_crawl_time, $developer_key, $max_comments);
$dashboard_module_cacher = new DashboardModuleCacher($instance);
try {
$youtube_crawler->initializeInstanceUser($options['youtube_client_id']->option_value, $options['youtube_client_secret']->option_value, $access_token, $refresh_token, $current_owner->id);
$youtube_crawler->fetchInstanceUserVideos();
} catch (Exception $e) {
$logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__);
}
$dashboard_module_cacher->cacheDashboardModules();
$instance_dao->save($youtube_crawler->instance, 0, $logger);
Reporter::reportVersion($instance);
$logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
}
}
}
public function crawl() {
$config = Config::getInstance();
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('flickrthumbnails', true);
$api_key = '';
if (isset($options['flickr_api_key']->option_value)) {
$api_key = $options['flickr_api_key']->option_value;
}
if (isset($api_key) && $api_key != '') {
$logger = Logger::getInstance();
$logger->setUsername(null);
$fa = new FlickrAPIAccessor($api_key);
$ldao = DAOFactory::getDAO('LinkDAO');
$flickrlinkstoexpand = $ldao->getLinksToExpandByURL('http://flic.kr/');
if (count($flickrlinkstoexpand) > 0) {
$logger->logUserInfo(count($flickrlinkstoexpand)." Flickr links to expand.", __METHOD__.','.__LINE__);
} else {
$logger->logUserInfo("There are no Flickr thumbnails to expand.", __METHOD__.','.__LINE__);
}
$total_thumbnails = 0;
$total_errors = 0;
foreach ($flickrlinkstoexpand as $fl) {
$eurl = $fa->getFlickrPhotoSource($fl);
if ($eurl["expanded_url"] != '') {
$ldao->saveExpandedUrl($fl, $eurl["expanded_url"], '', 1);
$total_thumbnails = $total_thumbnails + 1;
} elseif ($eurl["error"] != '') {
$ldao->saveExpansionError($fl, $eurl["error"]);
$total_errors = $total_errors + 1;
}
$logger->logUserSuccess($total_thumbnails." Flickr thumbnails expanded (".$total_errors." errors)",
__METHOD__.','.__LINE__);
}
}
}
public function crawl()
{
$logger = Logger::getInstance();
$config = Config::getInstance();
$instance_dao = DAOFactory::getDAO('InstanceDAO');
$owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
$owner_dao = DAOFactory::getDAO('OwnerDAO');
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('facebook', true);
//get cached
$max_crawl_time = isset($options['max_crawl_time']) ? $options['max_crawl_time']->option_value : 20;
//convert to seconds
$max_crawl_time = $max_crawl_time * 60;
$current_owner = $owner_dao->getByEmail(Session::getLoggedInUser());
//crawl Facebook user profiles and pages
$profiles = $instance_dao->getAllActiveInstancesStalestFirstByNetwork('facebook');
$pages = $instance_dao->getAllActiveInstancesStalestFirstByNetwork('facebook page');
$instances = array_merge($profiles, $pages);
foreach ($instances as $instance) {
if (!$owner_instance_dao->doesOwnerHaveAccessToInstance($current_owner, $instance)) {
// Owner doesn't have access to this instance; let's not crawl it.
continue;
}
$logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username);
$logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
$tokens = $owner_instance_dao->getOAuthTokens($instance->id);
$access_token = $tokens['oauth_access_token'];
$instance_dao->updateLastRun($instance->id);
$crawler = new FacebookCrawler($instance, $access_token, $max_crawl_time);
try {
$crawler->fetchPostsAndReplies();
} catch (Exception $e) {
$logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__);
}
$instance_dao->save($crawler->instance, 0, $logger);
$logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
}
}
/**
* Run when the crawler does
*/
public function crawl()
{
$logger = Logger::getInstance();
$logger->setUsername(null);
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('expandurls', true);
//Limit the number of links expanded each crawl
$this->link_limit = isset($options['links_to_expand']->option_value) ? (int) $options['links_to_expand']->option_value : 1500;
if ($this->link_limit != 0) {
//Flickr image thumbnails
if (isset($options['flickr_api_key']->option_value)) {
self::expandFlickrThumbnails($options['flickr_api_key']->option_value);
}
//Bit.ly URLs
if (isset($options['bitly_api_key']->option_value, $options['bitly_login']->option_value)) {
self::expandBitlyLinks($options['bitly_api_key']->option_value, $options['bitly_login']->option_value);
}
//Remaining links
self::expandRemainingURLs();
} else {
$logger->logUserInfo("Limit of links to expand reached.", __METHOD__ . ',' . __LINE__);
}
}
/**
* Run when the crawler does
* @TODO Set limit on total number of links to expand per crawler run in the plugin settings, for now 1500
*/
public function crawl() {
$logger = Logger::getInstance();
$logger->setUsername(null);
$ldao = DAOFactory::getDAO('LinkDAO');
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('expandurls', true);
$total_links_to_expand = isset($options['links_to_expand']->option_value) ?
(int)$options['links_to_expand']->option_value : 1500;
$linkstoexpand = $ldao->getLinksToExpand($total_links_to_expand);
$logger->logUserInfo(count($linkstoexpand)." links to expand. Please wait. Working...",
__METHOD__.','.__LINE__);
$total_expanded = 0;
$total_errors = 0;
foreach ($linkstoexpand as $l) {
if (Utils::validateURL($l)) {
$logger->logInfo("Expanding ".($total_expanded+1). " of ".count($linkstoexpand)." (".$l.")",
__METHOD__.','.__LINE__);
$eurl = self::untinyurl($l, $ldao);
if ($eurl != '') {
$ldao->saveExpandedUrl($l, $eurl);
$total_expanded = $total_expanded + 1;
} else {
$total_errors = $total_errors + 1;
}
} else {
$total_errors = $total_errors + 1;
$logger->logError($l." is not a valid URL; skipping expansion", __METHOD__.','.__LINE__);
}
}
$logger->logUserSuccess($total_expanded." URLs successfully expanded (".$total_errors." errors).",
__METHOD__.','.__LINE__);
}
