public function crawl()
{
$logger = Logger::getInstance();
$config = Config::getInstance();
$instance_dao = DAOFactory::getDAO('InstanceDAO');
$owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
$owner_dao = DAOFactory::getDAO('OwnerDAO');
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('googleplus', true);
//get cached
$current_owner = $owner_dao->getByEmail(Session::getLoggedInUser());
//crawl Google+ users
$instances = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'google+');
if (isset($options['google_plus_client_id']->option_value) && isset($options['google_plus_client_secret']->option_value)) {
foreach ($instances as $instance) {
$logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username);
$logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
$tokens = $owner_instance_dao->getOAuthTokens($instance->id);
$access_token = $tokens['oauth_access_token'];
$refresh_token = $tokens['oauth_access_token_secret'];
$instance_dao->updateLastRun($instance->id);
$crawler = new GooglePlusCrawler($instance, $access_token);
$insights_generator = new InsightsGenerator($instance);
try {
$crawler->initializeInstanceUser($options['google_plus_client_id']->option_value, $options['google_plus_client_secret']->option_value, $access_token, $refresh_token, $current_owner->id);
$crawler->fetchInstanceUserPosts();
} catch (Exception $e) {
$logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__);
}
$insights_generator->generateInsights();
$instance_dao->save($crawler->instance, 0, $logger);
$logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
}
}
}
public function crawl()
{
$logger = Logger::getInstance();
$config = Config::getInstance();
$instance_dao = DAOFactory::getDAO('InstanceDAO');
$owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
$owner_dao = DAOFactory::getDAO('OwnerDAO');
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('facebook', true);
//get cached
$max_crawl_time = isset($options['max_crawl_time']) ? $options['max_crawl_time']->option_value : 20;
//convert to seconds
$max_crawl_time = $max_crawl_time * 60;
$current_owner = $owner_dao->getByEmail(Session::getLoggedInUser());
//crawl Facebook user profiles and pages
$profiles = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'facebook');
$pages = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'facebook page');
$instances = array_merge($profiles, $pages);
foreach ($instances as $instance) {
$logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username);
$logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
$tokens = $owner_instance_dao->getOAuthTokens($instance->id);
$access_token = $tokens['oauth_access_token'];
$instance_dao->updateLastRun($instance->id);
$crawler = new FacebookCrawler($instance, $access_token, $max_crawl_time);
$insights_generator = new InsightsGenerator($instance);
try {
$crawler->fetchPostsAndReplies();
} catch (APIOAuthException $e) {
//The access token is invalid, save in owner_instances table
$owner_instance_dao->setAuthError($current_owner->id, $instance->id, $e->getMessage());
//Send email alert
$this->sendInvalidOAuthEmailAlert($current_owner->email, $instance->network_username);
$logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__);
} catch (Exception $e) {
$logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__);
}
$insights_generator->generateInsights();
$instance_dao->save($crawler->instance, 0, $logger);
$logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
}
}
/**
* Load instance dashboard
* @param str $username
* @param str $network
*/
private function loadDefaultDashboard()
{
if (isset($this->instance)) {
$this->setPageTitle($this->instance->network_username . "'s Dashboard");
$insight_dao = DAOFactory::getDAO('InsightDAO');
$hot_posts_data = $insight_dao->getPreCachedInsightData('PostMySQLDAO::getHotPosts', $this->instance->id, date('Y-m-d'));
if (isset($hot_posts_data)) {
$this->addToView('hot_posts_data', $hot_posts_data);
}
$click_stats_data = $insight_dao->getPreCachedInsightData('ShortLinkMySQLDAO::getRecentClickStats', $this->instance->id, date('Y-m-d'));
if (isset($click_stats_data)) {
$this->addToView('click_stats_data', $click_stats_data);
}
$post_dao = DAOFactory::getDAO('PostDAO');
$most_replied_to_1wk = $insight_dao->getPreCachedInsightData('PostMySQLDAO::getMostRepliedToPostsInLastWeek', $this->instance->id, date('Y-m-d'));
$this->addToView('most_replied_to_1wk', $most_replied_to_1wk);
$most_retweeted_1wk = $insight_dao->getPreCachedInsightData('PostMySQLDAO::getMostRetweetedPostsInLastWeek', $this->instance->id, date('Y-m-d'));
$this->addToView('most_retweeted_1wk', $most_retweeted_1wk);
//for now, only show most liked/faved posts on Facebook dashboard
//once we cache fave counts for Twitter, we can remove this conditional
if ($this->instance->network == "facebook" || $this->instance->network == "facebook page" || $this->instance->network == "google+") {
$most_faved_1wk = $post_dao->getMostFavedPostsInLastWeek($this->instance->network_username, $this->instance->network, 5);
$this->addToView('most_faved_1wk', $most_faved_1wk);
}
//follows - these are pre-cached in insights
$least_likely_followers = $insight_dao->getPreCachedInsightData('FollowMySQLDAO::getLeastLikelyFollowersThisWeek', $this->instance->id, date('Y-m-d'));
$this->addToView('least_likely_followers', $least_likely_followers);
//follower count history
//by day
$follower_count_dao = DAOFactory::getDAO('FollowerCountDAO');
$follower_count_history_by_day = $follower_count_dao->getHistory($this->instance->network_user_id, $this->instance->network, 'DAY', 5);
$this->addToView('follower_count_history_by_day', $follower_count_history_by_day);
//by week
$follower_count_history_by_week = $follower_count_dao->getHistory($this->instance->network_user_id, $this->instance->network, 'WEEK', 5);
$this->addToView('follower_count_history_by_week', $follower_count_history_by_week);
list($all_time_clients_usage, $latest_clients_usage) = $insight_dao->getPreCachedInsightData('PostMySQLDAO::getClientsUsedByUserOnNetwork', $this->instance->id, date('Y-m-d'));
$this->addToView('most_replied_to_1wk', $most_replied_to_1wk);
// The sliceVisibilityThreshold option in the chart will prevent small slices from being created
$all_time_clients_usage = InsightsGenerator::getClientUsageVisualizationData($all_time_clients_usage);
$this->addToView('all_time_clients_usage', $all_time_clients_usage);
// Only show the two most used clients for the last 25 posts
$latest_clients_usage = array_slice($latest_clients_usage, 0, 2);
$this->addToView('latest_clients_usage', $latest_clients_usage);
} else {
$this->addErrorMessage($username . " on " . ucwords($this->instance->network) . " isn't set up on this ThinkUp installation.");
}
}
public function crawl()
{
$config = Config::getInstance();
$logger = Logger::getInstance();
$instance_dao = DAOFactory::getDAO('TwitterInstanceDAO');
$owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
$owner_dao = DAOFactory::getDAO('OwnerDAO');
// get oauth values
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('twitter', true);
$current_owner = $owner_dao->getByEmail(Session::getLoggedInUser());
$instances = $instance_dao->getAllActiveInstancesStalestFirstByNetwork('twitter');
foreach ($instances as $instance) {
if (!$owner_instance_dao->doesOwnerHaveAccessToInstance($current_owner, $instance)) {
// Owner doesn't have access to this instance; let's not crawl it.
continue;
}
$logger->setUsername($instance->network_username);
$logger->logUserSuccess("Starting to collect data for " . $instance->network_username . " on Twitter.", __METHOD__ . ',' . __LINE__);
$tokens = $owner_instance_dao->getOAuthTokens($instance->id);
$noauth = true;
$num_twitter_errors = isset($options['num_twitter_errors']) ? $options['num_twitter_errors']->option_value : null;
$max_api_calls_per_crawl = isset($options['max_api_calls_per_crawl']) ? $options['max_api_calls_per_crawl']->option_value : 350;
if (isset($tokens['oauth_access_token']) && $tokens['oauth_access_token'] != '' && isset($tokens['oauth_access_token_secret']) && $tokens['oauth_access_token_secret'] != '') {
$noauth = false;
}
$api_calls_to_leave_unmade_per_minute = isset($options['api_calls_to_leave_unmade_per_minute']) ? $options['api_calls_to_leave_unmade_per_minute']->option_value : 2.0;
if ($noauth) {
$api = new CrawlerTwitterAPIAccessorOAuth('NOAUTH', 'NOAUTH', $options['oauth_consumer_key']->option_value, $options['oauth_consumer_secret']->option_value, $api_calls_to_leave_unmade_per_minute, $options['archive_limit']->option_value, $num_twitter_errors, $max_api_calls_per_crawl);
} else {
$api = new CrawlerTwitterAPIAccessorOAuth($tokens['oauth_access_token'], $tokens['oauth_access_token_secret'], $options['oauth_consumer_key']->option_value, $options['oauth_consumer_secret']->option_value, $api_calls_to_leave_unmade_per_minute, $options['archive_limit']->option_value, $num_twitter_errors, $max_api_calls_per_crawl);
}
$crawler = new TwitterCrawler($instance, $api);
$insights_generator = new InsightsGenerator($instance);
$api->init();
// budget our twitter calls
$call_limits = $this->budgetCrawlLimits($api->available_api_calls_for_crawler, $noauth);
$api->setCallerLimits($call_limits);
if ($api->available_api_calls_for_crawler > 0) {
$instance_dao->updateLastRun($instance->id);
// No auth for public Twitter users
$crawler->fetchInstanceUserTweets();
if (!$noauth) {
// Auth req'd, for calling user only
$crawler->fetchInstanceUserMentions();
$crawler->fetchInstanceUserFriends();
$crawler->fetchInstanceFavorites();
$crawler->fetchInstanceUserFollowers();
$crawler->fetchInstanceUserGroups();
$crawler->fetchRetweetsOfInstanceUser();
$crawler->cleanUpMissedFavsUnFavs();
$crawler->updateStaleGroupMemberships();
}
$crawler->fetchStrayRepliedToTweets();
$crawler->fetchUnloadedFollowerDetails();
$crawler->cleanUpFollows();
$crawler->fetchFriendTweetsAndFriends();
$insights_generator->generateInsights();
if ($noauth) {
// No auth req'd
$crawler->fetchSearchResults($instance->network_username);
}
// Save instance
if (isset($crawler->user)) {
$instance_dao->save($instance, $crawler->user->post_count, $logger);
}
$logger->logUserSuccess("Finished collecting data for " . $instance->network_username . " on Twitter.", __METHOD__ . ',' . __LINE__);
}
}
}
public function testGetClickStatsVisualizationData()
{
$click_stats = array(array('post_text' => 'Black Mirror punched me in the gut this weekend. Highly recommended. http://t.co/AnczD4Jc ' . 'Thx @annaleen & @fraying', 'click_count' => 50), array('post_text' => '@saenz a geeky uncle's only <span class="googid">+Sprint</span> http://t.co/cxZTmWhk', 'click_count' => 150), array('post_text' => 'I\'ll admit Glee made me cry last night. Then it made me cringe. http://t.co/lgjaJWcW ', 'click_count' => 23));
$result = InsightsGenerator::getClickStatsVisualizationData($click_stats);
$this->assertEqual(gettype($result), 'string');
$visualization_object = json_decode($result);
$this->assertEqual(sizeof($visualization_object->rows), 3);
$this->assertEqual(sizeof($visualization_object->cols), 2);
$this->assertEqual($visualization_object->cols[0]->label, 'Link');
$this->assertEqual($visualization_object->cols[1]->label, 'Clicks');
$this->assertEqual($visualization_object->rows[0]->c[0]->v, 'Black Mirror punched me in the gut this weekend. Highly recommended. http://t.co/AnczD4Jc Thx @annal...');
$this->assertEqual($visualization_object->rows[0]->c[1]->v, 50);
$this->assertEqual($visualization_object->rows[1]->c[0]->v, "@saenz a geeky uncle's only +Sprint http://t.co/cxZTmWhk...");
$this->assertEqual($visualization_object->rows[1]->c[1]->v, 150);
$this->assertEqual($visualization_object->rows[2]->c[0]->v, 'I\'ll admit Glee made me cry last night. Then it made me cringe. http://t.co/lgjaJWcW ...');
$this->assertEqual($visualization_object->rows[2]->c[1]->v, 23);
}