public function crawl()
{
$logger = Logger::getInstance();
$config = Config::getInstance();
$instance_dao = DAOFactory::getDAO('InstanceDAO');
$owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
$owner_dao = DAOFactory::getDAO('OwnerDAO');
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('youtube', true);
//get cached
$max_crawl_time = isset($options['max_crawl_time']) ? $options['max_crawl_time']->option_value : 20;
//convert to seconds
$max_crawl_time = $max_crawl_time * 60;
$developer_key = isset($options['developer_key']) ? $options['developer_key']->option_value : null;
$max_comments = isset($options['max_comments']) ? $options['max_comments']->option_value : null;
$current_owner = $owner_dao->getByEmail(Session::getLoggedInUser());
//crawl youtube users
$instances = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'youtube');
if (isset($options['youtube_client_id']->option_value) && isset($options['youtube_client_secret']->option_value)) {
foreach ($instances as $instance) {
$logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username);
$logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
$tokens = $owner_instance_dao->getOAuthTokens($instance->id);
$access_token = $tokens['oauth_access_token'];
$refresh_token = $tokens['oauth_access_token_secret'];
$instance_dao->updateLastRun($instance->id);
$youtube_crawler = new YouTubeCrawler($instance, $access_token, $max_crawl_time, $developer_key, $max_comments);
$dashboard_module_cacher = new DashboardModuleCacher($instance);
try {
$youtube_crawler->initializeInstanceUser($options['youtube_client_id']->option_value, $options['youtube_client_secret']->option_value, $access_token, $refresh_token, $current_owner->id);
$youtube_crawler->fetchInstanceUserVideos();
} catch (Exception $e) {
$logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__);
}
$dashboard_module_cacher->cacheDashboardModules();
$instance_dao->save($youtube_crawler->instance, 0, $logger);
Reporter::reportVersion($instance);
$logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
}
}
}
/**
* Add user auth link or process incoming auth requests.
* @param array $options Plugin options array
*/
protected function setUpYouTubeInteractions(array $options)
{
//get options
$client_id = $options['youtube_client_id']->option_value;
$client_secret = $options['youtube_client_secret']->option_value;
//prep redirect URI
$config = Config::getInstance();
$site_root_path = $config->getValue('site_root_path');
$redirect_uri = urlencode(Utils::getApplicationURL() . 'account/?p=youtube');
//create OAuth link
$oauth_link = "https://accounts.google.com/o/oauth2/auth?client_id=" . $client_id . "&redirect_uri=" . $redirect_uri . "&scope=https://www.googleapis.com/auth/youtube.readonly%20https://www.googleapis.com/auth/plus.me" . "%20https://www.googleapis.com/auth/yt-analytics.readonly&response_type=code&access_type=offline" . "&approval_prompt=force";
$this->addToView('oauth_link', $oauth_link);
// Google provided a code to get an access token
if (isset($_GET['code'])) {
$code = $_GET['code'];
$crawler_plugin_registrar = new YouTubeCrawler(null, null, null, null, null);
$tokens = $crawler_plugin_registrar->getOAuthTokens($client_id, $client_secret, $code, 'authorization_code', $redirect_uri);
if (isset($tokens->error)) {
$this->addErrorMessage("Oops! Something went wrong while obtaining OAuth tokens.<br>YouTube says \"" . $tokens->error . ".\" Please double-check your settings and try again.", 'authorization');
} else {
if (isset($tokens->access_token)) {
// Get user data
// First we need to query the YouTube API for the users G+ ID
$youtube_api_accessor = new YouTubeAPIAccessor();
$fields = array("part" => "contentDetails", "mine" => "true");
$gplus_user_id_query = $youtube_api_accessor->apiRequest('channels', $tokens->access_token, $fields);
// The error we could get from this call is a forbidden error if something went wrong with
// authentication.
if (isset($gplus_user_id_query->error)) {
if ($gplus_user_id_query->error->code == "401" && $gplus_user_id_query->error->message == 'Unauthorized') {
$this->addErrorMessage("Oops! Looks like YouTube API access isn't turned on. " . "<a href=\"http://code.google.com/apis/console#access\">In the Google APIs console</a>, " . "in Services, flip the YouTube and YouTube analytics API Status switch to 'On' and try again\n .", 'authorization');
} else {
$this->addErrorMessage("Oops! Something went wrong querying the YouTube API.<br>" . "Google says \"" . $gplus_user_id_query->error->code . ": " . $gplus_user_id_query->error->message . ".\" Please double-check your settings and try again.", 'authorization');
}
} else {
// We have should have the users G+ id so we now just need their username from the G+ API
$gplus_id = $gplus_user_id_query->items[0]->contentDetails->googlePlusUserId;
$gplus_api_accessor = new GooglePlusAPIAccessor();
if (isset($gplus_id)) {
$gplus_user = $gplus_api_accessor->apiRequest('people/' . $gplus_id, $tokens->access_token, null);
if (isset($gplus_user->error)) {
if ($gplus_user->error->code == "403" && $gplus_user->error->message == 'Access Not Configured') {
$this->addErrorMessage("Oops! Looks like Google+ API access isn't turned on. " . "<a href=\"http://code.google.com/apis/console#access\">In the Google APIs " . "console</a> in Services, flip the Google+ API Status switch to 'On' and " . "try again.", 'authorization');
} else {
$this->addErrorMessage("Oops! Something went wrong querying the Google+ API.<br>" . "Google says \"" . $gplus_user->error->code . ": " . $gplus_user->error->message . ".\" Please double-check your settings and try again.", 'authorization');
}
} else {
if (isset($gplus_user->id) && isset($gplus_user->displayName)) {
$gplus_user_id = $gplus_user->id;
$gplus_username = $gplus_user->displayName;
//Process tokens
$this->saveAccessTokens($gplus_user_id, $gplus_username, $tokens->access_token, $tokens->refresh_token);
} else {
$this->addErrorMessage("Oops! Something went wrong querying the Google+ API.<br>" . "Google says \"" . Utils::varDumpToString($gplus_user) . ".\" Please double-check your settings and try again.", 'authorization');
}
}
} else {
// It may be possible that the user has not linked their YouTube account to their G+ account
// so we might not get a G+ ID
$this->addErrorMessage("You don't have a Google+ ID associated with your YouTube account, " . "go to YouTube and link your Google+ account to your YouTube account to use this plugin. " . "For more information click <a href=https://www.thinkup.com/docs/userguide/settings/plugin" . "s/youtube.html>here</a>", 'authorization');
}
}
}
}
}
$instance_dao = DAOFactory::getDAO('InstanceDAO');
$owner_instances = $instance_dao->getByOwnerAndNetwork($this->owner, 'youtube');
$this->addToView('owner_instances', $owner_instances);
}
public function testFetchInstanceUserVideos()
{
$builders = self::buildData();
$ytc = new YouTubeCrawler($this->instance, 'at', 20);
$ytc->fetchInstanceUserVideos();
$video_dao = new VideoMySQLDAO();
$post_dao = new PostMySQLDAO();
$count_history_dao = new CountHistoryMySQLDAO();
// Check the first video got added
$video_one = $video_dao->getVideoByID('H', 'youtube');
$this->assertEqual($video_one->post_text, 'My Slideshow');
$this->assertEqual($video_one->description, 'I created this video with the YouTube Slideshow Creator (http://www.youtube.com/upload)');
$this->assertEqual($video_one->likes, 1);
$this->assertEqual($video_one->dislikes, 0);
$this->assertEqual($video_one->views, 6);
$this->assertEqual($video_one->post_id, 'H');
$this->assertEqual($video_one->is_protected, 0);
$this->assertEqual($video_one->favorites_added, 0);
$this->assertEqual($video_one->favorites_removed, 0);
$this->assertEqual($video_one->shares, 0);
$this->assertEqual($video_one->subscribers_gained, 0);
$this->assertEqual($video_one->subscribers_lost, 0);
$this->assertEqual($video_one->minutes_watched, 4);
$this->assertEqual($video_one->average_view_duration, 52);
$this->assertEqual($video_one->average_view_percentage, 24.8284);
$this->assertEqual($video_one->author_user_id, '113612142759476883204');
$this->assertEqual($video_one->author_username, 'Gina Trapani');
$this->assertEqual($video_one->author_fullname, 'Gina Trapani');
$this->assertEqual($video_one->author_avatar, 'http://www.myavatar.com');
$this->assertEqual($video_one->source, '');
$this->assertEqual($video_one->network, 'youtube');
// Check the all time counts were added correctly
$favs_added_all_time = $count_history_dao->getLatestCountByPostIDAndType('H', 'favorites_added_all_time');
$favs_removed_all_time = $count_history_dao->getLatestCountByPostIDAndType('H', 'favorites_removed_all_time');
$shares_all_time = $count_history_dao->getLatestCountByPostIDAndType('H', 'shares_all_time');
$subs_gained_all_time = $count_history_dao->getLatestCountByPostIDAndType('H', 'subscribers_gained_all_time');
$subs_lost_all_time = $count_history_dao->getLatestCountByPostIDAndType('H', 'subscribers_lost_all_time');
$mins_watched_all_time = $count_history_dao->getLatestCountByPostIDAndType('H', 'minutes_watched_all_time');
$avg_view_percent_all_time = $count_history_dao->getLatestCountByPostIDAndType('H', 'average_view_percentage_all_time');
$avg_view_dur_all_time = $count_history_dao->getLatestCountByPostIDAndType('H', 'average_view_duration_all_time');
$views_all_time = $count_history_dao->getLatestCountByPostIDAndType('H', 'views_all_time');
$likes_all_time = $count_history_dao->getLatestCountByPostIDAndType('H', 'likes_all_time');
$dislikes_all_time = $count_history_dao->getLatestCountByPostIDAndType('H', 'dislikes_all_time');
$this->assertEqual($favs_added_all_time['count'], 0);
$this->assertEqual($favs_removed_all_time['count'], 0);
$this->assertEqual($shares_all_time['count'], 0);
$this->assertEqual($subs_gained_all_time['count'], 0);
$this->assertEqual($subs_lost_all_time['count'], 0);
$this->assertEqual($mins_watched_all_time['count'], 4);
$this->assertEqual($avg_view_percent_all_time['count'], 25);
$this->assertEqual($avg_view_dur_all_time['count'], 52);
$this->assertEqual($views_all_time['count'], 6);
$this->assertEqual($likes_all_time['count'], 1);
$this->assertEqual($dislikes_all_time['count'], 0);
// Check the replies to it got added
// Basic 1st reply
$reply_one = $post_dao->getPost('jm_SGXNfF6AmF20tsHqF_2h_S_fV_0l2DU3AfqjbsNc', 'youtube');
$this->assertEqual($reply_one->post_id, 'jm_SGXNfF6AmF20tsHqF_2h_S_fV_0l2DU3AfqjbsNc');
$this->assertEqual($reply_one->post_text, 'Test comment');
$this->assertEqual($reply_one->author_username, 'Aaron Kalair');
$this->assertEqual($reply_one->author_fullname, 'Aaron Kalair');
$this->assertEqual($reply_one->author_avatar, 'https://lh5.googleusercontent.com/-Z2vFxu2wO6E/AAAAAAAAAAI/AAAAAAAAANQ/Pp0EB7dNKLY/photo.jpg?sz=50');
$this->assertEqual($reply_one->author_user_id, '115383827382290096528');
$this->assertEqual($reply_one->pub_date, '2013-06-05 08:13:43');
$this->assertEqual($reply_one->source, '');
$this->assertEqual($reply_one->is_protected, 0);
$this->assertEqual($reply_one->network, 'youtube');
$this->assertEqual($reply_one->in_reply_to_user_id, '113612142759476883204');
$this->assertEqual($reply_one->in_reply_to_post_id, 'H');
// Check we can itterate over replies in a single page
$reply_two = $post_dao->getPost('gm_SGXNffMJT58F20tsHqF_2h_S_fV_0l2DU3AfqjbsNc', 'youtube');
$this->assertEqual($reply_two->post_id, 'gm_SGXNffMJT58F20tsHqF_2h_S_fV_0l2DU3AfqjbsNc');
$this->assertEqual($reply_two->post_text, 'yet another comment');
$this->assertEqual($reply_two->author_username, 'Aaron Kalair');
$this->assertEqual($reply_two->author_fullname, 'Aaron Kalair');
$this->assertEqual($reply_two->author_avatar, 'https://lh5.googleusercontent.com/-Z2vFxu2wO6E/AAAAAAAAAAI/AAAAAAAAANQ/Pp0EB7dNKLY/photo.jpg?sz=50');
$this->assertEqual($reply_two->author_user_id, '115383827382290096528');
$this->assertEqual($reply_two->pub_date, '2013-06-15 07:45:11');
$this->assertEqual($reply_two->source, '');
$this->assertEqual($reply_two->is_protected, 0);
$this->assertEqual($reply_two->network, 'youtube');
$this->assertEqual($reply_two->in_reply_to_user_id, '113612142759476883204');
$this->assertEqual($reply_two->in_reply_to_post_id, 'H');
// Check we can get a reply from another page
$reply_three = $post_dao->getPost('hg_HF75HJSNY38JH_ht5_fh', 'youtube');
$this->assertEqual($reply_three->post_id, 'hg_HF75HJSNY38JH_ht5_fh');
$this->assertEqual($reply_three->post_text, 'A comment on the second page');
$this->assertEqual($reply_three->author_username, 'Aaron Kalair');
$this->assertEqual($reply_three->author_fullname, 'Aaron Kalair');
$this->assertEqual($reply_three->author_avatar, 'https://lh5.googleusercontent.com/-Z2vFxu2wO6E/AAAAAAAAAAI/AAAAAAAAANQ/Pp0EB7dNKLY/photo.jpg?sz=50');
$this->assertEqual($reply_three->author_user_id, '115383827382290096528');
$this->assertEqual($reply_three->pub_date, '2013-06-10 18:14:33');
$this->assertEqual($reply_three->source, '');
$this->assertEqual($reply_three->is_protected, 0);
$this->assertEqual($reply_three->network, 'youtube');
$this->assertEqual($reply_three->in_reply_to_user_id, '113612142759476883204');
$this->assertEqual($reply_three->in_reply_to_post_id, 'H');
// Check we can itterate over videos on the same page of replies
$video_two = $video_dao->getVideoByID('a', 'youtube');
$this->assertEqual($video_two->post_text, 'Same Page Video');
$this->assertEqual($video_two->description, 'This video is on the same page');
$this->assertEqual($video_two->likes, 10);
$this->assertEqual($video_two->dislikes, 110);
$this->assertEqual($video_two->views, 50);
$this->assertEqual($video_two->post_id, 'a');
$this->assertEqual($video_two->is_protected, 0);
$this->assertEqual($video_two->favorites_added, 5);
$this->assertEqual($video_two->favorites_removed, 1);
$this->assertEqual($video_two->shares, 0);
$this->assertEqual($video_two->subscribers_gained, 0);
$this->assertEqual($video_two->subscribers_lost, 0);
$this->assertEqual($video_two->minutes_watched, 1);
$this->assertEqual($video_two->average_view_duration, 2);
$this->assertEqual($video_two->average_view_percentage, 24.8284);
$this->assertEqual($video_two->author_user_id, '113612142759476883204');
$this->assertEqual($video_two->author_username, 'Gina Trapani');
$this->assertEqual($video_two->author_fullname, 'Gina Trapani');
$this->assertEqual($video_two->author_avatar, 'http://www.myavatar.com');
$this->assertEqual($video_two->source, '');
$this->assertEqual($video_two->network, 'youtube');
// Check the per day counts got added correctly
$favs_added = $count_history_dao->getLatestCountByPostIDAndType('a', 'favorites_added');
$favs_removed = $count_history_dao->getLatestCountByPostIDAndType('a', 'favorites_removed');
$shares = $count_history_dao->getLatestCountByPostIDAndType('a', 'shares');
$subs_gained = $count_history_dao->getLatestCountByPostIDAndType('a', 'subscribers_gained');
$subs_lost = $count_history_dao->getLatestCountByPostIDAndType('a', 'subscribers_lost');
$mins_watched = $count_history_dao->getLatestCountByPostIDAndType('a', 'minutes_watched');
$avg_view_percent = $count_history_dao->getLatestCountByPostIDAndType('a', 'average_view_percentage');
$avg_view_dur = $count_history_dao->getLatestCountByPostIDAndType('a', 'average_view_duration');
$views = $count_history_dao->getLatestCountByPostIDAndType('a', 'views');
$likes = $count_history_dao->getLatestCountByPostIDAndType('a', 'likes');
$dislikes = $count_history_dao->getLatestCountByPostIDAndType('a', 'dislikes');
$this->assertEqual($favs_added['count'], 4);
$this->assertEqual($favs_removed['count'], 2);
$this->assertEqual($shares['count'], 3);
$this->assertEqual($subs_gained['count'], 10);
$this->assertEqual($subs_lost['count'], 11);
$this->assertEqual($mins_watched['count'], 11);
$this->assertEqual($avg_view_percent['count'], 14);
$this->assertEqual($avg_view_dur['count'], 15);
$this->assertEqual($views['count'], 100);
$this->assertEqual($likes['count'], 0);
$this->assertEqual($dislikes['count'], 0);
// Check the all time counts were added correctly
$favs_added_all_time = $count_history_dao->getLatestCountByPostIDAndType('a', 'favorites_added_all_time');
$favs_removed_all_time = $count_history_dao->getLatestCountByPostIDAndType('a', 'favorites_removed_all_time');
$shares_all_time = $count_history_dao->getLatestCountByPostIDAndType('a', 'shares_all_time');
$subs_gained_all_time = $count_history_dao->getLatestCountByPostIDAndType('a', 'subscribers_gained_all_time');
$subs_lost_all_time = $count_history_dao->getLatestCountByPostIDAndType('a', 'subscribers_lost_all_time');
$mins_watched_all_time = $count_history_dao->getLatestCountByPostIDAndType('a', 'minutes_watched_all_time');
$avg_view_percent_all_time = $count_history_dao->getLatestCountByPostIDAndType('a', 'average_view_percentage_all_time');
$avg_view_dur_all_time = $count_history_dao->getLatestCountByPostIDAndType('a', 'average_view_duration_all_time');
$views_all_time = $count_history_dao->getLatestCountByPostIDAndType('a', 'views_all_time');
$likes_all_time = $count_history_dao->getLatestCountByPostIDAndType('a', 'likes_all_time');
$dislikes_all_time = $count_history_dao->getLatestCountByPostIDAndType('a', 'dislikes_all_time');
$this->assertEqual($favs_added_all_time['count'], 5);
$this->assertEqual($favs_removed_all_time['count'], 1);
$this->assertEqual($shares_all_time['count'], 0);
$this->assertEqual($subs_gained_all_time['count'], 0);
$this->assertEqual($subs_lost_all_time['count'], 0);
$this->assertEqual($mins_watched_all_time['count'], 1);
$this->assertEqual($avg_view_percent_all_time['count'], 25);
$this->assertEqual($avg_view_dur_all_time['count'], 2);
$this->assertEqual($views_all_time['count'], 50);
$this->assertEqual($likes_all_time['count'], 10);
$this->assertEqual($dislikes_all_time['count'], 110);
// Check we can get a video from the next page of videos
$video_three = $video_dao->getVideoByID('g', 'youtube');
$this->assertEqual($video_three->post_text, 'My Slideshow');
$this->assertEqual($video_three->description, 'I created this video with the YouTube Slideshow Creator (http://www.youtube.com/upload)');
$this->assertEqual($video_three->likes, 0);
$this->assertEqual($video_three->dislikes, 0);
$this->assertEqual($video_three->views, 5);
$this->assertEqual($video_three->post_id, 'g');
$this->assertEqual($video_three->is_protected, 0);
$this->assertEqual($video_three->favorites_added, 0);
$this->assertEqual($video_three->favorites_removed, 0);
$this->assertEqual($video_three->shares, 0);
$this->assertEqual($video_three->subscribers_gained, 0);
$this->assertEqual($video_three->subscribers_lost, 0);
$this->assertEqual($video_three->minutes_watched, 0);
$this->assertEqual($video_three->average_view_duration, 15);
$this->assertEqual($video_three->average_view_percentage, 42.7689);
$this->assertEqual($video_three->author_user_id, '113612142759476883204');
$this->assertEqual($video_three->author_username, 'Gina Trapani');
$this->assertEqual($video_three->author_fullname, 'Gina Trapani');
$this->assertEqual($video_three->author_avatar, 'http://www.myavatar.com');
$this->assertEqual($video_three->source, '');
$this->assertEqual($video_three->network, 'youtube');
// Check the all time counts were added correctly
$favs_added_all_time = $count_history_dao->getLatestCountByPostIDAndType('g', 'favorites_added_all_time');
$favs_removed_all_time = $count_history_dao->getLatestCountByPostIDAndType('g', 'favorites_removed_all_time');
$shares_all_time = $count_history_dao->getLatestCountByPostIDAndType('g', 'shares_all_time');
$subs_gained_all_time = $count_history_dao->getLatestCountByPostIDAndType('g', 'subscribers_gained_all_time');
$subs_lost_all_time = $count_history_dao->getLatestCountByPostIDAndType('g', 'subscribers_lost_all_time');
$mins_watched_all_time = $count_history_dao->getLatestCountByPostIDAndType('g', 'minutes_watched_all_time');
$avg_view_percent_all_time = $count_history_dao->getLatestCountByPostIDAndType('g', 'average_view_percentage_all_time');
$avg_view_dur_all_time = $count_history_dao->getLatestCountByPostIDAndType('g', 'average_view_duration_all_time');
$views_all_time = $count_history_dao->getLatestCountByPostIDAndType('g', 'views_all_time');
$likes_all_time = $count_history_dao->getLatestCountByPostIDAndType('g', 'likes_all_time');
$dislikes_all_time = $count_history_dao->getLatestCountByPostIDAndType('g', 'dislikes_all_time');
$this->assertEqual($favs_added_all_time['count'], 0);
$this->assertEqual($favs_removed_all_time['count'], 0);
$this->assertEqual($shares_all_time['count'], 0);
$this->assertEqual($subs_gained_all_time['count'], 0);
$this->assertEqual($subs_lost_all_time['count'], 0);
$this->assertEqual($mins_watched_all_time['count'], 0);
$this->assertEqual($avg_view_percent_all_time['count'], 43);
$this->assertEqual($avg_view_dur_all_time['count'], 15);
$this->assertEqual($views_all_time['count'], 5);
$this->assertEqual($likes_all_time['count'], 0);
$this->assertEqual($dislikes_all_time['count'], 0);
}
