/** * Gets called when crawler runs. * * About crawler exclusivity (mutex usage): * When launched by an admin, no other user, admin or not, will be able to launch a crawl until this one is done. * When launched by a non-admin, we first check that no admin run is under way, and if that's the case, * we launch a crawl for the current user only. * No user will be able to launch two crawls in parallel, but different non-admin users crawls can run in parallel. */ public function crawl() { if (!Session::isLoggedIn()) { throw new UnauthorizedUserException('You need a valid session to launch the crawler.'); } $mutex_dao = DAOFactory::getDAO('MutexDAO'); $owner_dao = DAOFactory::getDAO('OwnerDAO'); $owner = $owner_dao->getByEmail(Session::getLoggedInUser()); if (empty($owner)) { throw new UnauthorizedUserException('You need a valid session to launch the crawler.'); } $global_mutex_name = 'crawler'; // Everyone needs to check the global mutex $lock_successful = $mutex_dao->getMutex($global_mutex_name); if ($lock_successful) { // Global mutex was free, which means no admin crawls are under way if ($owner->is_admin) { // Nothing more needs to be done, since admins use the global mutex $mutex_name = $global_mutex_name; } else { // User is a non-admin; let's use a user mutex. $mutex_name = 'crawler-' . $owner->id; $lock_successful = $mutex_dao->getMutex($mutex_name); $mutex_dao->releaseMutex($global_mutex_name); } } if ($lock_successful) { $this->emitObjectMethod('crawl'); $mutex_dao->releaseMutex($mutex_name); } else { throw new CrawlerLockedException("Error starting crawler; another crawl is already in progress."); } }
public function testOutput() { //not logged in $controller = new TwitterPluginHashtagConfigurationController(null, 'twitter', 'ginatrapani'); $output = $controller->go(); $v_mgr = $controller->getViewManager(); $config = Config::getInstance(); $this->assertEqual('You must <a href="' . $config->getValue('site_root_path') . 'session/login.php">log in</a> to do this.', $v_mgr->getTemplateDataItem('error_msg')); //logged in, no user set up $this->simulateLogin('*****@*****.**'); $owner_dao = DAOFactory::getDAO('OwnerDAO'); $owner = $owner_dao->getByEmail(Session::getLoggedInUser()); $controller = new TwitterPluginHashtagConfigurationController(null, 'twitter', 'ginatrapani'); $output = $controller->go(); $v_mgr = $controller->getViewManager(); $this->assertIsA($v_mgr->getTemplateDataItem('user'), 'string'); $this->assertEqual('Twitter user @ginatrapani does not exist.', $v_mgr->getTemplateDataItem('error_msg')); //logged in, user set up $builders = array(); $builders[] = FixtureBuilder::build('instances', array('network_username' => 'ginatrapani', 'network' => 'twitter')); $controller = new TwitterPluginHashtagConfigurationController(null, 'twitter', 'ginatrapani'); $output = $controller->go(); $v_mgr = $controller->getViewManager(); $this->assertIsA($v_mgr->getTemplateDataItem('user'), 'string'); $this->assertIsA($v_mgr->getTemplateDataItem('hashtags'), 'array'); }
public function crawl() { $logger = Logger::getInstance(); $config = Config::getInstance(); $instance_dao = DAOFactory::getDAO('InstanceDAO'); $owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO'); $owner_dao = DAOFactory::getDAO('OwnerDAO'); $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('googleplus', true); //get cached $current_owner = $owner_dao->getByEmail(Session::getLoggedInUser()); //crawl Google+ users $instances = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'google+'); if (isset($options['google_plus_client_id']->option_value) && isset($options['google_plus_client_secret']->option_value)) { foreach ($instances as $instance) { $logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username); $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__); $tokens = $owner_instance_dao->getOAuthTokens($instance->id); $access_token = $tokens['oauth_access_token']; $refresh_token = $tokens['oauth_access_token_secret']; $instance_dao->updateLastRun($instance->id); $google_plus_crawler = new GooglePlusCrawler($instance, $access_token); $dashboard_module_cacher = new DashboardModuleCacher($instance); try { $google_plus_crawler->initializeInstanceUser($options['google_plus_client_id']->option_value, $options['google_plus_client_secret']->option_value, $access_token, $refresh_token, $current_owner->id); $google_plus_crawler->fetchInstanceUserPosts(); } catch (Exception $e) { $logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__); } $dashboard_module_cacher->cacheDashboardModules(); $instance_dao->save($google_plus_crawler->instance, 0, $logger); $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__); } } }
public function go() { if (isset($_GET['id'])) { $id = $_GET['id']; } if (isset($_GET['source']) && $_GET['source'] == "new") { $this->addSuccessMessage("Article has been added succesfully"); } if (isset($_POST['submit'])) { if ($_POST['title'] == '') { $this->addErrorMessage("Title of the article should not be empty"); } elseif (!isset($_POST['is_published'])) { $this->addErrorMessage("Please tell if the article has been published successfully?"); } elseif ($_POST['content'] == '') { $this->addErrorMessage("Article post should not be empty"); } else { $this->title = $_POST['title']; $this->is_published = $_POST['is_published']; $this->content = $_POST['content']; $this->last_modified = date("Y-m-d H-i-s"); $this->last_modified_by = Session::getLoggedInUser(); ArticleBackend::updateArticle($id, $this->title, $this->content, $this->last_modified, $this->last_modified_by); $this->addSuccessMessage("Article has been updated succesfully"); } } $article = ArticleBackend::getArticle($id); $this->setViewTemplate('editarticle.tpl'); $this->addToView('article', $article[0]); $this->generateView(); if (isset($_POST['deletesubmit'])) { ArticleBackend::deleteArticle($id); header('Location:' . SOURCE_ROOT_PATH . "admin/pages/articlemanager.php?source=del"); } }
public function crawl() { $config = Config::getInstance(); $logger = Logger::getInstance(); $instance_dao = DAOFactory::getDAO('TwitterInstanceDAO'); $owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO'); $owner_dao = DAOFactory::getDAO('OwnerDAO'); $instance_hashtag_dao = DAOFactory::getDAO('InstanceHashtagDAO'); // get oauth values $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('twitter', true); $current_owner = $owner_dao->getByEmail(Session::getLoggedInUser()); $instances = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'twitter'); foreach ($instances as $instance) { $logger->setUsername($instance->network_username); $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . " on Twitter.", __METHOD__ . ',' . __LINE__); $tokens = $owner_instance_dao->getOAuthTokens($instance->id); $num_twitter_errors = isset($options['num_twitter_errors']) ? $options['num_twitter_errors']->option_value : null; $dashboard_module_cacher = new DashboardModuleCacher($instance); try { if (isset($tokens['oauth_access_token']) && $tokens['oauth_access_token'] != '' && isset($tokens['oauth_access_token_secret']) && $tokens['oauth_access_token_secret'] != '') { $archive_limit = isset($options['archive_limit']->option_value) ? $options['archive_limit']->option_value : 3200; $api = new CrawlerTwitterAPIAccessorOAuth($tokens['oauth_access_token'], $tokens['oauth_access_token_secret'], $options['oauth_consumer_key']->option_value, $options['oauth_consumer_secret']->option_value, $archive_limit, $num_twitter_errors); $twitter_crawler = new TwitterCrawler($instance, $api); $instance_dao->updateLastRun($instance->id); $twitter_crawler->fetchInstanceUserTweets(); $twitter_crawler->fetchInstanceUserMentions(); $twitter_crawler->fetchInstanceUserFriends(); $twitter_crawler->fetchInstanceUserFollowers(); $twitter_crawler->fetchInstanceUserGroups(); $twitter_crawler->fetchRetweetsOfInstanceUser(); $twitter_crawler->fetchInstanceUserFavorites(); $twitter_crawler->updateStaleGroupMemberships(); $twitter_crawler->fetchStrayRepliedToTweets(); $twitter_crawler->fetchUserFriendsByIDs(); $twitter_crawler->fetchUnloadedFriendDetails(); $twitter_crawler->fetchUnloadedFollowerDetails(); $twitter_crawler->cleanUpFollows(); $twitter_crawler->updateFriendsProfiles(); //Retrieve search results for saved keyword/hashtags $instances_hashtags = $instance_hashtag_dao->getByInstance($instance->id); foreach ($instances_hashtags as $instance_hashtag) { $twitter_crawler->fetchInstanceHashtagTweets($instance_hashtag); } } else { throw new Exception('Missing Twitter OAuth tokens.'); } } catch (Exception $e) { $logger->logUserError(get_class($e) . " while crawling " . $instance->network_username . " on Twitter: " . $e->getMessage(), __METHOD__ . ',' . __LINE__); } $dashboard_module_cacher->cacheDashboardModules(); // Save instance if (isset($twitter_crawler->user)) { $instance_dao->save($instance, $twitter_crawler->user->post_count, $logger); } Reporter::reportVersion($instance); $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . " on Twitter.", __METHOD__ . ',' . __LINE__); } }
public function crawl() { $logger = Logger::getInstance(); $config = Config::getInstance(); $instance_dao = DAOFactory::getDAO('InstagramInstanceDAO'); $owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO'); $owner_dao = DAOFactory::getDAO('OwnerDAO'); $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('instagram', true); //get cached $max_api_calls = isset($options['max_api_calls']) ? $options['max_api_calls']->option_value : 2500; $current_owner = $owner_dao->getByEmail(Session::getLoggedInUser()); //crawl instagram user profiles and pages $instances = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'instagram'); foreach ($instances as $instance) { $logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username); $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__); $tokens = $owner_instance_dao->getOAuthTokens($instance->id); $access_token = $tokens['oauth_access_token']; $instance_dao->updateLastRun($instance->id); $dashboard_module_cacher = new DashboardModuleCacher($instance); try { /** * 1. Fetch user info, media + its likes and comments. * 2. Fetch user's likes. * 3. Fetch user's friends, and update stale relationships. * 4. Fetch user's followers, and update stale relationships. * 5. Update stale friends' profiles. */ $instagram_crawler = new InstagramCrawler($instance, $access_token, $max_api_calls); $instagram_crawler->fetchPostsAndReplies(); $instagram_crawler->fetchLikes(); $instagram_crawler->fetchFriends(); $instagram_crawler->fetchFollowers(); $instagram_crawler->updateStaleFriendsProfiles(); } catch (Instagram\Core\ApiAuthException $e) { //The access token is invalid, save in owner_instances table $owner_instance_dao->setAuthErrorByTokens($instance->id, $access_token, '', $e->getMessage()); //Send email alert //Get owner by auth tokens first, then send to that person $owner_email_to_notify = $owner_instance_dao->getOwnerEmailByInstanceTokens($instance->id, $access_token, ''); $email_attempt = $this->sendInvalidOAuthEmailAlert($owner_email_to_notify, $instance->network_username); if ($email_attempt) { $logger->logUserInfo('Sent reauth email to ' . $owner_email_to_notify, __METHOD__ . ',' . __LINE__); } else { $logger->logInfo('Didn\'t send reauth email to ' . $owner_email_to_notify, __METHOD__ . ',' . __LINE__); } $logger->logUserError(get_class($e) . ' ' . $e->getMessage(), __METHOD__ . ',' . __LINE__); } catch (Exception $e) { $logger->logUserError(get_class($e) . ' ' . $e->getMessage(), __METHOD__ . ',' . __LINE__); } $dashboard_module_cacher->cacheDashboardModules(); $instance_dao->save($instagram_crawler->instance, 0, $logger); Reporter::reportVersion($instance); $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__); } }
public function crawl() { $logger = Logger::getInstance(); $config = Config::getInstance(); $instance_dao = DAOFactory::getDAO('InstanceDAO'); $owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO'); $owner_dao = DAOFactory::getDAO('OwnerDAO'); $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('foursquare', true); $current_owner = $owner_dao->getByEmail(Session::getLoggedInUser()); $instances = $instance_dao->getAllActiveInstancesStalestFirstByNetwork('foursquare'); // Check the client id and secret are set or we can't crawl if (isset($options['foursquare_client_id']->option_value) && isset($options['foursquare_client_secret']->option_value)) { // For each instance of foursquare on this install foreach ($instances as $instance) { if (!$owner_instance_dao->doesOwnerHaveAccessToInstance($current_owner, $instance)) { // Owner doesn't have access to this instance; let's not crawl it. continue; } // Set the user name in the log $logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username); // Write to the log that we have started to collect data $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__); // Get the OAuth tokens for this user $tokens = $owner_instance_dao->getOAuthTokens($instance->id); // Set the access token $access_token = $tokens['oauth_access_token']; // Update the last time we crawled $instance_dao->updateLastRun($instance->id); // Create a new crawler $crawler = new FoursquareCrawler($instance, $access_token); // Check the OAuth tokens we have are valid try { $logger->logInfo("About to initializeInstanceUser", __METHOD__ . ',' . __LINE__); $user = $crawler->initializeInstanceUser($access_token, $current_owner->id); if (isset($user) && $user instanceof User) { $logger->logInfo("User initialized", __METHOD__ . ',' . __LINE__); } // Get the data we want and store it in the database $logger->logInfo("About to fetchInstanceUserCheckins", __METHOD__ . ',' . __LINE__); $crawler->fetchInstanceUserCheckins(); } catch (Exception $e) { // Catch any errors that happen when we check the validity of the OAuth tokens $logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__); } $logger->logInfo("About to cache dashboard modules", __METHOD__ . ',' . __LINE__); // Cache the insights to improve the page load speed of the dashboard $dashboard_module_cacher = new DashboardModuleCacher($instance); $dashboard_module_cacher->cacheDashboardModules(); $instance_dao->save($crawler->instance, 0, $logger); Reporter::reportVersion($instance); // Tell the user crawling was sucessful $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__); } } }
public function crawl() { $logger = Logger::getInstance(); $config = Config::getInstance(); $instance_dao = DAOFactory::getDAO('FacebookInstanceDAO'); $owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO'); $owner_dao = DAOFactory::getDAO('OwnerDAO'); $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('facebook', true); //get cached $max_crawl_time = isset($options['max_crawl_time']) ? $options['max_crawl_time']->option_value : 20; //convert to seconds $max_crawl_time = $max_crawl_time * 60; $current_owner = $owner_dao->getByEmail(Session::getLoggedInUser()); //crawl Facebook user profiles and pages $profiles = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'facebook'); $pages = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'facebook page'); $instances = array_merge($profiles, $pages); foreach ($instances as $instance) { $logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username); $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__); $tokens = $owner_instance_dao->getOAuthTokens($instance->id); $access_token = $tokens['oauth_access_token']; $instance_dao->updateLastRun($instance->id); $facebook_crawler = new FacebookCrawler($instance, $access_token, $max_crawl_time); $dashboard_module_cacher = new DashboardModuleCacher($instance); try { $facebook_crawler->fetchPostsAndReplies(); } catch (APIOAuthException $e) { $logger->logUserError(get_class($e) . ": " . $e->getMessage(), __METHOD__ . ',' . __LINE__); //Don't send reauth email if it's app-level API rate limting //https://developers.facebook.com/docs/reference/ads-api/api-rate-limiting/#applimit if (strpos($e->getMessage(), 'Application request limit reached') === false && strpos($e->getMessage(), 'Please retry your request later.') === false) { //The access token is invalid, save in owner_instances table $owner_instance_dao->setAuthErrorByTokens($instance->id, $access_token, '', $e->getMessage()); //Send email alert //Get owner by auth tokens first, then send to that person $owner_email_to_notify = $owner_instance_dao->getOwnerEmailByInstanceTokens($instance->id, $access_token, ''); $email_attempt = $this->sendInvalidOAuthEmailAlert($owner_email_to_notify, $instance->network_username); if ($email_attempt) { $logger->logUserInfo('Sent reauth email to ' . $owner_email_to_notify, __METHOD__ . ',' . __LINE__); } else { $logger->logInfo('Didn\'t send reauth email to ' . $owner_email_to_notify, __METHOD__ . ',' . __LINE__); } } else { $logger->logInfo('Facebook API returned an error: ' . $e->getMessage() . ' Do nothing now and try again later', __METHOD__ . ',' . __LINE__); } } catch (Exception $e) { $logger->logUserError(get_class($e) . ": " . $e->getMessage(), __METHOD__ . ',' . __LINE__); } $dashboard_module_cacher->cacheDashboardModules(); $instance_dao->save($facebook_crawler->instance, 0, $logger); Reporter::reportVersion($instance); $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__); } }
public static function checkUserPermission() { $user = Session::getLoggedInUser(); if (Session::isUser()) { // You are allowed to perform operation return true; } else { return Session::throwHimOut(); } }
private function getController($logged_in, $is_admin = false) { if ($logged_in) { $this->simulateLogin('*****@*****.**', $is_admin); $owner_dao = DAOFactory::getDAO('OwnerDAO'); $owner = $owner_dao->getByEmail(Session::getLoggedInUser()); $controller = new InsightsGeneratorPluginConfigurationController($owner); } else { $controller = new InsightsGeneratorPluginConfigurationController(true); } return $controller; }
public function crawl() { $logger = Logger::getInstance(); $config = Config::getInstance(); $id = DAOFactory::getDAO('InstanceDAO'); $oid = DAOFactory::getDAO('OwnerInstanceDAO'); $od = DAOFactory::getDAO('OwnerDAO'); $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('facebook', true); //get cached $current_owner = $od->getByEmail(Session::getLoggedInUser()); //crawl Facebook user profiles $instances = $id->getAllActiveInstancesStalestFirstByNetwork('facebook'); foreach ($instances as $instance) { if (!$oid->doesOwnerHaveAccess($current_owner, $instance)) { // Owner doesn't have access to this instance; let's not crawl it. continue; } $logger->setUsername($instance->network_username); $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . " on Facebook.", __METHOD__ . ',' . __LINE__); $tokens = $oid->getOAuthTokens($instance->id); $access_token = $tokens['oauth_access_token']; $id->updateLastRun($instance->id); $crawler = new FacebookCrawler($instance, $access_token); try { $crawler->fetchInstanceUserInfo(); $crawler->fetchUserPostsAndReplies($instance->network_user_id); } catch (Exception $e) { $logger->logUserError('PROFILE EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__); } $id->save($crawler->instance, 0, $logger); $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . " on Facebook.", __METHOD__ . ',' . __LINE__); } //crawl Facebook pages $instances = $id->getAllActiveInstancesStalestFirstByNetwork('facebook page'); foreach ($instances as $instance) { $logger->setUsername($instance->network_username); $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s Facebook Page.", __METHOD__ . ',' . __LINE__); $tokens = $oid->getOAuthTokens($instance->id); $access_token = $tokens['oauth_access_token']; $id->updateLastRun($instance->id); $crawler = new FacebookCrawler($instance, $access_token); try { $crawler->fetchPagePostsAndReplies($instance->network_user_id); } catch (Exception $e) { $logger->logUserError('PAGE EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__); } $id->save($crawler->instance, 0, $logger); $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s Facebook Page.", __METHOD__ . ',' . __LINE__); } }
public function crawl() { $logger = Logger::getInstance(); $config = Config::getInstance(); $id = DAOFactory::getDAO('InstanceDAO'); $oid = DAOFactory::getDAO('OwnerInstanceDAO'); $od = DAOFactory::getDAO('OwnerDAO'); $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('facebook', true); //get cached $current_owner = $od->getByEmail(Session::getLoggedInUser()); //crawl Facebook user profiles $instances = $id->getAllActiveInstancesStalestFirstByNetwork('facebook'); foreach ($instances as $instance) { if (!$oid->doesOwnerHaveAccess($current_owner, $instance)) { // Owner doesn't have access to this instance; let's not crawl it. continue; } $logger->setUsername($instance->network_username); $tokens = $oid->getOAuthTokens($instance->id); $session_key = $tokens['oauth_access_token']; $fb = new Facebook($options['facebook_api_key']->option_value, $options['facebook_api_secret']->option_value); $id->updateLastRun($instance->id); $crawler = new FacebookCrawler($instance, $fb); try { $crawler->fetchInstanceUserInfo($instance->network_user_id, $session_key); $crawler->fetchUserPostsAndReplies($instance->network_user_id, $session_key); } catch (Exception $e) { $logger->logStatus('PROFILE EXCEPTION: ' . $e->getMessage(), get_class($this)); } $id->save($crawler->instance, $crawler->owner_object->post_count, $logger); } //crawl Facebook pages $instances = $id->getAllActiveInstancesStalestFirstByNetwork('facebook page'); foreach ($instances as $instance) { $logger->setUsername($instance->network_username); $tokens = $oid->getOAuthTokens($instance->id); $session_key = $tokens['oauth_access_token']; $fb = new Facebook($options['facebook_api_key']->option_value, $options['facebook_api_secret']->option_value); $id->updateLastRun($instance->id); $crawler = new FacebookCrawler($instance, $fb); try { $crawler->fetchPagePostsAndReplies($instance->network_user_id, $instance->network_viewer_id, $session_key); } catch (Exception $e) { $logger->logStatus('PAGE EXCEPTION: ' . $e->getMessage(), get_class($this)); } $id->save($crawler->instance, 0, $logger); } $logger->close(); # Close logging }
public function testConfigOptionsIsAdmin() { // build some options data $builders = $this->buildPluginData(); $this->simulateLogin('*****@*****.**', true); $owner_dao = DAOFactory::getDAO('OwnerDAO'); $owner = $owner_dao->getByEmail(Session::getLoggedInUser()); $controller = new TwitterRealtimePluginConfigurationController($owner, 'twitterrealtime'); $output = $controller->go(); $this->assertPattern('/save options/', $output); // should have no submit option $this->assertPattern('/php_path/', $output); $this->assertPattern('/redis/', $output); // should have secret option }
public function testOutputNoParams() { // build some options data $options_arry = $this->buildPluginOptions(); //not logged in, no owner set $controller = new FlickrThumbnailsPluginConfigurationController(null, 'flickrthumbnails'); $output = $controller->go(); $v_mgr = $controller->getViewManager(); $config = Config::getInstance(); $this->assertEqual('You must <a href="' . $config->getValue('site_root_path') . 'session/login.php">log in</a> to do this.', $v_mgr->getTemplateDataItem('errormsg')); //logged in $this->simulateLogin('*****@*****.**'); $owner_dao = DAOFactory::getDAO('OwnerDAO'); $owner = $owner_dao->getByEmail(Session::getLoggedInUser()); $controller = new FlickrThumbnailsPluginConfigurationController($owner, 'flickrthumbnails'); $output = $controller->go(); $this->assertPattern('/Flickr API key/', $output); }
public function update($status) { if (!Session::isAdmin() && !Session::isTeacher()) { $username = Session::getLoggedInUser(); $url = $_SERVER['REQUEST_URI']; $url_components = explode("/", $url); $count_url_components = count($url_components); for ($i = 0; $url_components[$i] != "challenges"; $i++) { } $pkg_name = $url_components[$i + 1]; $user = User::findByUserName($username); $challenge = Challenge::getChallengeByPkgName($pkg_name); $user_id = $user->id; $challenge_id = $challenge[0]->id; if (!ChallengeAttempts::isChallengeCleared($user_id, $challenge_id)) { ChallengeAttempts::addChallengeAttempt($user_id, $challenge_id, $status); } } }
public function crawl() { $logger = Logger::getInstance(); $config = Config::getInstance(); $instance_dao = DAOFactory::getDAO('InstanceDAO'); $owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO'); $owner_dao = DAOFactory::getDAO('OwnerDAO'); $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('facebook', true); //get cached $max_crawl_time = isset($options['max_crawl_time']) ? $options['max_crawl_time']->option_value : 20; //convert to seconds $max_crawl_time = $max_crawl_time * 60; $current_owner = $owner_dao->getByEmail(Session::getLoggedInUser()); //crawl Facebook user profiles and pages $profiles = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'facebook'); $pages = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'facebook page'); $instances = array_merge($profiles, $pages); foreach ($instances as $instance) { $logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username); $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__); $tokens = $owner_instance_dao->getOAuthTokens($instance->id); $access_token = $tokens['oauth_access_token']; $instance_dao->updateLastRun($instance->id); $facebook_crawler = new FacebookCrawler($instance, $access_token, $max_crawl_time); $dashboard_module_cacher = new DashboardModuleCacher($instance); try { $facebook_crawler->fetchPostsAndReplies(); } catch (APIOAuthException $e) { //The access token is invalid, save in owner_instances table $owner_instance_dao->setAuthError($current_owner->id, $instance->id, $e->getMessage()); //Send email alert $this->sendInvalidOAuthEmailAlert($current_owner->email, $instance->network_username); $logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__); } catch (Exception $e) { $logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__); } $dashboard_module_cacher->cacheDashboardModules(); $instance_dao->save($facebook_crawler->instance, 0, $logger); Reporter::reportVersion($instance); $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__); } }
/** * Runs registered plugins' crawl function. * * About crawler exclusivity (mutex usage): * When launched by an admin, no other user, admin or not, will be able to launch a crawl until this one is done. * When launched by a non-admin, we first check that no admin run is under way, and if that's the case, * we launch a crawl for the current user only. * No user will be able to launch two crawls in parallel, but different non-admin users crawls can run in parallel. * @throws UnauthorizedUserException If user is not logged in * @throws CrawlerLockedException If a crawl is already in progress * @throws InstallerException If ThinkUp is in the midst of a database upgrade */ public function runRegisteredPluginsCrawl() { if (!Session::isLoggedIn()) { throw new UnauthorizedUserException('You need a valid session to launch the crawler.'); } $mutex_dao = DAOFactory::getDAO('MutexDAO'); $owner_dao = DAOFactory::getDAO('OwnerDAO'); $owner = $owner_dao->getByEmail(Session::getLoggedInUser()); if (empty($owner)) { throw new UnauthorizedUserException('You need a valid session to launch the crawler.'); } // are we in an upgrading state if (UpgradeDatabaseController::isUpgrading(true, 'Crawler')) { throw new InstallerException("ThinkUp needs a database migration, so we are unable to run the crawler."); } $global_mutex_name = self::GLOBAL_MUTEX; // Everyone needs to check the global mutex $lock_successful = 1; $mutex_dao->getMutex($global_mutex_name); // 1 if ($lock_successful) { // Global mutex was free, which means no admin crawls are under way if ($owner->is_admin) { // Nothing more needs to be done, since admins use the global mutex $mutex_name = $global_mutex_name; } else { // User is a non-admin; let's use a user mutex. $mutex_name = 'crawler-' . $owner->id; $lock_successful = $mutex_dao->getMutex($mutex_name); $mutex_dao->releaseMutex($global_mutex_name); } } if ($lock_successful) { $this->emitObjectFunction('crawl'); $mutex_dao->releaseMutex($mutex_name); //clear cache so that insight stream updates $v_mgr = new ViewManager(); $v_mgr->clear_all_cache(); } else { throw new CrawlerLockedException("Error starting crawler; another crawl is already in progress."); } }
public function crawl() { $logger = Logger::getInstance(); $config = Config::getInstance(); $instance_dao = DAOFactory::getDAO('InstanceDAO'); $owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO'); $owner_dao = DAOFactory::getDAO('OwnerDAO'); $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('youtube', true); //get cached $max_crawl_time = isset($options['max_crawl_time']) ? $options['max_crawl_time']->option_value : 20; //convert to seconds $max_crawl_time = $max_crawl_time * 60; $developer_key = isset($options['developer_key']) ? $options['developer_key']->option_value : null; $max_comments = isset($options['max_comments']) ? $options['max_comments']->option_value : null; $current_owner = $owner_dao->getByEmail(Session::getLoggedInUser()); //crawl youtube users $instances = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'youtube'); if (isset($options['youtube_client_id']->option_value) && isset($options['youtube_client_secret']->option_value)) { foreach ($instances as $instance) { $logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username); $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__); $tokens = $owner_instance_dao->getOAuthTokens($instance->id); $access_token = $tokens['oauth_access_token']; $refresh_token = $tokens['oauth_access_token_secret']; $instance_dao->updateLastRun($instance->id); $youtube_crawler = new YouTubeCrawler($instance, $access_token, $max_crawl_time, $developer_key, $max_comments); $dashboard_module_cacher = new DashboardModuleCacher($instance); try { $youtube_crawler->initializeInstanceUser($options['youtube_client_id']->option_value, $options['youtube_client_secret']->option_value, $access_token, $refresh_token, $current_owner->id); $youtube_crawler->fetchInstanceUserVideos(); } catch (Exception $e) { $logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__); } $dashboard_module_cacher->cacheDashboardModules(); $instance_dao->save($youtube_crawler->instance, 0, $logger); Reporter::reportVersion($instance); $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__); } } }
public function go() { $this->setViewTemplate('progressreport.tpl'); if ($this->isAdmin() || $this->isTeacher()) { $this->addToView('search_box', true); if (isset($_GET['username'])) { $username = $_GET['username']; } } else { $username = Session::getLoggedInUser(); } if (isset($username)) { $user = User::findByUserName($username); if (!$user) { $this->addErrorMessage("You provided an invalid username"); return $this->generateView(); } elseif ($user->type) { $this->addErrorMessage("Please select a student!"); return $this->generateView(); } $challenges_of_user = ClassChallenges::getChallengesOfUser($user->id); $attempts = ChallengeAttempts::getTotalAttempts($user->id); $cleared_challenges = ChallengeAttempts::getClearedChallenges($user->id); $data = array(); foreach ($challenges_of_user as $id => $title) { $attempt = isset($attempts[$id]) ? $attempts[$id] : 0; $cleared = isset($cleared_challenges[$id]['cleared']) ? $cleared_challenges[$id]['cleared'] : false; if ($cleared) { $cleared_on = $cleared_challenges[$id]['cleared_on']; } else { $cleared_on = false; } $arr = array('id' => $id, 'title' => $title, 'attempts' => $attempt, 'cleared' => $cleared, 'cleared_on' => $cleared_on); array_push($data, $arr); } $this->addToView('data', $data); } else { $this->addErrorMessage("Please select a student to see his progress"); } return $this->generateView(); }
public function crawl() { $logger = Logger::getInstance(); // Include all the insights files so they register themselves foreach (glob(THINKUP_WEBAPP_PATH . "plugins/insightsgenerator/insights/*.php") as $filename) { require_once $filename; } //Get instances by owner $instance_dao = DAOFactory::getDAO('InstanceDAO'); $owner_dao = DAOFactory::getDAO('OwnerDAO'); $current_owner = $owner_dao->getByEmail(Session::getLoggedInUser()); $instances = $instance_dao->getByOwner($current_owner, false, true); // Get posts for last 7 days $number_days = 7; $post_dao = DAOFactory::getDAO('PostDAO'); $insights_plugin_registrar = PluginRegistrarInsights::getInstance(); foreach ($instances as $instance) { $last_week_of_posts = $post_dao->getAllPostsByUsernameOrderedBy($instance->network_username, $network = $instance->network, $count = 0, $order_by = "pub_date", $in_last_x_days = $number_days, $iterator = false, $is_public = false); $insights_plugin_registrar->runRegisteredPluginsInsightGeneration($instance, $last_week_of_posts, $number_days); $logger->logUserSuccess("Completed insight generation for " . $instance->network_username . " on " . $instance->network, __METHOD__ . ',' . __LINE__); } }
public function crawl() { $logger = Logger::getInstance(); $config = Config::getInstance(); $instance_dao = DAOFactory::getDAO('InstanceDAO'); $owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO'); $owner_dao = DAOFactory::getDAO('OwnerDAO'); $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('facebook', true); //get cached $max_crawl_time = isset($options['max_crawl_time']) ? $options['max_crawl_time']->option_value : 20; //convert to seconds $max_crawl_time = $max_crawl_time * 60; $current_owner = $owner_dao->getByEmail(Session::getLoggedInUser()); //crawl Facebook user profiles and pages $profiles = $instance_dao->getAllActiveInstancesStalestFirstByNetwork('facebook'); $pages = $instance_dao->getAllActiveInstancesStalestFirstByNetwork('facebook page'); $instances = array_merge($profiles, $pages); foreach ($instances as $instance) { if (!$owner_instance_dao->doesOwnerHaveAccessToInstance($current_owner, $instance)) { // Owner doesn't have access to this instance; let's not crawl it. continue; } $logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username); $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__); $tokens = $owner_instance_dao->getOAuthTokens($instance->id); $access_token = $tokens['oauth_access_token']; $instance_dao->updateLastRun($instance->id); $crawler = new FacebookCrawler($instance, $access_token, $max_crawl_time); try { $crawler->fetchPostsAndReplies(); } catch (Exception $e) { $logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__); } $instance_dao->save($crawler->instance, 0, $logger); $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__); } }
public function go() { $this->setViewTemplate('editor.tpl'); if (isset($_POST['submit'])) { if ($_POST['title'] == '') { $this->addErrorMessage("Title of the article should not be empty"); } elseif (!isset($_POST['is_published'])) { $this->addErrorMessage("Please tell if the article has been published successfully?"); } elseif ($_POST['content'] == '') { $this->addErrorMessage("Article post should not be empty"); } else { $this->created_by = Session::getLoggedInUser(); $this->title = $_POST['title']; $this->is_published = $_POST['is_published']; $this->content = $_POST['content']; $this->date_posted = date("Y-m-d H:i:s"); ArticleBackend::addArticle($this->title, $this->content, $this->date_posted, $this->created_by, $this->is_published); $this->addSuccessMessage("Article has been added succesfully"); $id = ArticleBackend::insertId(); header('Location: ' . SOURCE_ROOT_PATH . "admin/pages/editarticle.php?id={$id}&source=new"); } } $this->generateView(); }
public function testOutput() { //not logged in $controller = new TwitterPluginHashtagConfigurationController(null, 'twitter', 'ginatrapani'); $output = $controller->go(); $this->assertPattern('/session\\/login.php\\?redirect\\=/', $controller->redirect_destination); //logged in, no user set up $this->simulateLogin('*****@*****.**'); $owner_dao = DAOFactory::getDAO('OwnerDAO'); $owner = $owner_dao->getByEmail(Session::getLoggedInUser()); $controller = new TwitterPluginHashtagConfigurationController(null, 'twitter', 'ginatrapani'); $output = $controller->go(); $v_mgr = $controller->getViewManager(); $this->assertIsA($v_mgr->getTemplateDataItem('user'), 'string'); $this->assertEqual('Twitter user @ginatrapani does not exist.', $v_mgr->getTemplateDataItem('error_msg')); //logged in, user set up $builders = array(); $builders[] = FixtureBuilder::build('instances', array('network_username' => 'ginatrapani', 'network' => 'twitter')); $controller = new TwitterPluginHashtagConfigurationController(null, 'twitter', 'ginatrapani'); $output = $controller->go(); $v_mgr = $controller->getViewManager(); $this->assertIsA($v_mgr->getTemplateDataItem('user'), 'string'); $this->assertIsA($v_mgr->getTemplateDataItem('hashtags'), 'array'); }
public function testConfigOptionsMissingRequiredValues() { $_SERVER['SERVER_NAME'] = 'mytestthinkup'; $this->simulateLogin('*****@*****.**', true); $owner_dao = DAOFactory::getDAO('OwnerDAO'); $owner = $owner_dao->getByEmail(Session::getLoggedInUser()); $controller = new TwitterPluginConfigurationController($owner, 'twitter'); $output = $controller->go(); // we have a text form element with proper data $this->assertPattern('/save options/', $output); // should have no submit option $this->assertPattern('/plugin_options_oauth_consumer_secret/', $output); // should have secret option $this->assertPattern('/plugin_options_archive_limit/', $output); // should have limit option $this->assertPattern('/plugin_options_oauth_consumer_key/', $output); // should have key option $this->assertPattern('/var is_admin = true/', $output); // is a js admin //app not configured $this->assertPattern('/var required_values_set = false/', $output); // is not configured //not SSL by default $this->assertNoPattern('/https:\\/\\/mytestthinkup/', $output); }
/** * Test config isa admin */ public function testConfigOptionsIsAdmin() { // build some options data $this->simulateLogin('*****@*****.**', $isadmin = true); $owner_dao = DAOFactory::getDAO('OwnerDAO'); $owner = $owner_dao->getByEmail(Session::getLoggedInUser()); $controller = new ExpandURLsPluginConfigurationController($owner, 'flickrthumbnails'); $output = $controller->go(); // we have a text form element with proper data $this->assertPattern('/save options/', $output); // should have submit option $this->assertPattern('/plugin_options_error_flickr_api_key/', $output); // should have api key option $this->assertPattern('/var is_admin = true/', $output); // is a js admin //app not configured $controller = new ExpandURLsPluginConfigurationController($owner, 'flickrthumbnails'); $output = $controller->go(); $this->assertPattern('/var required_values_set = false/', $output); // is not configured }
/** * Return username of logged-in user * * @return str username */ public function getLoggedInUser() { return Session::getLoggedInUser(); }
/** * Return email address of logged-in user * * @return str email */ protected function getLoggedInUser() { return Session::getLoggedInUser(); }
public function testLogOut() { $this->simulateLogin('*****@*****.**', true); $session = new Session(); $this->assertTrue(Session::isLoggedIn()); $this->assertTrue(Session::isAdmin()); $this->assertEqual(Session::getLoggedInUser(), '*****@*****.**'); $session->logOut(); $this->assertFalse(Session::isLoggedIn()); $this->assertFalse(Session::isAdmin()); $this->assertNull(Session::getLoggedInUser()); }
public function crawl() { $logger = Logger::getInstance(); // Include all the insights files so they register themselves foreach (glob(THINKUP_WEBAPP_PATH . "plugins/insightsgenerator/insights/*.php") as $filename) { require_once $filename; } //Get instances by owner $instance_dao = DAOFactory::getDAO('InstanceDAO'); $owner_dao = DAOFactory::getDAO('OwnerDAO'); $current_owner = $owner_dao->getByEmail(Session::getLoggedInUser()); $instances = $instance_dao->getByOwner($current_owner, false, true); // Get posts for last 7 days $number_days = 7; $post_dao = DAOFactory::getDAO('PostDAO'); $insights_plugin_registrar = PluginRegistrarInsights::getInstance(); foreach ($instances as $instance) { $last_week_of_posts = $post_dao->getAllPostsByUsernameOrderedBy($instance->network_username, $network = $instance->network, $count = 0, $order_by = "pub_date", $in_last_x_days = $number_days, $iterator = false, $is_public = false); $insights_plugin_registrar->runRegisteredPluginsInsightGeneration($instance, $last_week_of_posts, $number_days); $logger->logUserSuccess("Completed insight generation for " . $instance->network_username . " on " . $instance->network, __METHOD__ . ',' . __LINE__); } // Don't do email for regular users if (!$current_owner->is_admin) { return; } // Send email digest the first run after 4am if ((int) date('G', $this->current_timestamp) >= 4) { //Get plugin options $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash($this->folder_name, true); //Get plugin ID $plugin_dao = DAOFactory::getDAO('PluginDAO'); $plugin_id = $plugin_dao->getPluginId($this->folder_name); //Get today's date $today = date('Y-m-d', $this->current_timestamp); $do_daily = false; $do_weekly = false; $last_daily = isset($options['last_daily_email']) ? $options['last_daily_email']->option_value : null; if ($last_daily != $today) { if ($last_daily === null) { $plugin_option_dao->insertOption($plugin_id, 'last_daily_email', $today); } else { $plugin_option_dao->updateOption($options['last_daily_email']->id, 'last_daily_email', $today); } $do_daily = true; } $last_weekly = isset($options['last_weekly_email']) ? $options['last_weekly_email']->option_value : null; if ($last_weekly != $today && date('w', $this->current_timestamp) == self::WEEKLY_DIGEST_DAY_OF_WEEK) { if ($last_weekly === null) { $plugin_option_dao->insertOption($plugin_id, 'last_weekly_email', $today); } else { $plugin_option_dao->updateOption($options['last_weekly_email']->id, 'last_weekly_email', $today); } $do_weekly = true; } if ($do_daily || $do_weekly) { $owners = $owner_dao->getAllOwners(); } if ($do_daily) { foreach ($owners as $owner) { if ($this->sendDailyDigest($owner, $options)) { $logger->logUserSuccess("Mailed daily digest to " . $owner->email . ".", __METHOD__ . ',' . __LINE__); } } } if ($do_weekly) { foreach ($owners as $owner) { if ($this->sendWeeklyDigest($owner, $options)) { $logger->logUserSuccess("Mailed weekly digest to " . $owner->email . ".", __METHOD__ . ',' . __LINE__); } } } } }
/** * set up stream data */ private function setUpData($use_redis = 0) { $builder_owner = FixtureBuilder::build('owners', array('email' => '*****@*****.**', 'user_activated' => 1)); $builder_plugin = FixtureBuilder::build('plugins', array('folder_name' => 'twitterrealtime', 'is_active' => 1)); $plugin_id = $builder_plugin->columns['last_insert_id']; $namespace = OptionDAO::PLUGIN_OPTIONS . '-' . $plugin_id; $builder_plugin_options = FixtureBuilder::build('options', array('namespace' => $namespace, 'option_name' => 'use_redis', 'option_value' => $use_redis)); $this->simulateLogin('*****@*****.**'); $owner_dao = DAOFactory::getDAO('OwnerDAO'); $owner = $owner_dao->getByEmail(Session::getLoggedInUser()); return array($builder_owner, $builder_plugin, $builder_plugin_options); }