public function crawl() { $logger = Logger::getInstance(); $config = Config::getInstance(); $instance_dao = DAOFactory::getDAO('FacebookInstanceDAO'); $owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO'); $owner_dao = DAOFactory::getDAO('OwnerDAO'); $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('facebook', true); //get cached $max_crawl_time = isset($options['max_crawl_time']) ? $options['max_crawl_time']->option_value : 20; //convert to seconds $max_crawl_time = $max_crawl_time * 60; $current_owner = $owner_dao->getByEmail(Session::getLoggedInUser()); //crawl Facebook user profiles and pages $profiles = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'facebook'); $pages = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'facebook page'); $instances = array_merge($profiles, $pages); foreach ($instances as $instance) { $logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username); $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__); $tokens = $owner_instance_dao->getOAuthTokens($instance->id); $access_token = $tokens['oauth_access_token']; $instance_dao->updateLastRun($instance->id); $facebook_crawler = new FacebookCrawler($instance, $access_token, $max_crawl_time); $dashboard_module_cacher = new DashboardModuleCacher($instance); try { $facebook_crawler->fetchPostsAndReplies(); } catch (APIOAuthException $e) { $logger->logUserError(get_class($e) . ": " . $e->getMessage(), __METHOD__ . ',' . __LINE__); //Don't send reauth email if it's app-level API rate limting //https://developers.facebook.com/docs/reference/ads-api/api-rate-limiting/#applimit if (strpos($e->getMessage(), 'Application request limit reached') === false && strpos($e->getMessage(), 'Please retry your request later.') === false) { //The access token is invalid, save in owner_instances table $owner_instance_dao->setAuthErrorByTokens($instance->id, $access_token, '', $e->getMessage()); //Send email alert //Get owner by auth tokens first, then send to that person $owner_email_to_notify = $owner_instance_dao->getOwnerEmailByInstanceTokens($instance->id, $access_token, ''); $email_attempt = $this->sendInvalidOAuthEmailAlert($owner_email_to_notify, $instance->network_username); if ($email_attempt) { $logger->logUserInfo('Sent reauth email to ' . $owner_email_to_notify, __METHOD__ . ',' . __LINE__); } else { $logger->logInfo('Didn\'t send reauth email to ' . $owner_email_to_notify, __METHOD__ . ',' . __LINE__); } } else { $logger->logInfo('Facebook API returned an error: ' . $e->getMessage() . ' Do nothing now and try again later', __METHOD__ . ',' . __LINE__); } } catch (Exception $e) { $logger->logUserError(get_class($e) . ": " . $e->getMessage(), __METHOD__ . ',' . __LINE__); } $dashboard_module_cacher->cacheDashboardModules(); $instance_dao->save($facebook_crawler->instance, 0, $logger); Reporter::reportVersion($instance); $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__); } }
public function crawl() { $logger = Logger::getInstance(); $config = Config::getInstance(); $instance_dao = DAOFactory::getDAO('InstanceDAO'); $owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO'); $owner_dao = DAOFactory::getDAO('OwnerDAO'); $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('facebook', true); //get cached $max_crawl_time = isset($options['max_crawl_time']) ? $options['max_crawl_time']->option_value : 20; //convert to seconds $max_crawl_time = $max_crawl_time * 60; $current_owner = $owner_dao->getByEmail(Session::getLoggedInUser()); //crawl Facebook user profiles and pages $profiles = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'facebook'); $pages = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'facebook page'); $instances = array_merge($profiles, $pages); foreach ($instances as $instance) { $logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username); $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__); $tokens = $owner_instance_dao->getOAuthTokens($instance->id); $access_token = $tokens['oauth_access_token']; $instance_dao->updateLastRun($instance->id); $facebook_crawler = new FacebookCrawler($instance, $access_token, $max_crawl_time); $dashboard_module_cacher = new DashboardModuleCacher($instance); try { $facebook_crawler->fetchPostsAndReplies(); } catch (APIOAuthException $e) { //The access token is invalid, save in owner_instances table $owner_instance_dao->setAuthError($current_owner->id, $instance->id, $e->getMessage()); //Send email alert $this->sendInvalidOAuthEmailAlert($current_owner->email, $instance->network_username); $logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__); } catch (Exception $e) { $logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__); } $dashboard_module_cacher->cacheDashboardModules(); $instance_dao->save($facebook_crawler->instance, 0, $logger); Reporter::reportVersion($instance); $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__); } }
public function crawl() { $logger = Logger::getInstance(); $config = Config::getInstance(); $instance_dao = DAOFactory::getDAO('InstanceDAO'); $owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO'); $owner_dao = DAOFactory::getDAO('OwnerDAO'); $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('facebook', true); //get cached $max_crawl_time = isset($options['max_crawl_time']) ? $options['max_crawl_time']->option_value : 20; //convert to seconds $max_crawl_time = $max_crawl_time * 60; $current_owner = $owner_dao->getByEmail(Session::getLoggedInUser()); //crawl Facebook user profiles and pages $profiles = $instance_dao->getAllActiveInstancesStalestFirstByNetwork('facebook'); $pages = $instance_dao->getAllActiveInstancesStalestFirstByNetwork('facebook page'); $instances = array_merge($profiles, $pages); foreach ($instances as $instance) { if (!$owner_instance_dao->doesOwnerHaveAccessToInstance($current_owner, $instance)) { // Owner doesn't have access to this instance; let's not crawl it. continue; } $logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username); $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__); $tokens = $owner_instance_dao->getOAuthTokens($instance->id); $access_token = $tokens['oauth_access_token']; $instance_dao->updateLastRun($instance->id); $crawler = new FacebookCrawler($instance, $access_token, $max_crawl_time); try { $crawler->fetchPostsAndReplies(); } catch (Exception $e) { $logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__); } $instance_dao->save($crawler->instance, 0, $logger); $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__); } }
public function testPostReplyPaging() { $fbc = new FacebookCrawler($this->instance, 'fauxaccesstoken'); $fbc->fetchPostsAndReplies('133954286636768', true); $post_dao = new PostMySQLDAO(); $post = $post_dao->getPost('144568048938151', 'facebook page'); $this->assertEqual($post->reply_count_cache, 70); }
public function testPaginatedPostLikes() { $fbc = new FacebookCrawler($this->profile5_instance, 'fauxaccesstoken', 10); $fbc->fetchPostsAndReplies('501771984', 'facebook'); $post_dao = new PostMySQLDAO(); $post = $post_dao->getPost('10151734003261985', 'facebook'); $this->assertEqual($post->favlike_count_cache, 27); }
public function testFetchPostsAndRepliesForPage() { $fbc = new FacebookCrawler($this->page1_instance, 'abc', 10); $fbc->fetchPostsAndReplies(); $post_dao = new PostMySQLDAO(); $post = $post_dao->getPost('10152714332426356', 'facebook page'); $this->assertEqual($post->post_text, 'Looks like a very serene place to work:'); $this->assertFalse($post->is_protected); $this->assertEqual($post->reply_count_cache, 4); //test link with image $this->assertEqual(sizeof($post->links), 1); $this->assertEqual($post->links[0]->url, 'http://lifehac.kr/Jg0mR3E'); $this->assertEqual($post->links[0]->expanded_url, ''); $this->assertEqual($post->links[0]->image_src, 'https://fbexternal-a.akamaihd.net/safe_image.php?d=AQDJyDKV3JDoU0-y&w=130&h=130&url=http%3A%2F%2' . 'Fi.kinja-img.com%2Fgawker-media%2Fimage%2Fupload%2Fs--G2ekOhHZ--%2Fgobnypf78nigkkl7cwpl.png&cfs=1'); $this->assertEqual($post->links[0]->description, 'Today’s featured workspace looks immaculate, with its pristine white walls, simple white desk, and ' . 'wood accents. Those skylights help too.'); //assert user network is set to Facebook, not Facebook Page $ud = new UserMySQLDAO(); $user = $ud->getUserByName('Gregory Robert Dumas', 'facebook'); $this->assertEqual($user->full_name, 'Gregory Robert Dumas'); $this->assertEqual($user->network, 'facebook'); $this->assertTrue($user->is_protected); $user = $ud->getUserByName('Matthew Fleisher', 'facebook page'); $this->assertEqual($user, null); $fav_dao = new FavoritePostMySQLDAO(); $favs = $fav_dao->getUsersWhoFavedPost('10152714332426356', 'facebook page'); $this->assertEqual($favs[0]['user_name'], 'Peter-Sarah Crofton'); $this->assertEqual($favs[0]['user_id'], '25913266'); // Test Facebook paging by confirming post on second "page" was captured $post = $post_dao->getPost('10152712264921356', 'facebook page'); $this->assertNotNull($post); $this->assertEqual($post->author_user_id, '7568536355'); }
public function crawl() { $logger = Logger::getInstance(); $config = Config::getInstance(); $instance_dao = DAOFactory::getDAO('InstanceDAO'); $owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO'); $owner_dao = DAOFactory::getDAO('OwnerDAO'); $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO'); $options = $plugin_option_dao->getOptionsHash('facebook', true); //get cached $current_owner = $owner_dao->getByEmail(Session::getLoggedInUser()); //crawl Facebook user profiles $instances = $instance_dao->getAllActiveInstancesStalestFirstByNetwork('facebook'); foreach ($instances as $instance) { if (!$owner_instance_dao->doesOwnerHaveAccess($current_owner, $instance)) { // Owner doesn't have access to this instance; let's not crawl it. continue; } $logger->setUsername($instance->network_username); $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . " on Facebook.", __METHOD__ . ',' . __LINE__); $tokens = $owner_instance_dao->getOAuthTokens($instance->id); $access_token = $tokens['oauth_access_token']; $instance_dao->updateLastRun($instance->id); $crawler = new FacebookCrawler($instance, $access_token); try { $crawler->fetchInstanceUserInfo(); $crawler->fetchPostsAndReplies($instance->network_user_id, false); } catch (Exception $e) { $logger->logUserError('PROFILE EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__); } $instance_dao->save($crawler->instance, 0, $logger); $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . " on Facebook.", __METHOD__ . ',' . __LINE__); } //crawl Facebook pages $instances = $instance_dao->getAllActiveInstancesStalestFirstByNetwork('facebook page'); foreach ($instances as $instance) { $logger->setUsername($instance->network_username); $logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s Facebook Page.", __METHOD__ . ',' . __LINE__); $tokens = $owner_instance_dao->getOAuthTokens($instance->id); $access_token = $tokens['oauth_access_token']; $instance_dao->updateLastRun($instance->id); $crawler = new FacebookCrawler($instance, $access_token); try { $crawler->fetchPostsAndReplies($instance->network_user_id, true); } catch (Exception $e) { $logger->logUserError('PAGE EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__); } $instance_dao->save($crawler->instance, 0, $logger); $logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s Facebook Page.", __METHOD__ . ',' . __LINE__); } }