public function crawl()
{
$logger = Logger::getInstance();
$config = Config::getInstance();
$instance_dao = DAOFactory::getDAO('FacebookInstanceDAO');
$owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
$owner_dao = DAOFactory::getDAO('OwnerDAO');
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('facebook', true);
//get cached
$max_crawl_time = isset($options['max_crawl_time']) ? $options['max_crawl_time']->option_value : 20;
//convert to seconds
$max_crawl_time = $max_crawl_time * 60;
$current_owner = $owner_dao->getByEmail(Session::getLoggedInUser());
//crawl Facebook user profiles and pages
$profiles = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'facebook');
$pages = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'facebook page');
$instances = array_merge($profiles, $pages);
foreach ($instances as $instance) {
$logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username);
$logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
$tokens = $owner_instance_dao->getOAuthTokens($instance->id);
$access_token = $tokens['oauth_access_token'];
$instance_dao->updateLastRun($instance->id);
$facebook_crawler = new FacebookCrawler($instance, $access_token, $max_crawl_time);
$dashboard_module_cacher = new DashboardModuleCacher($instance);
try {
$facebook_crawler->fetchPostsAndReplies();
} catch (APIOAuthException $e) {
$logger->logUserError(get_class($e) . ": " . $e->getMessage(), __METHOD__ . ',' . __LINE__);
//Don't send reauth email if it's app-level API rate limting
//https://developers.facebook.com/docs/reference/ads-api/api-rate-limiting/#applimit
if (strpos($e->getMessage(), 'Application request limit reached') === false && strpos($e->getMessage(), 'Please retry your request later.') === false) {
//The access token is invalid, save in owner_instances table
$owner_instance_dao->setAuthErrorByTokens($instance->id, $access_token, '', $e->getMessage());
//Send email alert
//Get owner by auth tokens first, then send to that person
$owner_email_to_notify = $owner_instance_dao->getOwnerEmailByInstanceTokens($instance->id, $access_token, '');
$email_attempt = $this->sendInvalidOAuthEmailAlert($owner_email_to_notify, $instance->network_username);
if ($email_attempt) {
$logger->logUserInfo('Sent reauth email to ' . $owner_email_to_notify, __METHOD__ . ',' . __LINE__);
} else {
$logger->logInfo('Didn\'t send reauth email to ' . $owner_email_to_notify, __METHOD__ . ',' . __LINE__);
}
} else {
$logger->logInfo('Facebook API returned an error: ' . $e->getMessage() . ' Do nothing now and try again later', __METHOD__ . ',' . __LINE__);
}
} catch (Exception $e) {
$logger->logUserError(get_class($e) . ": " . $e->getMessage(), __METHOD__ . ',' . __LINE__);
}
$dashboard_module_cacher->cacheDashboardModules();
$instance_dao->save($facebook_crawler->instance, 0, $logger);
Reporter::reportVersion($instance);
$logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
}
}
public function crawl()
{
$logger = Logger::getInstance();
$config = Config::getInstance();
$instance_dao = DAOFactory::getDAO('InstanceDAO');
$owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
$owner_dao = DAOFactory::getDAO('OwnerDAO');
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('facebook', true);
//get cached
$max_crawl_time = isset($options['max_crawl_time']) ? $options['max_crawl_time']->option_value : 20;
//convert to seconds
$max_crawl_time = $max_crawl_time * 60;
$current_owner = $owner_dao->getByEmail(Session::getLoggedInUser());
//crawl Facebook user profiles and pages
$profiles = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'facebook');
$pages = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'facebook page');
$instances = array_merge($profiles, $pages);
foreach ($instances as $instance) {
$logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username);
$logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
$tokens = $owner_instance_dao->getOAuthTokens($instance->id);
$access_token = $tokens['oauth_access_token'];
$instance_dao->updateLastRun($instance->id);
$facebook_crawler = new FacebookCrawler($instance, $access_token, $max_crawl_time);
$dashboard_module_cacher = new DashboardModuleCacher($instance);
try {
$facebook_crawler->fetchPostsAndReplies();
} catch (APIOAuthException $e) {
//The access token is invalid, save in owner_instances table
$owner_instance_dao->setAuthError($current_owner->id, $instance->id, $e->getMessage());
//Send email alert
$this->sendInvalidOAuthEmailAlert($current_owner->email, $instance->network_username);
$logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__);
} catch (Exception $e) {
$logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__);
}
$dashboard_module_cacher->cacheDashboardModules();
$instance_dao->save($facebook_crawler->instance, 0, $logger);
Reporter::reportVersion($instance);
$logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
}
}
public function crawl()
{
$logger = Logger::getInstance();
$config = Config::getInstance();
$instance_dao = DAOFactory::getDAO('InstanceDAO');
$owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
$owner_dao = DAOFactory::getDAO('OwnerDAO');
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('facebook', true);
//get cached
$max_crawl_time = isset($options['max_crawl_time']) ? $options['max_crawl_time']->option_value : 20;
//convert to seconds
$max_crawl_time = $max_crawl_time * 60;
$current_owner = $owner_dao->getByEmail(Session::getLoggedInUser());
//crawl Facebook user profiles and pages
$profiles = $instance_dao->getAllActiveInstancesStalestFirstByNetwork('facebook');
$pages = $instance_dao->getAllActiveInstancesStalestFirstByNetwork('facebook page');
$instances = array_merge($profiles, $pages);
foreach ($instances as $instance) {
if (!$owner_instance_dao->doesOwnerHaveAccessToInstance($current_owner, $instance)) {
// Owner doesn't have access to this instance; let's not crawl it.
continue;
}
$logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username);
$logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
$tokens = $owner_instance_dao->getOAuthTokens($instance->id);
$access_token = $tokens['oauth_access_token'];
$instance_dao->updateLastRun($instance->id);
$crawler = new FacebookCrawler($instance, $access_token, $max_crawl_time);
try {
$crawler->fetchPostsAndReplies();
} catch (Exception $e) {
$logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__);
}
$instance_dao->save($crawler->instance, 0, $logger);
$logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
}
}
public function testPostReplyPaging()
{
$fbc = new FacebookCrawler($this->instance, 'fauxaccesstoken');
$fbc->fetchPostsAndReplies('133954286636768', true);
$post_dao = new PostMySQLDAO();
$post = $post_dao->getPost('144568048938151', 'facebook page');
$this->assertEqual($post->reply_count_cache, 70);
}
public function testPaginatedPostLikes()
{
$fbc = new FacebookCrawler($this->profile5_instance, 'fauxaccesstoken', 10);
$fbc->fetchPostsAndReplies('501771984', 'facebook');
$post_dao = new PostMySQLDAO();
$post = $post_dao->getPost('10151734003261985', 'facebook');
$this->assertEqual($post->favlike_count_cache, 27);
}
public function testFetchPostsAndRepliesForPage()
{
$fbc = new FacebookCrawler($this->page1_instance, 'abc', 10);
$fbc->fetchPostsAndReplies();
$post_dao = new PostMySQLDAO();
$post = $post_dao->getPost('10152714332426356', 'facebook page');
$this->assertEqual($post->post_text, 'Looks like a very serene place to work:');
$this->assertFalse($post->is_protected);
$this->assertEqual($post->reply_count_cache, 4);
//test link with image
$this->assertEqual(sizeof($post->links), 1);
$this->assertEqual($post->links[0]->url, 'http://lifehac.kr/Jg0mR3E');
$this->assertEqual($post->links[0]->expanded_url, '');
$this->assertEqual($post->links[0]->image_src, 'https://fbexternal-a.akamaihd.net/safe_image.php?d=AQDJyDKV3JDoU0-y&w=130&h=130&url=http%3A%2F%2' . 'Fi.kinja-img.com%2Fgawker-media%2Fimage%2Fupload%2Fs--G2ekOhHZ--%2Fgobnypf78nigkkl7cwpl.png&cfs=1');
$this->assertEqual($post->links[0]->description, 'Today’s featured workspace looks immaculate, with its pristine white walls, simple white desk, and ' . 'wood accents. Those skylights help too.');
//assert user network is set to Facebook, not Facebook Page
$ud = new UserMySQLDAO();
$user = $ud->getUserByName('Gregory Robert Dumas', 'facebook');
$this->assertEqual($user->full_name, 'Gregory Robert Dumas');
$this->assertEqual($user->network, 'facebook');
$this->assertTrue($user->is_protected);
$user = $ud->getUserByName('Matthew Fleisher', 'facebook page');
$this->assertEqual($user, null);
$fav_dao = new FavoritePostMySQLDAO();
$favs = $fav_dao->getUsersWhoFavedPost('10152714332426356', 'facebook page');
$this->assertEqual($favs[0]['user_name'], 'Peter-Sarah Crofton');
$this->assertEqual($favs[0]['user_id'], '25913266');
// Test Facebook paging by confirming post on second "page" was captured
$post = $post_dao->getPost('10152712264921356', 'facebook page');
$this->assertNotNull($post);
$this->assertEqual($post->author_user_id, '7568536355');
}
public function crawl()
{
$logger = Logger::getInstance();
$config = Config::getInstance();
$instance_dao = DAOFactory::getDAO('InstanceDAO');
$owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
$owner_dao = DAOFactory::getDAO('OwnerDAO');
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('facebook', true);
//get cached
$current_owner = $owner_dao->getByEmail(Session::getLoggedInUser());
//crawl Facebook user profiles
$instances = $instance_dao->getAllActiveInstancesStalestFirstByNetwork('facebook');
foreach ($instances as $instance) {
if (!$owner_instance_dao->doesOwnerHaveAccess($current_owner, $instance)) {
// Owner doesn't have access to this instance; let's not crawl it.
continue;
}
$logger->setUsername($instance->network_username);
$logger->logUserSuccess("Starting to collect data for " . $instance->network_username . " on Facebook.", __METHOD__ . ',' . __LINE__);
$tokens = $owner_instance_dao->getOAuthTokens($instance->id);
$access_token = $tokens['oauth_access_token'];
$instance_dao->updateLastRun($instance->id);
$crawler = new FacebookCrawler($instance, $access_token);
try {
$crawler->fetchInstanceUserInfo();
$crawler->fetchPostsAndReplies($instance->network_user_id, false);
} catch (Exception $e) {
$logger->logUserError('PROFILE EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__);
}
$instance_dao->save($crawler->instance, 0, $logger);
$logger->logUserSuccess("Finished collecting data for " . $instance->network_username . " on Facebook.", __METHOD__ . ',' . __LINE__);
}
//crawl Facebook pages
$instances = $instance_dao->getAllActiveInstancesStalestFirstByNetwork('facebook page');
foreach ($instances as $instance) {
$logger->setUsername($instance->network_username);
$logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s Facebook Page.", __METHOD__ . ',' . __LINE__);
$tokens = $owner_instance_dao->getOAuthTokens($instance->id);
$access_token = $tokens['oauth_access_token'];
$instance_dao->updateLastRun($instance->id);
$crawler = new FacebookCrawler($instance, $access_token);
try {
$crawler->fetchPostsAndReplies($instance->network_user_id, true);
} catch (Exception $e) {
$logger->logUserError('PAGE EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__);
}
$instance_dao->save($crawler->instance, 0, $logger);
$logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s Facebook Page.", __METHOD__ . ',' . __LINE__);
}
}
