/** * For a given post, extract URLs and store them, including image_src if that's from a known source like Twitpic, * Twitgoo, Yfrog, Instagr.am. * @param str $post_text * @param int $post_id * @param str $network * @param Logger $logger * @param arr $urls Array of URLs, optionally set, defaults to null */ public static function processPostURLs($post_text, $post_id, $network, $logger, $urls = null) { if (!$urls) { $urls = Post::extractURLs($post_text); } if ($urls) { $link_dao = DAOFactory::getDAO('LinkDAO'); $post_dao = DAOFactory::getDAO('PostDAO'); $post = $post_dao->getPost($post_id, $network); if (isset($post->id)) { foreach ($urls as $url) { $logger->logInfo("Processing URL {$url}", __METHOD__ . ',' . __LINE__); $image_src = self::getImageSource($url); //if we have an image_src, the URL is a known image source not in need of expansion $expanded_url = $image_src !== '' ? $url : ''; $link_array = array('url' => $url, 'expanded_url' => $expanded_url, "image_src" => $image_src, 'post_key' => $post->id); $link = new Link($link_array); try { $link_dao->insert($link); $logger->logSuccess("Inserted " . $url . " " . ($image_src == '' ? '' : "(thumbnail " . $image_src . ") ") . "into links table", __METHOD__ . ',' . __LINE__); } catch (DuplicateLinkException $e) { $logger->logInfo($url . " " . ($image_src == '' ? '' : "(thumbnail " . $image_src . ") ") . " already exists in links table", __METHOD__ . ',' . __LINE__); } catch (DataExceedsColumnWidthException $e) { $logger->logInfo($url . " " . ($image_src == '' ? '' : "(thumbnail " . $image_src . ") ") . " data exceeds table column width", __METHOD__ . ',' . __LINE__); } } } } }
/** * Update profiles of users who are friends of the instance user, and haven't been checked in 1 day. * @return void */ public function updateFriendsProfiles() { if (!isset($this->user)) { $this->fetchInstanceUserInfo(); } if (isset($this->user)) { //Get stalest friends $follow_dao = DAOFactory::getDAO('FollowDAO'); $stalest_friends = $follow_dao->getStalestFriends($this->user->user_id, 'twitter', $number_days_old = 1, $limit = 25); $status_message = count($stalest_friends) . ' friends haven\'t been updated recently.'; $this->logger->logInfo($status_message, __METHOD__ . ',' . __LINE__); while (isset($stalest_friends) && count($stalest_friends) > 0) { try { foreach ($stalest_friends as $user) { $this->fetchAndAddUser($user->user_id, "Friends stale update"); } $stalest_friends = $follow_dao->getStalestFriends($this->user->user_id, 'twitter', $number_days_old = 1, $limit = 25); $status_message = count($stalest_friends) . ' friends haven\'t been updated recently.'; $this->logger->logInfo($status_message, __METHOD__ . ',' . __LINE__); } catch (APICallLimitExceededException $e) { $this->logger->logInfo($e->getMessage(), __METHOD__ . ',' . __LINE__); break; } } } }
/** * Update profiles of users who are friends of the instance user, and haven't been checked in 2 days. * @return void */ public function updateStaleFriendsProfiles() { if (!isset($this->user)) { //Force-refresh instance user in data store $this->user = self::fetchUser($this->instance->network_user_id, 'Owner info', $this->instance->network_username, null, null, true); } if (isset($this->user)) { //Get stalest friends $follow_dao = DAOFactory::getDAO('FollowDAO'); $stalest_friends = $follow_dao->getStalestFriends($this->user->user_id, 'instagram', $number_days_old = 2, $limit = 25); $status_message = count($stalest_friends) . ' friends haven\'t been updated recently.'; $this->logger->logInfo($status_message, __METHOD__ . ',' . __LINE__); while (isset($stalest_friends) && count($stalest_friends) > 0) { try { foreach ($stalest_friends as $user) { $this->fetchUser($user->user_id, "Friends stale update", $user->username, $user->full_name, $user->avatar, true); } $stalest_friends = $follow_dao->getStalestFriends($this->user->user_id, 'instagram', $number_days_old = 2, $limit = 25); $status_message = count($stalest_friends) . ' friends haven\'t been updated recently.'; $this->logger->logInfo($status_message, __METHOD__ . ',' . __LINE__); } catch (APICallLimitExceededException $e) { $this->logger->logInfo($e->getMessage(), __METHOD__ . ',' . __LINE__); break; } } } }
/** * This helper method returns the parsed favorites from a given favorites page. * @param int $since_id * @return array ($tweets, $http_status, $payload) */ private function getFavorites($since_id) { $endpoint = $this->api->endpoints['favorites']; $args = array(); $args["screen_name"] = $this->user->username; $args["include_entities"] = "false"; $args["count"] = 100; if ($since_id != "") { $args["since_id"] = $since_id; } try { list($http_status, $payload) = $this->api->apiRequest($endpoint, $args); if ($http_status == 200) { // Parse the JSON file $tweets = $this->api->parseJSONTweets($payload); if (!(isset($tweets) && sizeof($tweets) == 0) && $tweets == null) { // arghh, empty array evals to null $this->logger->logInfo("in getFavorites; could not extract any tweets from response", __METHOD__ . ',' . __LINE__); throw new Exception("could not extract any tweets from response"); } } } catch (APICallLimitExceededException $e) { $this->logger->logInfo($e->getMessage(), __METHOD__ . ',' . __LINE__); } return array($tweets, $http_status, $payload); }
private function storePostsAndAuthors($posts, $posts_source) { $total_added_posts = 0; $added_posts = 0; $post_dao = DAOFactory::getDAO('PostDAO'); foreach ($posts as $post) { if (isset($post['author_user_id'])) { $user_object = $this->fetchUserInfo($post['author_user_id'], 'facebook', $posts_source); if (isset($user_object)) { $post["author_username"] = $user_object->full_name; $post["author_fullname"] = $user_object->full_name; $post["author_avatar"] = $user_object->avatar; $post["location"] = $user_object->location; } } $added_posts = $post_dao->addPost($post); if ($added_posts == 0 && isset($post['favlike_count_cache'])) { //post already exists in storage, so update its like count only $post_dao->updateFavLikeCount($post['post_id'], $post['network'], $post['favlike_count_cache']); $this->logger->logInfo("Updated Like count for post ID " . $post["post_id"] . " to " . $post['favlike_count_cache'], __METHOD__ . ',' . __LINE__); } if ($added_posts > 0) { $this->logger->logInfo("Added post ID " . $post["post_id"] . " on " . $post["network"] . " for " . $post["author_username"] . ":" . $post["post_text"], __METHOD__ . ',' . __LINE__); } $total_added_posts = $total_added_posts + $added_posts; $added_posts = 0; } return $added_posts; }
/** * Check the validity of G+'s OAuth token by requestig the instance user's details. * Fetch details from Google+ API for the current instance user and insert into the datastore. * @param str $client_id * @param str $client_secret * @param str $access_token * @param str $refresh_token * @param str $owner_id * @return User */ public function initializeInstanceUser($client_id, $client_secret, $access_token, $refresh_token, $owner_id) { $network = 'google+'; $user_dao = DAOFactory::getDAO('UserDAO'); $user_object = null; // Get owner user details and save them to DB $fields = array('fields' => 'displayName,id,image,tagline'); $user_details = $this->api_accessor->apiRequest('people/me', $this->access_token, $fields); if (isset($user_details->error->code) && $user_details->error->code == '401') { //Token has expired, fetch and save a new one $tokens = self::getOAuthTokens($client_id, $client_secret, $refresh_token, 'refresh_token'); $owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO'); $owner_instance_dao->updateTokens($owner_id, $this->instance->id, $access_token, $refresh_token); $this->access_token = $tokens->access_token; //try again $user_details = $this->api_accessor->apiRequest('people/me', $this->access_token, $fields); } $user_details->network = $network; $user = $this->parseUserDetails($user_details); if (isset($user)) { $user_object = new User($user, 'Owner initialization'); $user_dao->updateUser($user_object); } if (isset($user_object)) { $this->logger->logSuccess("Successfully fetched " . $user_object->username . " " . $user_object->network . "'s details from Google+", __METHOD__ . ',' . __LINE__); } else { $this->logger->logInfo("Error fetching " . $user_id . " " . $network . "'s details from the Google+ API, " . "response was " . Utils::varDumpToString($user_details), __METHOD__ . ',' . __LINE__); } return $user_object; }
/** * For fetching details about a user who does not have a Gooogle Plus ID * If user doesn't exist in the datastore, fetch details from YouTube API and insert into the datastore. * If $reload_from_youtube is true, update existing user details in store with data from YouTube API. * @param int $user_id YouTube user ID * @param str $found_in Where the user was found * @param bool $reload_from_youtube Defaults to false; if true will query YouTube API and update existing user * @return User */ public function fetchUserFromYouTube($user_id, $found_in, $force_reload_from_youtube = false) { $network = 'youtube'; $user_dao = DAOFactory::getDAO('UserDAO'); $user_object = null; if ($force_reload_from_youtube || !$user_dao->isUserInDB($user_id, $network)) { // Get owner user details and save them to DB $fields = array('alt' => 'json'); if (isset($this->developer_key)) { $fields['key'] = $this->developer_key; } $user_details = $this->youtube_api_v2_accessor->apiRequest('users/' . $user_id_string[sizeof($user_id_string) - 1], $fields); $user = $this->parseYouTubeUserDetails($user_details->entry); if (isset($user)) { $user_object = new User($user, $found_in); $user_dao->updateUser($user_object); } if (isset($user_object)) { $this->logger->logSuccess("Successfully fetched " . $user_id . " " . $network . "'s details from YouTube", __METHOD__ . ',' . __LINE__); } else { $this->logger->logInfo("Error fetching " . $user_id . " " . $network . "'s details from the YouTube API, " . "response was " . Utils::varDumpToString($user_details), __METHOD__ . ',' . __LINE__); } } return $user_object; }
/** * If link is an image (Twitpic/Twitgoo/Yfrog/Flickr for now), insert direct path to thumb as expanded url. * @TODO Move image thumbnail processng to Expand URLs plugin. * @param Logger $logger * @param str $tweet * @param Array $urls */ public static function processTweetURLs($logger, $tweet, $urls = null) { $link_dao = DAOFactory::getDAO('LinkDAO'); if (!$urls) { $urls = Post::extractURLs($tweet['post_text']); } foreach ($urls as $u) { $logger->logInfo("processing url: {$u}", __METHOD__ . ',' . __LINE__); $is_image = 0; $title = ''; $eurl = ''; if (substr($u, 0, strlen('http://twitpic.com/')) == 'http://twitpic.com/') { $eurl = 'http://twitpic.com/show/thumb/' . substr($u, strlen('http://twitpic.com/')); $is_image = 1; } elseif (substr($u, 0, strlen('http://yfrog.com/')) == 'http://yfrog.com/') { $eurl = $u . '.th.jpg'; $is_image = 1; } elseif (substr($u, 0, strlen('http://twitgoo.com/')) == 'http://twitgoo.com/') { $eurl = 'http://twitgoo.com/show/thumb/' . substr($u, strlen('http://twitgoo.com/')); $is_image = 1; } elseif (substr($u, 0, strlen('http://picplz.com/')) == 'http://picplz.com/') { $eurl = $u . '/thumb/'; $is_image = 1; } elseif (substr($u, 0, strlen('http://flic.kr/')) == 'http://flic.kr/') { $is_image = 1; } elseif (substr($u, 0, strlen('http://instagr.am/')) == 'http://instagr.am/') { $is_image = 1; } if ($link_dao->insert($u, $eurl, $title, $tweet['post_id'], 'twitter', $is_image)) { $logger->logSuccess("Inserted " . $u . " (" . $eurl . ", " . $is_image . "), into links table", __METHOD__ . ',' . __LINE__); } else { $logger->logError("Did NOT insert " . $u . " (" . $eurl . ") into links table", __METHOD__ . ',' . __LINE__); } } }
/** * Make an API request. * @param str $type String representing the endpoint, 'user', 'followers', 'media', 'relationship' * @param arr $params API call params * @return Object */ public function apiRequest($type, $params = array()) { if ($this->total_api_calls >= $this->max_api_calls) { $this->logger->logInfo("Throw APICallLimitExceededException", __METHOD__ . ',' . __LINE__); throw new APICallLimitExceededException(); } else { $this->total_api_calls++; $this->logger->logInfo("Made " . $this->total_api_calls . " API calls", __METHOD__ . ',' . __LINE__); } try { if ($type == 'user') { return $this->instagram->getUser($params['user_id']); } else { if ($type == 'followers') { return $this->current_user->getFollowers($params); } else { if ($type == 'friends') { return $this->current_user->getFollows($params); } else { if ($type == 'media') { return $this->current_user->getMedia($params); } else { if ($type == 'relationship') { return $this->current_user->getRelationship($params['user_id']); } else { if ($type == 'likes') { return $this->current_user->getLikedMedia($params); } } } } } } } catch (Instagram\Core\ApiException $e) { if ($e->getMessage() == 'you cannot view this resource') { $this->logger->logInfo("Throw APICallPermissionDeniedException: " . $e->getMessage(), __METHOD__ . ',' . __LINE__); throw new APICallPermissionDeniedException($e->getMessage()); } elseif (strpos($e->getMessage(), 'exceeded the maximum number of requests per hour') !== false) { $this->logger->logInfo("Throw APICallLimitExceededException: " . $e->getMessage(), __METHOD__ . ',' . __LINE__); throw new APICallLimitExceededException($e->getMessage()); } else { $this->logger->logInfo("Throw APIErrorException: " . $e->getMessage(), __METHOD__ . ',' . __LINE__); throw new APIErrorException($e->getMessage()); } } }
/** * Capture the current instance users's posts and store them in the database. * @return null */ public function fetchInstanceUserPosts() { //For now only capture the most recent 20 posts //@TODO Page back through all the archives $fields = array('alt' => 'json', 'maxResults' => 20, 'pp' => 1); $user_posts = $this->api_accessor->apiRequest('people/' . $this->instance->network_user_id . '/activities/public', $this->access_token, $fields); if (isset($user_posts->items)) { $post_dao = DAOFactory::getDAO('PostDAO'); $link_dao = DAOFactory::getDAO('LinkDAO'); foreach ($user_posts->items as $item) { $should_capture_post = false; //For now we're only capturing posts and shares //@TODO Capture all types of posts if ($item->verb == "post") { $post['post_text'] = $item->object->content; $should_capture_post = true; } elseif ($item->verb == "share") { $post['post_text'] = isset($item->annotation) ? $item->annotation : ''; $should_capture_post = true; } if ($should_capture_post) { $post['post_id'] = $item->id; $post['author_username'] = $item->actor->displayName; $post['author_fullname'] = $item->actor->displayName; $post['author_avatar'] = $item->actor->image->url; $post['author_user_id'] = $item->actor->id; $post['pub_date'] = $item->published; $post['source'] = ''; $post['is_protected'] = false; $post['network'] = 'google+'; $post['reply_count_cache'] = $item->object->replies->totalItems; $post['favlike_count_cache'] = $item->object->plusoners->totalItems; $post['retweet_count_cache'] = $item->object->resharers->totalItems; $inserted_post_key = $post_dao->addPost($post); //If no post was added, at least update reply/fave/reshare counts and post text if ($inserted_post_key === false) { $post_dao->updateFavLikeCount($post['post_id'], 'google+', $post['favlike_count_cache']); $post_dao->updateReplyCount($post['post_id'], 'google+', $post['reply_count_cache']); $post_dao->updateRetweetCount($post['post_id'], 'google+', $post['retweet_count_cache']); $post_dao->updatePostText($post['post_id'], 'google+', $post['post_text']); } if (isset($item->object->attachments) && isset($item->object->attachments[0]->url)) { $link_url = $item->object->attachments[0]->url; $link = new Link(array("url" => $link_url, "expanded_url" => '', "image_src" => isset($item->object->attachments[0]->image->url) ? $item->object->attachments[0]->image->url : '', "caption" => '', "description" => isset($item->object->attachments[0]->content) ? $item->object->attachments[0]->content : '', "title" => isset($item->object->attachments[0]->displayName) ? $item->object->attachments[0]->displayName : '', "post_key" => $inserted_post_key)); try { $added_links = $link_dao->insert($link); } catch (DuplicateLinkException $e) { $this->logger->logInfo($link->url . " already exists in links table", __METHOD__ . ',' . __LINE__); } catch (DataExceedsColumnWidthException $e) { $this->logger->logInfo($link->url . " data exceeds table column width", __METHOD__ . ',' . __LINE__); } } } $post = null; $link = null; } } }
/** * Retrives all of a users friends from the Instagram API and stores them in the database * @return null */ private function storeFriends() { if ($this->instance->network != 'instagram') { return; } //Retrieve friends via the Instagram API $user_id = $this->instance->network_user_id; $access_token = $this->access_token; $network = $user_id == $this->instance->network_user_id ? $this->instance->network : 'instagram'; try { $friends = InstagramAPIAccessor::apiRequest('friends', $user_id, $access_token); } catch (Instagram\Core\ApiException $e) { $this->logger->logInfo(get_class($e) . " Error fetching friends from Instagram API, error was " . $e->getMessage(), __METHOD__ . ',' . __LINE__); return; } if (isset($friends)) { //store relationships in follows table $follows_dao = DAOFactory::getDAO('FollowDAO'); $count_dao = DAOFactory::getDAO('CountHistoryDAO'); $user_dao = DAOFactory::getDAO('UserDAO'); foreach ($friends as $friend) { $follower_id = null; try { $follower_id = $friend->getId(); } catch (Instagram\Core\ApiException $e) { $this->logger->logInfo(get_class($e) . " Error fetching " . Utils::varDumpToString($friend) . "'s details from Instagram API, error was " . $e->getMessage(), __METHOD__ . ',' . __LINE__); } if (isset($follower_id)) { if ($follows_dao->followExists($user_id, $follower_id, $network)) { // follow relationship already exists $follows_dao->update($user_id, $follower_id, $network); } else { // follow relationship does not exist yet $follows_dao->insert($user_id, $follower_id, $network); } $follower_details = $friend; if (isset($follower_details)) { $follower_details->network = $network; } $follower = $this->parseUserDetails($follower_details); $follower_object = new User($follower); if (isset($follower_object)) { $user_dao->updateUser($follower_object); } } } //totals in follower_count table $count_dao->insert($user_id, $network, $friends->count(), null, 'followers'); } else { throw new Instagram\Core\ApiAuthException('Error retrieving friends'); } $this->logger->logInfo("Ending", __METHOD__ . ',' . __LINE__); }
/** * Fetch a save the posts and replies on a Facebook page. * @param int $pid Page ID */ public function fetchPagePostsAndReplies($pid) { $stream = FacebookGraphAPIAccessor::apiRequest('/'.$pid.'/posts', $this->access_token); if (isset($stream->data) && is_array($stream->data) && sizeof($stream->data > 0)) { $this->logger->logSuccess(sizeof($stream->data)." Facebook posts found for page ID $pid.", __METHOD__.','.__LINE__); $thinkup_data = $this->parseStream($stream, 'facebook page'); $posts = $thinkup_data["posts"]; $post_dao = DAOFactory::getDAO('PostDAO'); $added_posts = 0; foreach ($posts as $post) { if ($post['author_username']== "" && isset($post['author_user_id'])) { $commenter_object = $this->fetchUserInfo($post['author_user_id'], 'facebook', 'Facebook page comments'); if (isset($commenter_object)) { $post["author_username"] = $commenter_object->full_name; $post["author_fullname"] = $commenter_object->full_name; $post["author_avatar"] = $commenter_object->avatar; } } $added_posts = $added_posts + $post_dao->addPost($post); $this->logger->logInfo("Added post ID ".$post["post_id"]." on ".$post["network"]. " for ".$post["author_username"].":".$post["post_text"], __METHOD__.','.__LINE__); } $added_users = 0; $users = $thinkup_data["users"]; if (count($users) > 0) { foreach ($users as $user) { $user["post_count"] = $post_dao->getTotalPostsByUser($user['user_id'], $user['network']); $found_in = 'Facebook page stream'; $user_object = new User($user, $found_in); $user_dao = DAOFactory::getDAO('UserDAO'); $user_dao->updateUser($user_object); $added_users = $added_users + 1; } } if ($added_posts > 0 || $added_users > 0) { $this->logger->logUserSuccess($added_posts." post(s) added; ".$added_users." user(s) updated.", __METHOD__.','.__LINE__); } else { $this->logger->logUserInfo("No new page posts found.", __METHOD__.','.__LINE__); } } else { $this->logger->logInfo("No Facebook posts found for page ID $pid", __METHOD__.','.__LINE__); } }
public function testAddRmOldFavMaintSearch() { $this->logger->logInfo("in testAddRmOldFavMaintSearch", __METHOD__.','.__LINE__); //set plugin cfg values $namespace = OptionDAO::PLUGIN_OPTIONS . '-1'; $builder2 = FixtureBuilder::build('options', array('namespace' => $namespace, 'option_name'=>'favs_older_pages','option_value'=>1)); $builder3 = FixtureBuilder::build('options', array('namespace' => $namespace, 'option_name'=>'favs_cleanup_pages','option_value'=>3)); $id = DAOFactory::getDAO('InstanceDAO'); self::setUpInstanceUserAmygdala(); $this->api->available_api_calls_for_crawler = 3; $this->api->to->setDataPath('webapp/plugins/twitter/tests/testdata/favs_tests/favs_stage3/'); $tc = new TwitterCrawler($this->instance, $this->api); $tc->fetchInstanceUserInfo(); $retval = $tc->cleanUpMissedFavsUnFavs(); $this->assertEqual($retval, true); $this->assertEqual($this->instance->last_unfav_page_checked, 3); // Save instance if (isset($tc->user)) { $id->save($this->instance, $tc->user->post_count, $this->logger); } $this->instance = $id->getByUsernameOnNetwork("amygdala", "twitter"); // check fav count $this->assertEqual($this->instance->owner_favs_in_system, 40); $this->logger->logInfo("in testAddRmOldFavMaintSearch, second traversal", __METHOD__.','.__LINE__ ); // now add an additional older fav , remove one, and traverse again $this->api->available_api_calls_for_crawler = 3; $this->instance->last_unfav_page_checked = 2; $this->api->to->setDataPath('webapp/plugins/twitter/tests/testdata/favs_tests/favs_stage6/'); $tc = new TwitterCrawler($this->instance, $this->api); $tc->fetchInstanceUserInfo(); $retval = $tc->cleanUpMissedFavsUnFavs(); $this->assertEqual($retval, true); // Save instance if (isset($tc->user)) { $id->save($this->instance, $tc->user->post_count, $this->logger); } $this->instance = $id->getByUsernameOnNetwork("amygdala", "twitter"); // check fav count- should have removed 2 and added 21... // update: due to issue with TwitterAPI, not currently removing un-favs from database // $this->assertEqual($this->instance->owner_favs_in_system, 59); $this->assertEqual($this->instance->owner_favs_in_system, 61); $builder2 = null; $builder3 = null; }
private function apiRequest($method, $encoded_params) { $api_call = $this->api_url . $method . implode('&', $encoded_params); //Don't log this for privacy reasons; it will output the API key to the log //$this->logger->logInfo("Bit.ly API call: $api_call", __METHOD__.','.__LINE__); $resp = Utils::getURLContents($api_call); //$this->logger->logInfo("Bit.ly API call response: ".$resp, __METHOD__.','.__LINE__); if ($resp != false) { $bit_link = json_decode($resp, true); return $bit_link; } else { $this->logger->logInfo("ERROR: No response from Bit.ly API", __METHOD__ . ',' . __LINE__); return array("expanded_url" => '', "title" => '', "clicks" => '', "error" => 'No response from Bit.ly API'); } }
/** * Store links. * @param arr $links * @return int Total links stored */ private function storeLinks($links) { $total_links_added = 0; $link_dao = DAOFactory::getDAO('LinkDAO'); foreach ($links as $link) { try { $added_links = $link_dao->insert($link); $total_links_added = $total_links_added + ($added_links ? 1 : 0); } catch (DuplicateLinkException $e) { $this->logger->logInfo($link->url . " already exists in links table", __METHOD__ . ',' . __LINE__); } catch (DataExceedsColumnWidthException $e) { $this->logger->logInfo($link->url . " data exceeds table column width", __METHOD__ . ',' . __LINE__); } } return $total_links_added; }
private function storePostsAndAuthors($posts, $posts_source) { $total_added_posts = 0; $added_post = 0; foreach ($posts as $post) { $added_post_key = $this->storePostAndAuthor($post, $posts_source); if ($added_post !== false) { $this->logger->logInfo("Added post ID " . $post["post_id"] . " on " . $post["network"] . " for " . $post["author_username"] . ":" . substr($post["post_text"], 0, 20) . "...", __METHOD__ . ',' . __LINE__); $total_added_posts = $total_added_posts++; } else { $this->logger->logInfo("Didn't add post " . $post["post_id"] . " on " . $post["network"] . " for " . $post["author_username"] . ":" . substr($post["post_text"], 0, 20) . "...", __METHOD__ . ',' . __LINE__); } $added_post = 0; } return $total_added_posts; }
/** * If link is an image (Twitpic/Twitgoo/Yfrog/Flickr for now), insert direct path to thumb as expanded url. * @TODO Move image thumbnail processng to Expand URLs plugin. * @param Logger $logger * @param str $tweet * @param Array $urls */ public static function processTweetURLs($logger, $tweet, $urls = null) { $link_dao = DAOFactory::getDAO('LinkDAO'); if (!$urls) { $urls = Post::extractURLs($tweet['post_text']); } foreach ($urls as $u) { $logger->logInfo("processing url: {$u}", __METHOD__ . ',' . __LINE__); $is_image = 0; $title = ''; $eurl = ''; if (substr($u, 0, strlen('http://twitpic.com/')) == 'http://twitpic.com/') { $eurl = 'http://twitpic.com/show/thumb/' . substr($u, strlen('http://twitpic.com/')); $is_image = 1; } elseif (substr($u, 0, strlen('http://yfrog.com/')) == 'http://yfrog.com/') { $eurl = $u . '.th.jpg'; $is_image = 1; } elseif (substr($u, 0, strlen('http://twitgoo.com/')) == 'http://twitgoo.com/') { $eurl = 'http://twitgoo.com/show/thumb/' . substr($u, strlen('http://twitgoo.com/')); $is_image = 1; } elseif (substr($u, 0, strlen('http://picplz.com/')) == 'http://picplz.com/') { $eurl = $u . '/thumb/'; $is_image = 1; } elseif (substr($u, 0, strlen('http://flic.kr/')) == 'http://flic.kr/') { $is_image = 1; } elseif (substr($u, 0, strlen('http://instagr.am/')) == 'http://instagr.am/') { // see: http://instagr.am/developer/embedding/ for reference // the following does a redirect to the actual jpg // make a check for an end slash in the url -- if it is there (likely) then adding a second // slash prior to the 'media' string will break the expanded url if ($u[strlen($u) - 1] == '/') { $eurl = $u . 'media/'; } else { $eurl = $u . '/media/'; } $logger->logDebug("expanded instagram URL to: " . $eurl, __METHOD__ . ',' . __LINE__); $is_image = 1; } if ($link_dao->insert($u, $eurl, $title, $tweet['post_id'], 'twitter', $is_image)) { $logger->logSuccess("Inserted " . $u . " (" . $eurl . ", " . $is_image . "), into links table", __METHOD__ . ',' . __LINE__); } else { $logger->logError("Did NOT insert " . $u . " (" . $eurl . ") into links table", __METHOD__ . ',' . __LINE__); } } }
/** * Expand Bit.ly links and recheck click count on any links less than 2 days old. * * @param str bitly api key * @param str bitly login name */ public function acquireBitlyClickStats($api_key, $bit_login) { $this->logger->setUsername(null); $api_accessor = new BitlyAPIAccessor($api_key, $bit_login); $bitly_urls = array('http://bit.ly/', 'http://bitly.com/', 'http://j.mp/'); foreach ($bitly_urls as $bitly_url) { if ($this->link_limit != 0) { //all short links first seen in the last 48 hours $bitly_links_to_update = $this->short_link_dao->getLinksToUpdate($bitly_url); if (count($bitly_links_to_update) > 0) { $this->logger->logUserInfo(count($bitly_links_to_update) . " {$bitly_url}" . " links to acquire click stats for.", __METHOD__ . ',' . __LINE__); } else { $this->logger->logUserInfo("There are no " . $bitly_url . " links to fetch click stats for.", __METHOD__ . ',', __LINE__); } $total_links = 0; $total_errors = 0; $total_updated = 0; foreach ($bitly_links_to_update as $link) { $this->logger->logInfo("Getting bit.ly click stats for " . ($total_updated + 1) . " of " . count($bitly_links_to_update) . " " . $bitly_url . " links (" . $link->short_url . ")", __METHOD__ . ',' . __LINE__); $link_data = $api_accessor->getBitlyLinkData($link->short_url); if ($link_data["clicks"] != '') { //save click total here $this->short_link_dao->saveClickCount($link->short_url, $link_data["clicks"]); // Save title to links table if ($link_data["title"] != '') { $this->link_dao->updateTitle($link->link_id, $link_data["title"]); } $total_links = $total_links + 1; $total_updated = $total_updated + 1; } elseif ($link_data["error"] != '') { $this->link_dao->saveExpansionError($link->short_url, $link_data["error"]); $total_errors = $total_errors + 1; $total_updated = $total_updated + 1; } } $this->logger->logUserSuccess($total_links . " " . $bitly_url . " link click stats acquired (" . $total_errors . " errors)", __METHOD__ . ',' . __LINE__); } } }
/** * For a given post, extract URLs and store them, including image_src if that's from a known source like Twitpic, * Twitgoo, Yfrog, Instagr.am. * @param str $post_text * @param int $post_id * @param str $network * @param Logger $logger * @param arr $urls Array of URLs, optionally set, defaults to null */ public static function processPostURLs($post_text, $post_id, $network, $logger, $urls = null) { if (!$urls) { $urls = Post::extractURLs($post_text); } if ($urls) { $link_dao = DAOFactory::getDAO('LinkDAO'); foreach ($urls as $url) { $logger->logInfo("Processing URL: {$url}", __METHOD__ . ',' . __LINE__); $image_src = self::getImageSource($url); //if we have an image_src, the URL is a known image source not in need of expansion $expanded_url = isset($image_src) ? $url : ''; $link_array = array('url' => $url, 'expanded_url' => $expanded_url, "image_src" => $image_src, 'post_id' => $post_id, 'network' => $network); $link = new Link($link_array); if ($link_dao->insert($link)) { $logger->logSuccess("Inserted " . $url . " (thumbnail " . $image_src . "), into links table", __METHOD__ . ',' . __LINE__); } else { $logger->logError("Did NOT insert " . $url . " (thumbnail " . $image_src . ") into links table", __METHOD__ . ',' . __LINE__); } } } }
private function apiRequest($method, $encoded_params) { $api_call = $this->api_url . $method . implode('&', $encoded_params); $this->logger->logInfo("Bit.ly API call: {$api_call}", __METHOD__ . ',' . __LINE__); //$resp = Utils::getURLContents($api_call); $api_call = str_replace('http://', '', $api_call); $api_call = str_replace('/', '_', $api_call); $api_call = str_replace('?', '-', $api_call); $api_call = str_replace('&', '-', $api_call); if ($this->debug) { echo "READING LOCAL DATA FILE: " . $this->test_data_path . $api_call . "\n"; } if (file_exists($this->test_data_path . $api_call)) { $resp = file_get_contents($this->test_data_path . $api_call); $bit_link = json_decode($resp, true); $this->logger->logInfo("API RESPONSE DECODED: " . Utils::varDumpToString($bit_link), __METHOD__ . ',' . __LINE__); return $bit_link; } else { $this->logger->logInfo("ERROR: No response from Bit.ly API", __METHOD__ . ',' . __LINE__); return array("expanded_url" => '', "title" => '', "clicks" => '', "error" => 'No response from Bit.ly API'); } }
/** * @param $content * @return array */ private function parsePost($content) { $logger = Logger::getInstance('stream_log_location'); $rt_string = "RT @"; $post = array(); $mentions = array(); $urls = array(); $hashtags = array(); $entities = array(); $user_array = array(); try { $post['is_rt'] = false; $post['in_rt_of_user_id'] = ''; $user = $content['user']; // parse info into user and post arrays $post['post_id'] = $content['id_str']; $post['author_user_id'] = $user['id_str']; $post['author_username'] = $user['screen_name']; $post['author_fullname'] = $user['name']; $post['author_avatar'] = $user['profile_image_url']; $post['author_follower_count'] = $user['followers_count']; $post['post_text'] = $content['text']; $post['is_protected'] = $user['protected']; $post['source'] = $content['source']; $post['location'] = $user['location']; $post['description'] = $user['description']; $post['url'] = $user['url']; $post['author_friends_count'] = $user['friends_count']; $post['author_post_count'] = $user['statuses_count']; $post['author_joined'] = gmdate("Y-m-d H:i:s", strToTime($user['created_at'])); $post['favorited'] = $content['favorited']; $user_array['url'] = $user['url']; // for now, retain existing 'place' handling, where a place is set in the post. // Set new place_id field as well, and add point coord information if it exists $logger->logDebug("point coords: " . print_r($content['coordinates'], true), __METHOD__ . ',' . __LINE__); $place = $content['place']; if ($place != null) { $post['place'] = $place['full_name']; $post['place_id'] = $place['id']; if (isset($content['coordinates'])) { $place['point_coords'] = $content['coordinates']; } } else { $post['place'] = null; $post['place_id'] = null; // it's possible for the point coords to be set even if the place is not. if (isset($content['coordinates'])) { $place = array(); $place['point_coords'] = $content['coordinates']; } } $post['pub_date'] = gmdate("Y-m-d H:i:s", strToTime($content['created_at'])); $post['in_reply_to_user_id'] = $content['in_reply_to_user_id_str']; $post['in_reply_to_post_id'] = $content['in_reply_to_status_id_str']; $post['network'] = 'twitter'; $post['reply_count_cache'] = 0; if (isset($content['entities'])) { foreach ($content['entities']['user_mentions'] as $m) { $mention_info = array(); $mention_info['user_id'] = $m['id_str']; $mention_info['user_name'] = $m['screen_name']; $mentions[] = $mention_info; } // get urls foreach ($content['entities']['urls'] as $u) { // This block broken under 0.11 /* $url_info = array(); $url_info['url']= $u['url']; if (isset($u['expanded_url'])) { $url_info['expanded_url'] = $u['expanded_url']; print "expanded url for: " . $url_info['url'] . ": " . $url_info['expanded_url'] . "\n"; } else { $url_info['expanded_url'] = ''; } $urls[] = $url_info; */ // just need an array of urls now... if (isset($u['expanded_url'])) { array_push($urls, $u['expanded_url']); } else { if (isset($u['url'])) { array_push($urls, $u['url']); } } } // get hashtags foreach ($content['entities']['hashtags'] as $h) { $hashtags[] = $h['text']; } } $logger->logDebug($post['post_text'] . " -- " . $post['author_username'], __METHOD__ . ',' . __LINE__); if (!isset($content['retweeted_status'])) { if (isset($content['retweet_count'])) { // do this only for the original post (rt will have rt count too) $retweet_count_api = $content['retweet_count']; $pos = strrpos($content['retweet_count'], '+'); if ($pos != false) { // remove '+', e.g. '100+' -- so currently 100 is max that can be indicated $retweet_count_api = substr($content['retweet_count'], 0, $pos); } $post['retweet_count_api'] = $retweet_count_api; $this->logger->logDebug($content['id_str'] . " is not a retweet but orig., count is: " . $content['retweet_count'] . "/ " . $retweet_count_api, __METHOD__ . ',' . __LINE__); } // // parse to see if 'old-style' retweet "RT @..." for first 'mention'-- if so, set that information if (sizeof($mentions) > 0) { $first_mention = $mentions[0]['user_name']; $logger->logDebug("first mention: {$first_mention}", __METHOD__ . ',' . __LINE__); if (RetweetDetector::isRetweet($post['post_text'], $first_mention)) { $post['is_rt'] = true; $post['in_rt_of_user_id'] = $mentions[0]['user_id']; $logger->logDebug("detected retweet of: " . $post['in_rt_of_user_id'] . ", " . $first_mention, __METHOD__ . ',' . __LINE__); } } } else { // then this is a retweet. // Process its original too. $this->logger->logInfo("this is a retweet, will first process original post " . $content['retweeted_status']['id_str'] . " from user " . $content['retweeted_status']['user']['id_str'], __METHOD__ . ',' . __LINE__); list($orig_post, $orig_entities, $orig_user_array) = $this->parsePost($content['retweeted_status']); $rtp = array(); $rtp['content'] = $orig_post; $rtp['entities'] = $orig_entities; $rtp['user_array'] = $orig_user_array; $post['retweeted_post'] = $rtp; $post['in_retweet_of_post_id'] = $content['retweeted_status']['id_str']; $post['in_rt_of_user_id'] = $content['retweeted_status']['user']['id_str']; } $user_array = $this->parseUser($user, $post['pub_date']); } catch (Exception $e) { $logger->logErrro("exception: {$e}", __METHOD__ . ',' . __LINE__); } $entities['urls'] = $urls; $entities['mentions'] = $mentions; $entities['hashtags'] = $hashtags; $entities['place'] = $place; // add 'place' object to entities array; may be null return array($post, $entities, $user_array); }
function usunOpisyDlaZdjecia($katalog, $nazwaPlikuZdjecia) { $zapytanie = "DELETE FROM komentarzDoZdjecia WHERE jestOpisem = 't' AND katalog = '$katalog' AND nazwaPlikuZdjecia = '$nazwaPlikuZdjecia'"; $wynik = mysql_query($zapytanie, $this->polaczenie); if (!$wynik) { Logger::logMysqlError("MysqlKomentarzDoZdjecia->usunOpisyDlaZdjecia", $zapytanie, mysql_error($this->polaczenie)); return false; } else { Logger::logInfo($zapytanie); return true; } }
/** * Retrieve tweets in search results for a keyword/hashtag. * @param InstanceHashtag $instance_hashtag * @return void */ public function fetchInstanceHashtagTweets($instance_hashtag) { if (isset($this->instance)) { $status_message = ""; $continue_fetching = true; $since_id = 0; $max_id = 0; $instance_hashtag_dao = DAOFactory::getDAO('InstanceHashtagDAO'); $post_dao = DAOFactory::getDAO('PostDAO'); $user_dao = DAOFactory::getDAO('UserDAO'); $hashtagpost_dao = DAOFactory::getDAO('HashtagPostDAO'); $hashtag_dao = DAOFactory::getDAO('HashtagDAO'); //Get hashtag $hashtag = $hashtag_dao->getHashtagByID($instance_hashtag->hashtag_id); while ($continue_fetching) { $endpoint = $this->api->endpoints['search_tweets']; $args = array(); $args["q"] = $hashtag->hashtag; $count_arg = isset($this->twitter_options['tweet_count_per_call']) ? $this->twitter_options['tweet_count_per_call']->option_value : 100; $args["count"] = $count_arg; $args["include_entities"] = "true"; if ($since_id == 0) { $since_id = $instance_hashtag->last_post_id; } if ($since_id > 0) { $args["since_id"] = $since_id; } if ($max_id > $since_id) { $args["max_id"] = $max_id; } try { list($http_status, $payload) = $this->api->apiRequest($endpoint, $args); } catch (APICallLimitExceededException $e) { $this->logger->logInfo($e->getMessage(), __METHOD__ . ',' . __LINE__); break; } if ($http_status == 200) { $this->logger->logDebug('Search tweets 200 ' . $endpoint->getPath(), __METHOD__ . ',' . __LINE__); $count = 0; $user_count = 0; $tweets = $this->api->parseJSONTweetsFromSearch($payload); foreach ($tweets as $tweet) { $this->logger->logDebug('Processing ' . $tweet['post_id'], __METHOD__ . ',' . __LINE__); $this->logger->logDebug('Processing ' . Utils::varDumpToString($tweet), __METHOD__ . ',' . __LINE__); $inserted_post_key = $post_dao->addPost($tweet, $this->user, $this->logger); //We need to check if post exists before add relationship between post and hashtag if ($post_dao->isPostInDB($tweet['post_id'], 'twitter')) { if (!$hashtagpost_dao->isHashtagPostInStorage($hashtag->id, $tweet['post_id'], 'twitter')) { $count = $count + 1; $hashtagpost_dao->insertHashtagPost($hashtag->hashtag, $tweet['post_id'], 'twitter'); $user = new User($tweet); $rows_updated = $user_dao->updateUser($user); if ($rows_updated > 0) { $user_count = $user_count + $rows_updated; } $this->logger->logDebug('User has been updated', __METHOD__ . ',' . __LINE__); if (isset($tweet['retweeted_post']) && isset($tweet['retweeted_post']['content'])) { $this->logger->logDebug('Retweeted post info set', __METHOD__ . ',' . __LINE__); if (!$hashtagpost_dao->isHashtagPostInStorage($hashtag->id, $tweet['retweeted_post']['content']['post_id'], 'twitter')) { $this->logger->logDebug('Retweeted post not in storage', __METHOD__ . ',' . __LINE__); $count++; $hashtagpost_dao->insertHashtagPost($hashtag->hashtag, $tweet['retweeted_post']['content']['post_id'], 'twitter'); $user_retweet = new User($tweet['retweeted_post']['content']); $rows_retweet_updated = $user_dao->updateUser($user_retweet); if ($rows_retweet_updated > 0) { $user_count = $user_count + $rows_retweet_updated; } } else { $this->logger->logDebug('Retweeted post in storage', __METHOD__ . ',' . __LINE__); } } else { $this->logger->logDebug('Retweeted post info not set', __METHOD__ . ',' . __LINE__); } $this->logger->logDebug('About to process URLs', __METHOD__ . ',' . __LINE__); URLProcessor::processPostURLs($tweet['post_text'], $tweet['post_id'], 'twitter', $this->logger); $this->logger->logDebug('URLs have been processed', __METHOD__ . ',' . __LINE__); } } if ($tweet['post_id'] > $instance_hashtag->last_post_id) { $instance_hashtag->last_post_id = $tweet['post_id']; } if ($instance_hashtag->earliest_post_id == 0 || $tweet['post_id'] < $instance_hashtag->earliest_post_id) { $instance_hashtag->earliest_post_id = $tweet['post_id']; } if ($max_id == 0 || $tweet['post_id'] < $max_id) { $max_id = $tweet['post_id']; } $this->logger->logDebug('Instance hashtag markers updated', __METHOD__ . ',' . __LINE__); } //Status message for tweets and users $status_message = ' ' . count($tweets) . " tweet(s) found and {$count} saved"; $this->logger->logUserSuccess($status_message, __METHOD__ . ',' . __LINE__); $status_message = ' ' . count($tweets) . " tweet(s) found and {$user_count} users saved"; $this->logger->logUserSuccess($status_message, __METHOD__ . ',' . __LINE__); //Save instance_hashtag important values if ($instance_hashtag->last_post_id > 0) { $instance_hashtag_dao->updateLastPostID($instance_hashtag->instance_id, $instance_hashtag->hashtag_id, $instance_hashtag->last_post_id); } if ($instance_hashtag->earliest_post_id > 0) { $instance_hashtag_dao->updateEarliestPostID($instance_hashtag->instance_id, $instance_hashtag->hashtag_id, $instance_hashtag->earliest_post_id); } //Not to continue fetching if search not return the maxim number of tweets if (count($tweets) < $count_arg) { $continue_fetching = false; } } else { $status_message = "Stop fetching tweets. cURL_status = " . $cURL_status; $this->logger->logUserSuccess($status_message, __METHOD__ . ',' . __LINE__); $continue_fetching = false; } } } }
/** *V4.0 获取商户结果通知平台请求商户数据 * @param $map 请求数据 * @return HashMap */ public static function getSplitMerRefundNotifyReqData($map) { $log = new Logger(); if ($map == null || $map->size() == 0) { die("获取通知数据失败:待解析的数据对象为空!"); } $plain = self::getSplitMerRefundNotifyPlain($map); $plain = iconv("UTF-8", "GBK", $plain); $log->logInfo("getSplitMerRefundNotifyReqData plain=[" . $plain . "]"); $sign = $map->get("sign"); $log->logInfo("getSplitMerRefundNotifyReqData sign=" . $sign); //进行请求数据验签 $checked = SignUtil::verify($plain, $sign); if (!$checked) { die("支付结果通知平台请求数据验签失败!"); } return $map; }
public function onFinish($serv, $task_id, $data) { Logger::logInfo("tasking_num: [" . $serv->stats()['tasking_num'] . "] pid: [" . posix_getpid() . "] task_id: [" . $task_id . "] message: [" . $data . "]"); }
function usun($katalog) { $zapytanie = "DELETE FROM galeriaZdjec WHERE katalog = '$katalog'"; $wynik = mysql_query($zapytanie, $this->polaczenie); if (!$wynik) { Logger::logMysqlError("MysqlGaleriaZdjec->usun", $zapytanie, mysql_error($this->polaczenie)); return false; } else { Logger::logInfo($zapytanie); return true; } }
/** * 真正的校验逻辑,通过配置的正则表达式校验字段是否符合规范 * * @param key * 待校验字段 * @param dc * @param must * 字段是否是必须的 * @throws Exception */ protected static function check($key, $map, $must) { $log = new Logger(); $value = $map->get($key); $tValue = $value; if (empty($value)) { // 必须字段为空,抛出异常 if ($must) { die("The parameters of the request " . $key . " can't be empty"); } } else { if (count(CheckReqDataAndEncrypt::$regexkeysarray) <= 0) { $log->logInfo("loading mapping_regex.ini"); CheckReqDataAndEncrypt::$regexkeysarray = parse_ini_file("mapping_regex.ini"); } $regex = CheckReqDataAndEncrypt::$regexkeysarray["Regex." . $key]; if (empty($regex)) { $log->logInfo("The parameters of the request " . $key . " not configured regular validation rules"); return; } // 不符合正则表达式 if (!preg_match($regex, $tValue)) { $log->logInfo("Request parameters [" . $key . "(" . $tValue . ")] format or value can not match the regular rules [" . $regex . "]"); die("Request parameters [" . $key . "(" . $tValue . ")] format or value is not correct"); } else { $log->logInfo("Request parameters [" . $key . "(" . $tValue . ")] format or value match the regular rules [" . $regex . "]"); } } }
/** * cleanUpMissedFavsUnFavs pages back through the older pages of favs, checking for favs that are not yet in * the database, as well as favs that were added to the db but are no longer returned by Twitter's API. * However, that latter calculation, for un-fav'd tweets, is currently not reliable due to a bug on Twitter's end, * and so such tweets are not currently removed from the database. * Due to the same issue with the API, it's not clear whether all favs of older tweets are going to be actually * returned from Twitter (that is, it is currently not returning some actually-favorited tweets in a given range). * So, we may miss some older tweets that were in fact favorited, until Twitter fixes this. * The number of pages to page back for each run of the crawler is set by favs_cleanup_pages option. */ public function cleanUpMissedFavsUnFavs() { // first, check that we have the resources to do work if (!($this->api->available && $this->api->available_api_calls_for_crawler)) { $this->logger->logInfo("terminating cleanUpMissedFavsUnFavs-- no API calls available", __METHOD__ . ',' . __LINE__); return true; } $this->logger->logInfo("In cleanUpMissedFavsUnFavs", __METHOD__ . ',' . __LINE__); $this->logger->logInfo("User id: " . $this->user->user_id . "\n", __METHOD__ . ',' . __LINE__); $fcount = 0; $favs_cleanup_pages = 1; // default number of pages to process each time the crawler runs // get plugin option value if it exists & is positive int, otherwise use default $topt = $this->twitter_options; if (isset($topt['favs_cleanup_pages'])) { $conf_favs_cleanup_pages = $topt['favs_cleanup_pages']->option_value; $this->logger->logInfo("conf_favs_cleanup_pages: {$conf_favs_cleanup_pages} ", __METHOD__ . ',' . __LINE__); if (is_integer((int) $conf_favs_cleanup_pages) && $conf_favs_cleanup_pages > 0) { $favs_cleanup_pages = $conf_favs_cleanup_pages; } } $this->logger->logInfo("favs_cleanup_pages: {$favs_cleanup_pages} ", __METHOD__ . ',' . __LINE__); $fpd = DAOFactory::getDAO('FavoritePostDAO'); $pagesize = 20; // number of favs per page retrieved from the API call... (tbd: any way to get //this from the API?) // get 'favs_older_pages' plugin option value if it exists & is pos. int. Use it to calculate default start // page if set, otherwise use default value. $default_start_page = 2; $topt = $this->twitter_options; if (isset($topt['favs_older_pages'])) { $conf_older_favs_pages = $topt['favs_older_pages']->option_value; if (is_integer((int) $conf_older_favs_pages) && $conf_older_favs_pages > 0) { $default_start_page = $conf_older_favs_pages + 1; } } $this->logger->logInfo("default start page: {$default_start_page} ", __METHOD__ . ',' . __LINE__); $last_page_of_favs = round($this->api->archive_limit / $pagesize); $last_unfav_page_checked = $this->instance->last_unfav_page_checked; $start_page = $last_unfav_page_checked > 0 ? $last_unfav_page_checked + 1 : $default_start_page; $this->logger->logInfo("start page: {$start_page}, with {$favs_cleanup_pages} cleanup pages", __METHOD__ . ',' . __LINE__); $curr_favs_count = $this->user->favorites_count; $count = 0; $page = $start_page; while ($count < $favs_cleanup_pages && $this->api->available && $this->api->available_api_calls_for_crawler) { // get the favs from that page try { list($tweets, $cURL_status, $twitter_data) = $this->getFavsPage($page); } catch (APICallLimitExceededException $e) { break; } if ($cURL_status != 200 || $tweets == -1) { // todo - handle more informatively $this->logger->logInfo("in cleanUpMissedFavsUnFavs, error with: {$twitter_data}", __METHOD__ . ',' . __LINE__); throw new Exception("in cleanUpUnFavs: error parsing favs"); } if (sizeof($tweets) == 0) { // then done paging backwards through the favs. // reset pointer so that we start at the recent favs again next time through. $this->instance->last_unfav_page_checked = 0; break; } $min_tweet = $tweets[sizeof($tweets) - 1]['post_id']; $max_tweet = $tweets[0]['post_id']; $this->logger->logInfo("in cleanUpUnFavs, page {$page} min and max: {$min_tweet}, {$max_tweet}", __METHOD__ . ',' . __LINE__); foreach ($tweets as $fav) { $fav['network'] = 'twitter'; // check whether the tweet is in the db-- if not, add it. if ($fpd->addFavorite($this->user->user_id, $fav) > 0) { URLProcessor::processPostURLs($fav['post_text'], $fav['post_id'], 'twitter', $this->logger); $this->logger->logInfo("added fav " . $fav['post_id'], __METHOD__ . ',' . __LINE__); $fcount++; } else { $status_message = "have already stored fav " . $fav['post_id']; $this->logger->logDebug($status_message, __METHOD__ . ',' . __LINE__); } } // now for each favorited tweet in the database within the fetched range, check whether it's still // favorited. This part of the method is currently disabled due to issues with the Twitter API, which // is not returning all of the favorited tweets any more. So, the fact that a previously-archived // tweet is not returned, no longer indicates that it was un-fav'd. // The method still IDs the 'missing' tweets, but no longer deletes them. We may want to get rid of // this check altogether at some point. $fposts = $fpd->getAllFavoritePostsUpperBound($this->user->user_id, 'twitter', $pagesize, $max_tweet + 1); foreach ($fposts as $old_fav) { $old_fav_id = $old_fav->post_id; if ($old_fav_id < $min_tweet) { $this->logger->logInfo("Old fav {$old_fav_id} out of range ", __METHOD__ . ',' . __LINE__); break; // all the rest will be out of range also then } // look for the old_fav_id in the array of fetched favs $found = false; foreach ($tweets as $tweet) { if ($old_fav_id == $tweet['post_id']) { $found = true; break; } } if (!$found) { // if it's not there... // 14/10 arghh -- Twitter is suddenly (temporarily?) not returning all fav'd tweets in a // sequence. // skipping the delete for now, keep tabs on it. Can check before delete with extra API // request, but the point of doing it this way was to avoid the additional API request. $this->logger->logInfo("Twitter claims tweet not still favorited, but this is currently " . "broken, so not deleting: " . $old_fav_id, __METHOD__ . ',' . __LINE__); // 'unfavorite' by removing from favorites table // $fpd->unFavorite($old_fav_id, $this->user->user_id); } } $this->instance->last_unfav_page_checked = $page++; if ($page > $last_page_of_favs) { $page = 0; break; } $count++; } $this->logger->logUserSuccess("Added {$fcount} older missed favorites", __METHOD__ . ',' . __LINE__); return true; }
/** * Parse the JSON returned from foursquare and store them in the database * @param JSON $checkins */ private function parseResults($checkins) { $post_dao = DAOFactory::getDAO('PostDAO'); $place_dao = DAOFactory::getDAO('PlaceDAO'); $link_dao = DAOFactory::getDAO('LinkDAO'); $number_stored = 0; // Check we actually got a set of checkins if (isset($checkins->response->checkins->items) && sizeof($checkins->response->checkins->items) > 0) { // Make a query out to the API for this users details, like name etc. $user = $this->api_accessor->apiRequest('users/self', $this->access_token); // For each checkin store it in the database foreach ($checkins->response->checkins->items as $item) { // The post ID, is the checkin ID foursquare provides $post['post_id'] = $item->id; // The post text is the text they enter when checking in $post['post_text'] = isset($item->shout) ? $item->shout : " "; // The author username is the users foursquare email address $post['author_username'] = $user->response->user->contact->email; // The author full name is the name they gave foursquare $post['author_fullname'] = $user->response->user->firstName . " " . $user->response->user->lastName; // The avatar is the one they have set on foursquare $post["author_avatar"] = $user->response->user->photo->prefix . "100x100" . $user->response->user->photo->suffix; // The author user id is there foursquare user ID $post['author_user_id'] = $user->response->user->id; // The date they checked in $post['pub_date'] = date('Y-m-d H:i:s', $item->createdAt); // Source of the checkin $post['source'] = $item->source->name; // Check if the checkin was marked as private if (isset($item->private)) { $post['is_protected'] = true; } else { $post['is_protected'] = false; } // Set the network to foursquare $post['network'] = 'foursquare'; // Set place to the name of the place their checking into $post['place'] = $item->venue->name; // There are a few parameters that may or may not be set for location so let the method figure it out $post['location'] = $this->getLocation($item); //$post['location'] = $item->location->postalCode; // Place ID is an ID foursquare provides $post['place_id'] = $item->venue->id; // Set geo to the lat,lng that foursquare provides $post['geo'] = $item->venue->location->lat . "," . $item->venue->location->lng; // These parameters cant be null but we don't need them for our plugin so set them to any empty string $post['reply_count_cache'] = ''; $post['favlike_count_cache'] = ''; $post['retweet_count_cache'] = ''; $post['author_follower_count'] = ''; // Store the checkin details in the database $done = $post_dao->addPost($post); if ($done != null) { $number_stored++; } // Check if any photos are attached to this checkin if ($item->photos->count > 0 && $done != null) { foreach ($item->photos->items as $photo) { $photo_store = new Link(array('url' => $photo->url, 'expanded_url' => $photo->url, 'title' => ' ', 'description' => ' ', 'image_src' => $photo->url, 'caption' => ' ', 'clicks' => 0, 'post_key' => $done, 'error' => 'none')); // Insert the photo into the database $link_dao->insert($photo_store); // Delete the current photo info ready for the next one $photo_store = null; } } // If there are any comments on this checkin capture them if ($item->comments->count > 0) { // Make a query out for the comments $comments = $this->api_accessor->apiRequest('checkins/' . $item->id, $this->access_token); foreach ($comments->response->checkin->comments->items as $comment) { // The post ID, is the comment ID foursquare provides $comment_store['post_id'] = $comment->id; // The post text is the comment they made $comment_store['post_text'] = $comment->text; // The author username is the users foursquare email address (which we need to query for) $name = $this->api_accessor->apiRequest('users/' . $comment->user->id, $this->access_token); $user_name = $name->response->user->contact->email; $comment_store['author_username'] = isset($user_name) ? $user_name : 'email address withheld'; // The author full name is the name they gave foursquare $comment_store['author_fullname'] = $comment->user->firstName . " " . $comment->user->lastName; // The avatar is the one they have set on foursquare $comment_store["author_avatar"] = $comment->user->photo->prefix . "100x100" . $comment->user->photo->suffix; // The author user id is there foursquare user ID $comment_store['author_user_id'] = $comment->user->id; // The date they posted the comment $comment_store['pub_date'] = date('Y-m-d H:i:s', $comment->createdAt); // Source of the comment $comment_store['source'] = ""; // Comments can not be private $comment_store['is_protected'] = false; // Set the network to foursquare $comment_store['network'] = 'foursquare'; // Set place to the name of the place the comment is about $comment_store['place'] = $comments->response->checkin->venue->name; // A few parameters may or may not be set for location so let the method do the work $comment_store['location'] = $this->getLocation($item); // Place ID is an ID foursquare provides $comment_store['place_id'] = $comments->response->checkin->venue->id; // Set geo to the lat,lng that foursquare provides $comment_store['geo'] = $item->venue->location->lat . "," . $item->venue->location->lng; // The ID of the author of the checkin $comment_store['in_reply_to_user_id'] = $user->response->user->id; // The ID of the checkin this is a reply to $comment_store['in_reply_to_post_id'] = $item->id; // The number of replies this checkin has $comment_store['reply_count_cache'] = $item->comments->count; // These parameters cant be null but we don't need them so set them to any empty string $comment_store['reply_count_cache'] = ''; $comment_store['favlike_count_cache'] = ''; $comment_store['retweet_count_cache'] = ''; $comment_store['author_follower_count'] = ''; self::fetchUser($comment_store['author_user_id'], 'comment'); // Now store the comment in the database $post_dao->addPost($comment_store); $comment = null; } } // Store the details about this place in the place table if it doesn't already exist // See if this place is already in the database $place_test = $place_dao->getPlaceByID($item->venue->id); // If it isn't already in the database if ($place_test == null) { // Insert it $places['id'] = $item->venue->id; $places['place_type'] = $item->venue->categories[0]->name; $places['name'] = $item->venue->name; $places['full_name'] = $item->venue->name; $places['icon'] = $item->venue->categories[0]->icon->prefix . '64' . $item->venue->categories[0]->icon->suffix; $places['lat_lng'] = 'POINT(' . $item->venue->location->lat . " " . $item->venue->location->lng . ')'; $places['map_image'] = $this->generateMap($item); $place_dao->insertGenericPlace($places, 'foursquare'); } // Blank out the details ready for the next checkin $post = null; $places = null; } } else { $this->logger->logInfo("No checkins found " . Utils::varDumpToString($checkins)); } }
/** * @param $data */ public function batchUpdate($data = array()) { if ($data && is_array($data)) { try { $docs =& $this->_getBatchBuff('update'); if (count($docs) > self::MAXBATCHNUM) { $c = $this->_getColctObj(); $batch = new MongoUpdateBatch($c); foreach ($docs as $doc) { $batch->add((object) $doc); } $batch->execute(); $docs = array(); } else { if (isset($data['userkey'])) { $update = array('q' => array('userkey' => $data['userkey']), 'u' => array('$set' => $data), 'multi' => false, 'upsert' => true); array_push($docs, $update); } } } catch (MongoException $e) { Logger::logInfo('mongo in lib batchupdate failed, message [' . $e->getMessage() . '] code: [' . $e->getCode() . ']'); return false; } } else { Logger::logInfo('data of batchupdate is invaild'); return false; } }