/** * Fetch a save the posts and replies on a Facebook page. * @param int $pid Page ID */ public function fetchPagePostsAndReplies($pid) { $stream = FacebookGraphAPIAccessor::apiRequest('/'.$pid.'/posts', $this->access_token); if (isset($stream->data) && is_array($stream->data) && sizeof($stream->data > 0)) { $this->logger->logSuccess(sizeof($stream->data)." Facebook posts found for page ID $pid.", __METHOD__.','.__LINE__); $thinkup_data = $this->parseStream($stream, 'facebook page'); $posts = $thinkup_data["posts"]; $post_dao = DAOFactory::getDAO('PostDAO'); $added_posts = 0; foreach ($posts as $post) { if ($post['author_username']== "" && isset($post['author_user_id'])) { $commenter_object = $this->fetchUserInfo($post['author_user_id'], 'facebook', 'Facebook page comments'); if (isset($commenter_object)) { $post["author_username"] = $commenter_object->full_name; $post["author_fullname"] = $commenter_object->full_name; $post["author_avatar"] = $commenter_object->avatar; } } $added_posts = $added_posts + $post_dao->addPost($post); $this->logger->logInfo("Added post ID ".$post["post_id"]." on ".$post["network"]. " for ".$post["author_username"].":".$post["post_text"], __METHOD__.','.__LINE__); } $added_users = 0; $users = $thinkup_data["users"]; if (count($users) > 0) { foreach ($users as $user) { $user["post_count"] = $post_dao->getTotalPostsByUser($user['user_id'], $user['network']); $found_in = 'Facebook page stream'; $user_object = new User($user, $found_in); $user_dao = DAOFactory::getDAO('UserDAO'); $user_dao->updateUser($user_object); $added_users = $added_users + 1; } } if ($added_posts > 0 || $added_users > 0) { $this->logger->logUserSuccess($added_posts." post(s) added; ".$added_users." user(s) updated.", __METHOD__.','.__LINE__); } else { $this->logger->logUserInfo("No new page posts found.", __METHOD__.','.__LINE__); } } else { $this->logger->logInfo("No Facebook posts found for page ID $pid", __METHOD__.','.__LINE__); } }
/** * Expand Bit.ly links and recheck click count on any links less than 2 days old. * * @param str bitly api key * @param str bitly login name */ public function acquireBitlyClickStats($api_key, $bit_login) { $this->logger->setUsername(null); $api_accessor = new BitlyAPIAccessor($api_key, $bit_login); $bitly_urls = array('http://bit.ly/', 'http://bitly.com/', 'http://j.mp/'); foreach ($bitly_urls as $bitly_url) { if ($this->link_limit != 0) { //all short links first seen in the last 48 hours $bitly_links_to_update = $this->short_link_dao->getLinksToUpdate($bitly_url); if (count($bitly_links_to_update) > 0) { $this->logger->logUserInfo(count($bitly_links_to_update) . " {$bitly_url}" . " links to acquire click stats for.", __METHOD__ . ',' . __LINE__); } else { $this->logger->logUserInfo("There are no " . $bitly_url . " links to fetch click stats for.", __METHOD__ . ',', __LINE__); } $total_links = 0; $total_errors = 0; $total_updated = 0; foreach ($bitly_links_to_update as $link) { $this->logger->logInfo("Getting bit.ly click stats for " . ($total_updated + 1) . " of " . count($bitly_links_to_update) . " " . $bitly_url . " links (" . $link->short_url . ")", __METHOD__ . ',' . __LINE__); $link_data = $api_accessor->getBitlyLinkData($link->short_url); if ($link_data["clicks"] != '') { //save click total here $this->short_link_dao->saveClickCount($link->short_url, $link_data["clicks"]); // Save title to links table if ($link_data["title"] != '') { $this->link_dao->updateTitle($link->link_id, $link_data["title"]); } $total_links = $total_links + 1; $total_updated = $total_updated + 1; } elseif ($link_data["error"] != '') { $this->link_dao->saveExpansionError($link->short_url, $link_data["error"]); $total_errors = $total_errors + 1; $total_updated = $total_updated + 1; } } $this->logger->logUserSuccess($total_links . " " . $bitly_url . " link click stats acquired (" . $total_errors . " errors)", __METHOD__ . ',' . __LINE__); } } }
/** * cleanUpMissedFavsUnFavs pages back through the older pages of favs, checking for favs that are not yet in * the database, as well as favs that were added to the db but are no longer returned by Twitter's API. * However, that latter calculation, for un-fav'd tweets, is currently not reliable due to a bug on Twitter's end, * and so such tweets are not currently removed from the database. * Due to the same issue with the API, it's not clear whether all favs of older tweets are going to be actually * returned from Twitter (that is, it is currently not returning some actually-favorited tweets in a given range). * So, we may miss some older tweets that were in fact favorited, until Twitter fixes this. * The number of pages to page back for each run of the crawler is set by favs_cleanup_pages option. */ public function cleanUpMissedFavsUnFavs() { // first, check that we have the resources to do work if (!($this->api->available && $this->api->available_api_calls_for_crawler)) { $this->logger->logInfo("terminating cleanUpMissedFavsUnFavs-- no API calls available", __METHOD__ . ',' . __LINE__); return true; } $this->logger->logInfo("In cleanUpMissedFavsUnFavs", __METHOD__ . ',' . __LINE__); $this->logger->logInfo("User id: " . $this->user->user_id . "\n", __METHOD__ . ',' . __LINE__); $fcount = 0; $favs_cleanup_pages = 1; // default number of pages to process each time the crawler runs // get plugin option value if it exists & is positive int, otherwise use default $topt = $this->twitter_options; if (isset($topt['favs_cleanup_pages'])) { $conf_favs_cleanup_pages = $topt['favs_cleanup_pages']->option_value; $this->logger->logInfo("conf_favs_cleanup_pages: {$conf_favs_cleanup_pages} ", __METHOD__ . ',' . __LINE__); if (is_integer((int) $conf_favs_cleanup_pages) && $conf_favs_cleanup_pages > 0) { $favs_cleanup_pages = $conf_favs_cleanup_pages; } } $this->logger->logInfo("favs_cleanup_pages: {$favs_cleanup_pages} ", __METHOD__ . ',' . __LINE__); $fpd = DAOFactory::getDAO('FavoritePostDAO'); $pagesize = 20; // number of favs per page retrieved from the API call... (tbd: any way to get //this from the API?) // get 'favs_older_pages' plugin option value if it exists & is pos. int. Use it to calculate default start // page if set, otherwise use default value. $default_start_page = 2; $topt = $this->twitter_options; if (isset($topt['favs_older_pages'])) { $conf_older_favs_pages = $topt['favs_older_pages']->option_value; if (is_integer((int) $conf_older_favs_pages) && $conf_older_favs_pages > 0) { $default_start_page = $conf_older_favs_pages + 1; } } $this->logger->logInfo("default start page: {$default_start_page} ", __METHOD__ . ',' . __LINE__); $last_page_of_favs = round($this->api->archive_limit / $pagesize); $last_unfav_page_checked = $this->instance->last_unfav_page_checked; $start_page = $last_unfav_page_checked > 0 ? $last_unfav_page_checked + 1 : $default_start_page; $this->logger->logInfo("start page: {$start_page}, with {$favs_cleanup_pages} cleanup pages", __METHOD__ . ',' . __LINE__); $curr_favs_count = $this->user->favorites_count; $count = 0; $page = $start_page; while ($count < $favs_cleanup_pages && $this->api->available && $this->api->available_api_calls_for_crawler) { // get the favs from that page try { list($tweets, $cURL_status, $twitter_data) = $this->getFavsPage($page); } catch (APICallLimitExceededException $e) { break; } if ($cURL_status != 200 || $tweets == -1) { // todo - handle more informatively $this->logger->logInfo("in cleanUpMissedFavsUnFavs, error with: {$twitter_data}", __METHOD__ . ',' . __LINE__); throw new Exception("in cleanUpUnFavs: error parsing favs"); } if (sizeof($tweets) == 0) { // then done paging backwards through the favs. // reset pointer so that we start at the recent favs again next time through. $this->instance->last_unfav_page_checked = 0; break; } $min_tweet = $tweets[sizeof($tweets) - 1]['post_id']; $max_tweet = $tweets[0]['post_id']; $this->logger->logInfo("in cleanUpUnFavs, page {$page} min and max: {$min_tweet}, {$max_tweet}", __METHOD__ . ',' . __LINE__); foreach ($tweets as $fav) { $fav['network'] = 'twitter'; // check whether the tweet is in the db-- if not, add it. if ($fpd->addFavorite($this->user->user_id, $fav) > 0) { URLProcessor::processPostURLs($fav['post_text'], $fav['post_id'], 'twitter', $this->logger); $this->logger->logInfo("added fav " . $fav['post_id'], __METHOD__ . ',' . __LINE__); $fcount++; } else { $status_message = "have already stored fav " . $fav['post_id']; $this->logger->logDebug($status_message, __METHOD__ . ',' . __LINE__); } } // now for each favorited tweet in the database within the fetched range, check whether it's still // favorited. This part of the method is currently disabled due to issues with the Twitter API, which // is not returning all of the favorited tweets any more. So, the fact that a previously-archived // tweet is not returned, no longer indicates that it was un-fav'd. // The method still IDs the 'missing' tweets, but no longer deletes them. We may want to get rid of // this check altogether at some point. $fposts = $fpd->getAllFavoritePostsUpperBound($this->user->user_id, 'twitter', $pagesize, $max_tweet + 1); foreach ($fposts as $old_fav) { $old_fav_id = $old_fav->post_id; if ($old_fav_id < $min_tweet) { $this->logger->logInfo("Old fav {$old_fav_id} out of range ", __METHOD__ . ',' . __LINE__); break; // all the rest will be out of range also then } // look for the old_fav_id in the array of fetched favs $found = false; foreach ($tweets as $tweet) { if ($old_fav_id == $tweet['post_id']) { $found = true; break; } } if (!$found) { // if it's not there... // 14/10 arghh -- Twitter is suddenly (temporarily?) not returning all fav'd tweets in a // sequence. // skipping the delete for now, keep tabs on it. Can check before delete with extra API // request, but the point of doing it this way was to avoid the additional API request. $this->logger->logInfo("Twitter claims tweet not still favorited, but this is currently " . "broken, so not deleting: " . $old_fav_id, __METHOD__ . ',' . __LINE__); // 'unfavorite' by removing from favorites table // $fpd->unFavorite($old_fav_id, $this->user->user_id); } } $this->instance->last_unfav_page_checked = $page++; if ($page > $last_page_of_favs) { $page = 0; break; } $count++; } $this->logger->logUserSuccess("Added {$fcount} older missed favorites", __METHOD__ . ',' . __LINE__); return true; }
/** * Convert parsed JSON of a profile or page's posts into ThinkUp posts and users * @param Object $stream * @param str $source The network for the post, either 'facebook' or 'facebook page' * @param int Page number being processed * @return int $total_added_posts How many posts (excluding comments) got added to the data store */ private function processStream($stream, $network, $page_number) { $thinkup_posts = array(); $total_added_posts = 0; $total_added_comments = 0; $thinkup_users = array(); $total_added_users = 0; $thinkup_links = array(); $total_links_added = 0; $thinkup_likes = array(); $total_added_likes = 0; $profiles = array(); //efficiency control vars $must_process_likes = true; $must_process_comments = true; $post_comments_added = 0; $post_likes_added = 0; $comments_difference = false; $likes_difference = false; $post_dao = DAOFactory::getDAO('PostDAO'); foreach ($stream->data as $index => $p) { $post_id = explode("_", $p->id); $post_id = $post_id[1]; $this->logger->logInfo("Beginning to process " . $post_id . ", post " . ($index + 1) . " of " . count($stream->data) . " on page " . $page_number, __METHOD__ . ',' . __LINE__); // stream can contain posts from multiple users. get profile for this post $profile = null; if (!empty($profiles[$p->from->id])) { $profile = $profiles[$p->from->id]; } else { $profile = $this->fetchUser($p->from->id, 'Post stream', true); $profiles[$p->from->id] = $profile; } //Assume profile comments are private and page posts are public $is_protected = $network == 'facebook' ? 1 : 0; //Get likes count $likes_count = 0; //Normalize likes to be one array if (isset($p->likes)) { $likes_count = $p->likes->summary->total_count; $p->likes = $this->normalizeLikes($p->likes); } // Normalize comments to be one array $comments_count = 0; if (isset($p->comments)) { $comments_count = $p->comments->summary->total_count; $p->comments = $this->normalizeComments($p->comments); } $post_in_storage = $post_dao->getPost($post_id, $network); //Figure out if we have to process likes and comments if (isset($post_in_storage)) { $this->logger->logInfo("Post " . $post_id . " already in storage", __METHOD__ . ',' . __LINE__); if ($post_in_storage->favlike_count_cache >= $likes_count) { $must_process_likes = false; $this->logger->logInfo("Already have " . $likes_count . " like(s) for post " . $post_id . " in storage; skipping like processing", __METHOD__ . ',' . __LINE__); } else { $likes_difference = $likes_count - $post_in_storage->favlike_count_cache; $this->logger->logInfo($likes_difference . " new like(s) to process for post " . $post_id, __METHOD__ . ',' . __LINE__); } if (isset($p->comments->summary->total_count)) { if ($post_in_storage->reply_count_cache >= $p->comments->summary->total_count) { $must_process_comments = false; $this->logger->logInfo("Already have " . $post_in_storage->reply_count_cache . " comment(s) for post " . $post_id . "; skipping comment processing", __METHOD__ . ',' . __LINE__); } else { $comments_difference = $p->comments->summary->total_count - $post_in_storage->reply_count_cache; $this->logger->logInfo($comments_difference . " new comment(s) of " . $p->comments->summary->total_count . " total to process for post " . $post_id, __METHOD__ . ',' . __LINE__); } } } else { $this->logger->logInfo("Post " . $post_id . " not in storage", __METHOD__ . ',' . __LINE__); } if (!isset($profile)) { $this->logger->logError("No profile set", __METHOD__ . ',' . __LINE__); } else { if (!isset($post_in_storage)) { $this->logger->logInfo("Post " . $post_id . " has " . $comments_count . " comments", __METHOD__ . ',' . __LINE__); $post_to_process = array("post_id" => $post_id, "author_username" => $profile->username, "author_fullname" => $profile->username, "author_avatar" => $profile->avatar, "author_user_id" => $p->from->id, "post_text" => isset($p->message) ? $p->message : '', "pub_date" => $p->created_time, "favlike_count_cache" => $likes_count, "reply_count_cache" => $comments_count, "in_reply_to_user_id" => isset($p->to->data[0]->id) ? $p->to->data[0]->id : '', "in_reply_to_post_id" => '', "source" => '', 'network' => $network, 'is_protected' => $is_protected, 'location' => ''); $new_post_key = $this->storePostAndAuthor($post_to_process, "Owner stream"); if ($new_post_key !== false) { $total_added_posts++; } if (isset($p->source) || isset($p->link)) { // there's a link to store $link_url = isset($p->source) ? $p->source : $p->link; $link = new Link(array("url" => $link_url, "expanded_url" => '', "image_src" => isset($p->picture) ? $p->picture : '', "caption" => isset($p->caption) ? $p->caption : '', "description" => isset($p->description) ? $p->description : '', "title" => isset($p->name) ? $p->name : '', "post_key" => $new_post_key)); array_push($thinkup_links, $link); } $total_links_addded = $total_links_added + $this->storeLinks($thinkup_links); if ($total_links_added > 0) { $this->logger->logUserSuccess("Collected {$total_links_added} new links", __METHOD__ . ',' . __LINE__); } //free up memory $thinkup_links = array(); } else { // post already exists in storage if ($must_process_likes) { //update its like count only $post_dao->updateFavLikeCount($post_id, $network, $likes_count); $this->logger->logInfo("Updated Like count for post " . $post_id . " to " . $likes_count, __METHOD__ . ',' . __LINE__); } } if ($must_process_comments) { if (isset($p->comments)) { $comments_captured = 0; if (isset($p->comments->data)) { $post_comments = $p->comments->data; $post_comments_count = isset($post_comments) ? sizeof($post_comments) : 0; if (is_array($post_comments) && sizeof($post_comments) > 0) { foreach ($post_comments as $c) { if (isset($c->from)) { // Sometimes the id is parent_poster_postId // sometimes it's just parent_postId $comment_id = explode("_", $c->id); if (count($comment_id) == 3) { $comment_id = $comment_id[2]; } else { $comment_id = $comment_id[1]; } //only add to queue if not already in storage $comment_in_storage = $post_dao->getPost($comment_id, $network); if (!isset($comment_in_storage)) { $comment_to_process = array("post_id" => $comment_id, "author_username" => $c->from->name, "author_fullname" => $c->from->name, "author_gender" => $c->from->gender, "author_birthday" => $c->from->birthday, "author_avatar" => 'https://graph.facebook.com/' . $c->from->id . '/picture', "author_user_id" => $c->from->id, "post_text" => $c->message, "pub_date" => $c->created_time, "in_reply_to_user_id" => $profile->user_id, "in_reply_to_post_id" => $post_id, "source" => '', 'network' => $network, 'is_protected' => $is_protected, 'location' => ''); array_push($thinkup_posts, $comment_to_process); $comments_captured = $comments_captured + 1; } } } } } $post_comments_added = $post_comments_added + $this->storePostsAndAuthors($thinkup_posts, "Post stream comments"); //free up memory $thinkup_posts = array(); if (is_int($comments_difference) && $post_comments_added >= $comments_difference) { $must_process_comments = false; if (isset($comments_stream->paging->next)) { $this->logger->logInfo("Caught up on post " . $post_id . "'s balance of " . $comments_difference . " comments; stopping comment processing", __METHOD__ . ',' . __LINE__); } } // collapsed comment thread if (isset($p->comments->summary->total_count) && $p->comments->summary->total_count > $comments_captured && $must_process_comments) { if (is_int($comments_difference)) { $offset = $p->comments->summary->total_count - $comments_difference; $offset_arr = array('offset' => $offset, 'limit' => $comments_difference); } else { $offset_arr = null; } $api_call = $p->from->id . '_' . $post_id . '/comments'; do { $comments_stream = FacebookGraphAPIAccessor::apiRequest($api_call, $this->access_token, $offset_arr); if (isset($comments_stream) && isset($comments_stream->data) && is_array($comments_stream->data)) { foreach ($comments_stream->data as $c) { if (isset($c->from)) { $comment_id = explode("_", $c->id); $comment_id = $comment_id[sizeof($comment_id) - 1]; //only add to queue if not already in storage $comment_in_storage = $post_dao->getPost($comment_id, $network); if (!isset($comment_in_storage)) { $comment_to_process = array("post_id" => $comment_id, "author_username" => $c->from->name, "author_fullname" => $c->from->name, "author_avatar" => 'https://graph.facebook.com/' . $c->from->id . '/picture', "author_user_id" => $c->from->id, "post_text" => $c->message, "pub_date" => $c->created_time, "in_reply_to_user_id" => $profile->user_id, "in_reply_to_post_id" => $post_id, "source" => '', 'network' => $network, 'is_protected' => $is_protected, 'location' => ''); array_push($thinkup_posts, $comment_to_process); } } } $post_comments_added = $post_comments_added + $this->storePostsAndAuthors($thinkup_posts, "Posts stream comments collapsed"); if (is_int($comments_difference) && $post_comments_added >= $comments_difference) { $must_process_comments = false; if (isset($comments_stream->paging->next)) { $this->logger->logInfo("Caught up on post " . $post_id . "'s balance of " . $comments_difference . " comments; stopping comment processing", __METHOD__ . ',' . __LINE__); } } //free up memory $thinkup_posts = array(); if (isset($comments_stream->paging->next)) { $api_call = str_replace('\\u00257C', '|', $comments_stream->paging->next); } } else { // no comments (pun intended) break; } } while (isset($comments_stream->paging->next) && $must_process_comments); } } if ($post_comments_added > 0) { //let user know $this->logger->logUserSuccess("Added " . $post_comments_added . " comment(s) for post " . $post_id, __METHOD__ . ',' . __LINE__); } else { $this->logger->logInfo("Added " . $post_comments_added . " comment(s) for post " . $post_id, __METHOD__ . ',' . __LINE__); } $total_added_comments = $total_added_comments + $post_comments_added; } //Inserting comments also increments the original post's reply_count_cache; reset it here $post_dao->updateReplyCount($post_id, $network, $comments_count); //process "likes" if ($must_process_likes) { if (isset($p->likes)) { $likes_captured = 0; if (isset($p->likes->data)) { $post_likes = $p->likes->data; $post_likes_count = isset($post_likes) ? sizeof($post_likes) : 0; if (is_array($post_likes) && sizeof($post_likes) > 0) { foreach ($post_likes as $l) { if (isset($l->name) && isset($l->id)) { //Get users $user_to_add = array("user_name" => $l->name, "full_name" => $l->name, "user_id" => $l->id, "avatar" => 'https://graph.facebook.com/' . $l->id . '/picture', "location" => '', "description" => '', "url" => '', "is_protected" => 1, "follower_count" => 0, "post_count" => 0, "joined" => '', "found_in" => "Likes", "network" => 'facebook'); //Users are always set to network=facebook array_push($thinkup_users, $user_to_add); $fav_to_add = array("favoriter_id" => $l->id, "network" => $network, "author_user_id" => $profile->user_id, "post_id" => $post_id); array_push($thinkup_likes, $fav_to_add); $likes_captured = $likes_captured + 1; } } } } $total_added_users = $total_added_users + $this->storeUsers($thinkup_users, "Likes"); $post_likes_added = $post_likes_added + $this->storeLikes($thinkup_likes); //free up memory $thinkup_users = array(); $thinkup_likes = array(); if (is_int($likes_difference) && $post_likes_added >= $likes_difference) { $must_process_likes = false; if (isset($likes_stream->paging->next)) { $this->logger->logInfo("Caught up on post " . $post_id . "'s balance of " . $likes_difference . " likes; stopping like processing", __METHOD__ . ',' . __LINE__); } } // collapsed likes if (isset($p->likes->count) && $p->likes->count > $likes_captured && $must_process_likes) { if (is_int($likes_difference)) { $offset = $p->likes->count - $likes_difference; $offset_arr = array('offset' => $offset); } else { $offset_arr = null; } $api_call = $p->from->id . '_' . $post_id . '/likes'; do { $likes_stream = FacebookGraphAPIAccessor::apiRequest($api_call, $this->access_token, $offset_arr); if (isset($likes_stream) && is_array($likes_stream->data)) { foreach ($likes_stream->data as $l) { if (isset($l->name) && isset($l->id)) { //Get users $user_to_add = array("user_name" => $l->name, "full_name" => $l->name, "user_id" => $l->id, "avatar" => 'https://graph.facebook.com/' . $l->id . '/picture', "is_protected" => 1, "location" => '', "description" => '', "url" => '', "follower_count" => 0, "post_count" => 0, "joined" => '', "found_in" => "Likes", "network" => 'facebook'); //Users are always set to network=facebook array_push($thinkup_users, $user_to_add); $fav_to_add = array("favoriter_id" => $l->id, "network" => $network, "author_user_id" => $p->from->id, "post_id" => $post_id); array_push($thinkup_likes, $fav_to_add); $likes_captured = $likes_captured + 1; } } $total_added_users = $total_added_users + $this->storeUsers($thinkup_users, "Likes"); $post_likes_added = $post_likes_added + $this->storeLikes($thinkup_likes); //free up memory $thinkup_users = array(); $thinkup_likes = array(); if (is_int($likes_difference) && $post_likes_added >= $likes_difference) { $must_process_likes = false; if (isset($likes_stream->paging->next)) { $this->logger->logInfo("Caught up on post " . $post_id . "'s balance of " . $likes_difference . " likes; stopping like processing", __METHOD__ . ',' . __LINE__); } } if (isset($likes_stream->paging->next)) { $api_call = str_replace('\\u00257C', '|', $likes_stream->paging->next); } } else { // no likes break; } } while (isset($likes_stream->paging->next) && $must_process_likes); } } $this->logger->logInfo("Added " . $post_likes_added . " like(s) for post " . $post_id, __METHOD__ . ',' . __LINE__); $total_added_likes = $total_added_likes + $post_likes_added; } //free up memory $thinkup_users = array(); $thinkup_likes = array(); } //reset control vars for next post $must_process_likes = true; $must_process_comments = true; $post_comments_added = 0; $post_likes_added = 0; $comments_difference = false; $likes_difference = false; } $this->logger->logUserSuccess("On page " . $page_number . ", captured " . $total_added_posts . " post(s), " . $total_added_comments . " comment(s), " . $total_added_users . " user(s) and " . $total_added_likes . " like(s)", __METHOD__ . ',' . __LINE__); return $total_added_posts; }
/** * Retrieve tweets in search results for a keyword/hashtag. * @param InstanceHashtag $instance_hashtag * @return void */ public function fetchInstanceHashtagTweets($instance_hashtag) { if (isset($this->instance)) { $status_message = ""; $continue_fetching = true; $since_id = 0; $max_id = 0; $instance_hashtag_dao = DAOFactory::getDAO('InstanceHashtagDAO'); $post_dao = DAOFactory::getDAO('PostDAO'); $user_dao = DAOFactory::getDAO('UserDAO'); $hashtagpost_dao = DAOFactory::getDAO('HashtagPostDAO'); $hashtag_dao = DAOFactory::getDAO('HashtagDAO'); //Get hashtag $hashtag = $hashtag_dao->getHashtagByID($instance_hashtag->hashtag_id); while ($continue_fetching) { $endpoint = $this->api->endpoints['search_tweets']; $args = array(); $args["q"] = $hashtag->hashtag; $count_arg = isset($this->twitter_options['tweet_count_per_call']) ? $this->twitter_options['tweet_count_per_call']->option_value : 100; $args["count"] = $count_arg; $args["include_entities"] = "true"; if ($since_id == 0) { $since_id = $instance_hashtag->last_post_id; } if ($since_id > 0) { $args["since_id"] = $since_id; } if ($max_id > $since_id) { $args["max_id"] = $max_id; } try { list($http_status, $payload) = $this->api->apiRequest($endpoint, $args); } catch (APICallLimitExceededException $e) { $this->logger->logInfo($e->getMessage(), __METHOD__ . ',' . __LINE__); break; } if ($http_status == 200) { $this->logger->logDebug('Search tweets 200 ' . $endpoint->getPath(), __METHOD__ . ',' . __LINE__); $count = 0; $user_count = 0; $tweets = $this->api->parseJSONTweetsFromSearch($payload); foreach ($tweets as $tweet) { $this->logger->logDebug('Processing ' . $tweet['post_id'], __METHOD__ . ',' . __LINE__); $this->logger->logDebug('Processing ' . Utils::varDumpToString($tweet), __METHOD__ . ',' . __LINE__); $inserted_post_key = $post_dao->addPost($tweet, $this->user, $this->logger); //We need to check if post exists before add relationship between post and hashtag if ($post_dao->isPostInDB($tweet['post_id'], 'twitter')) { if (!$hashtagpost_dao->isHashtagPostInStorage($hashtag->id, $tweet['post_id'], 'twitter')) { $count = $count + 1; $hashtagpost_dao->insertHashtagPost($hashtag->hashtag, $tweet['post_id'], 'twitter'); $user = new User($tweet); $rows_updated = $user_dao->updateUser($user); if ($rows_updated > 0) { $user_count = $user_count + $rows_updated; } $this->logger->logDebug('User has been updated', __METHOD__ . ',' . __LINE__); if (isset($tweet['retweeted_post']) && isset($tweet['retweeted_post']['content'])) { $this->logger->logDebug('Retweeted post info set', __METHOD__ . ',' . __LINE__); if (!$hashtagpost_dao->isHashtagPostInStorage($hashtag->id, $tweet['retweeted_post']['content']['post_id'], 'twitter')) { $this->logger->logDebug('Retweeted post not in storage', __METHOD__ . ',' . __LINE__); $count++; $hashtagpost_dao->insertHashtagPost($hashtag->hashtag, $tweet['retweeted_post']['content']['post_id'], 'twitter'); $user_retweet = new User($tweet['retweeted_post']['content']); $rows_retweet_updated = $user_dao->updateUser($user_retweet); if ($rows_retweet_updated > 0) { $user_count = $user_count + $rows_retweet_updated; } } else { $this->logger->logDebug('Retweeted post in storage', __METHOD__ . ',' . __LINE__); } } else { $this->logger->logDebug('Retweeted post info not set', __METHOD__ . ',' . __LINE__); } $this->logger->logDebug('About to process URLs', __METHOD__ . ',' . __LINE__); URLProcessor::processPostURLs($tweet['post_text'], $tweet['post_id'], 'twitter', $this->logger); $this->logger->logDebug('URLs have been processed', __METHOD__ . ',' . __LINE__); } } if ($tweet['post_id'] > $instance_hashtag->last_post_id) { $instance_hashtag->last_post_id = $tweet['post_id']; } if ($instance_hashtag->earliest_post_id == 0 || $tweet['post_id'] < $instance_hashtag->earliest_post_id) { $instance_hashtag->earliest_post_id = $tweet['post_id']; } if ($max_id == 0 || $tweet['post_id'] < $max_id) { $max_id = $tweet['post_id']; } $this->logger->logDebug('Instance hashtag markers updated', __METHOD__ . ',' . __LINE__); } //Status message for tweets and users $status_message = ' ' . count($tweets) . " tweet(s) found and {$count} saved"; $this->logger->logUserSuccess($status_message, __METHOD__ . ',' . __LINE__); $status_message = ' ' . count($tweets) . " tweet(s) found and {$user_count} users saved"; $this->logger->logUserSuccess($status_message, __METHOD__ . ',' . __LINE__); //Save instance_hashtag important values if ($instance_hashtag->last_post_id > 0) { $instance_hashtag_dao->updateLastPostID($instance_hashtag->instance_id, $instance_hashtag->hashtag_id, $instance_hashtag->last_post_id); } if ($instance_hashtag->earliest_post_id > 0) { $instance_hashtag_dao->updateEarliestPostID($instance_hashtag->instance_id, $instance_hashtag->hashtag_id, $instance_hashtag->earliest_post_id); } //Not to continue fetching if search not return the maxim number of tweets if (count($tweets) < $count_arg) { $continue_fetching = false; } } else { $status_message = "Stop fetching tweets. cURL_status = " . $cURL_status; $this->logger->logUserSuccess($status_message, __METHOD__ . ',' . __LINE__); $continue_fetching = false; } } } }
/** * Convert a collection of profile posts into ThinkUp posts and users * @param Object $posts * @param str $source The network for the post, always 'instagram' * @param int Page number being processed */ private function processPosts(Instagram\Collection\MediaCollection $posts, $network, $page_number) { $thinkup_posts = array(); $total_added_posts = 0; $thinkup_users = array(); $total_added_users = 0; $thinkup_likes = array(); $total_added_likes = 0; //efficiency control vars $must_process_likes = true; $must_process_comments = true; $post_comments_added = 0; $post_likes_added = 0; $comments_difference = false; $likes_difference = false; $post_dao = DAOFactory::getDAO('PostDAO'); foreach ($posts as $index => $p) { $post_id = $p->getId(); $this->logger->logInfo("Beginning to process " . $post_id . ", post " . ($index + 1) . " of " . count($posts->count()) . " on page " . $page_number, __METHOD__ . ',' . __LINE__); // stream can contain posts from multiple users. get profile for this post $profile = $p->getUser(); $is_protected = 0; //Get likes count $likes_count = $p->getLikesCount(); $comments = $p->getComments(); $post_in_storage = $post_dao->getPost($post_id, $network); //Figure out if we have to process likes and comments if (isset($post_in_storage)) { $this->logger->logInfo("Post " . $post_id . " already in storage", __METHOD__ . ',' . __LINE__); if ($post_in_storage->favlike_count_cache >= $likes_count) { $must_process_likes = false; $this->logger->logInfo("Already have " . $likes_count . " like(s) for post " . $post_id . "in storage; skipping like processing", __METHOD__ . ',' . __LINE__); } else { $likes_difference = $likes_count - $post_in_storage->favlike_count_cache; $this->logger->logInfo($likes_difference . " new like(s) to process for post " . $post_id, __METHOD__ . ',' . __LINE__); } $commentsCount = $comments->count(); if ($commentsCount > 0) { if ($post_in_storage->reply_count_cache >= $commentsCount) { $must_process_comments = false; $this->logger->logInfo("Already have " . $commentsCount . " comment(s) for post " . $post_id . "; skipping comment processing", __METHOD__ . ',' . __LINE__); } else { $comments_difference = $commentsCount - $post_in_storage->reply_count_cache; $this->logger->logInfo($comments_difference . " new comment(s) of " . $commentsCount . " total to process for post " . $post_id, __METHOD__ . ',' . __LINE__); } } } else { $this->logger->logInfo("Post " . $post_id . " not in storage", __METHOD__ . ',' . __LINE__); } // If we dont already have this photo if (!isset($post_in_storage)) { // Photos may be posted without a caption // Note that if you post a photo without a caption and then reply to it with the first comment instagram // treats this as your caption. if (strlen($p->getCaption()) > 0) { $text = $p->getCaption(); } else { $text = ""; } $photo_to_process = array("post_id" => $post_id, "author_username" => $profile->getUserName(), "author_fullname" => $profile->getFullName(), "author_avatar" => $profile->getProfilePicture(), "author_user_id" => $profile->getId(), "post_text" => $text, "pub_date" => DateTime::createFromFormat('U', $p->getCreatedTime())->format('Y-m-d H:i'), "favlike_count_cache" => $likes_count, "in_reply_to_user_id" => '', "in_reply_to_post_id" => '', "source" => '', 'network' => $network, 'is_protected' => $is_protected, 'location' => '', 'permalink' => $p->getLink(), 'standard_resolution_url' => $p->getStandardRes()->url, 'low_resolution_url' => $p->getLowRes()->url, 'thumbnail_url' => $p->getThumbnail()->url, 'filter' => $p->getFilter()); $new_photo_key = $this->storePhotoAndAuthor($photo_to_process, "Owner stream"); if ($new_photo_key !== false) { $total_added_posts++; } } else { // post already exists in storage if ($must_process_likes) { //update its like count only $post_dao->updateFavLikeCount($post_id, $network, $likes_count); $this->logger->logInfo("Updated Like count for post " . $post_id . " to " . $likes_count, __METHOD__ . ',' . __LINE__); } } if ($must_process_comments) { if ($comments->count() > 0) { $comments_captured = 0; $post_comments = $comments; if ($post_comments->count() > 0) { foreach ($post_comments as $c) { $comment_id = $c->getId(); //only add to queue if not already in storage $comment_in_storage = $post_dao->getPost($comment_id, $network); if (!isset($comment_in_storage)) { $comment_author = $c->getUser(); $comment_to_process = array("post_id" => $comment_id, "author_username" => $comment_author->getUserName(), "author_fullname" => $comment_author->getFullName(), "author_avatar" => $comment_author->getProfilePicture(), "author_user_id" => $comment_author->getId(), "post_text" => $c->getText(), "pub_date" => DateTime::createFromFormat('U', $p->getCreatedTime())->format('Y-m-d H:i'), "in_reply_to_user_id" => $profile->getId(), "in_reply_to_post_id" => $post_id, "source" => '', 'network' => $network, 'is_protected' => $is_protected, 'location' => ''); array_push($thinkup_posts, $comment_to_process); $comments_captured = $comments_captured + 1; } } } $post_comments_added = $post_comments_added + $this->storePostsAndAuthors($thinkup_posts, "Post stream comments"); //free up memory $thinkup_posts = array(); if (is_int($comments_difference) && $post_comments_added >= $comments_difference) { $must_process_comments = false; $this->logger->logInfo("Caught up on post " . $post_id . "'s balance of " . $comments_difference . " comments; stopping comment processing", __METHOD__ . ',' . __LINE__); } } if ($post_comments_added > 0) { //let user know $this->logger->logUserSuccess("Added " . $post_comments_added . " comment(s) for post " . $post_id, __METHOD__ . ',' . __LINE__); } else { $this->logger->logInfo("Added " . $post_comments_added . " comment(s) for post " . $post_id, __METHOD__ . ',' . __LINE__); } $total_added_posts = $total_added_posts + $post_comments_added; } //process "likes" if ($must_process_likes) { if ($likes_count > 0) { $likes_captured = 0; $post_likes = $p->getLikes(); $post_likes_count = $likes_count; if ($post_likes_count > 0) { foreach ($post_likes as $l) { //Get users $user_to_add = array("user_name" => $l->getUserName(), "full_name" => $l->getFullName(), "user_id" => $l->getId(), "avatar" => $l->getProfilePicture(), "description" => '', "url" => '', "is_protected" => 0, "follower_count" => 0, "post_count" => 0, "joined" => '', "found_in" => "Likes", "network" => 'instagram'); //Users are always set to network=instagram array_push($thinkup_users, $user_to_add); $fav_to_add = array("favoriter_id" => $l->getId(), "network" => $network, "author_user_id" => $profile->getId(), "post_id" => $post_id); array_push($thinkup_likes, $fav_to_add); $likes_captured = $likes_captured + 1; } } $total_added_users = $total_added_users + $this->calculateNumberOfUsersStored($thinkup_users, "Likes"); $post_likes_added = $post_likes_added + $this->storeLikes($thinkup_likes); //free up memory $thinkup_users = array(); $thinkup_likes = array(); if (is_int($likes_difference) && $post_likes_added >= $likes_difference) { $must_process_likes = false; $this->logger->logInfo("Caught up on post " . $post_id . "'s balance of " . $likes_difference . " likes; stopping like processing", __METHOD__ . ',' . __LINE__); } } $this->logger->logInfo("Added " . $post_likes_added . " like(s) for post " . $post_id, __METHOD__ . ',' . __LINE__); $total_added_likes = $total_added_likes + $post_likes_added; } //free up memory $thinkup_users = array(); $thinkup_likes = array(); //reset control vars for next post $must_process_likes = true; $must_process_comments = true; $post_comments_added = 0; $post_likes_added = 0; $comments_difference = false; $likes_difference = false; } $this->logger->logUserSuccess("On page " . $page_number . ", captured " . $total_added_posts . " post(s), " . $total_added_users . " user(s) and " . $total_added_likes . " like(s)", __METHOD__ . ',' . __LINE__); }
/** * Fetch the instance user's friends' tweets. */ public function fetchInstanceUserFriends() { if (!isset($this->user)) { $this->fetchInstanceUserInfo(); } if (isset($this->user)) { $follow_dao = DAOFactory::getDAO('FollowDAO'); $this->instance->total_friends_in_system = $follow_dao->countTotalFriends($this->instance->network_user_id, 'twitter'); if ($this->instance->total_friends_in_system < $this->user->friend_count) { $this->instance->is_archive_loaded_friends = false; $this->logger->logUserInfo($this->instance->total_friends_in_system . " friends in system, " . $this->user->friend_count . " friends according to Twitter; Friend archive is not loaded", __METHOD__ . ',' . __LINE__); } else { $this->instance->is_archive_loaded_friends = true; $this->logger->logInfo("Friend archive loaded", __METHOD__ . ',' . __LINE__); } $status_message = ""; // Fetch friend pages $continue_fetching = true; $updated_follow_count = 0; $inserted_follow_count = 0; while ($continue_fetching && !$this->instance->is_archive_loaded_friends) { $endpoint = $this->api->endpoints['following']; $args = array(); if (!isset($next_cursor)) { $next_cursor = -1; } $args['cursor'] = strval($next_cursor); $args['skip_status'] = 'true'; $args['include_user_entities'] = 'false'; try { list($http_status, $payload) = $this->api->apiRequest($endpoint, $args); } catch (APICallLimitExceededException $e) { $this->logger->logInfo($e->getMessage(), __METHOD__ . ',' . __LINE__); break; } if ($http_status > 200) { $continue_fetching = false; } else { $status_message = "Parsing JSON. "; $status_message .= "Cursor " . $next_cursor . ":"; $users = $this->api->parseJSONUsers($payload); $next_cursor = $this->api->getNextCursor(); $status_message .= count($users) . " friends queued to update. "; $this->logger->logInfo($status_message, __METHOD__ . ',' . __LINE__); $status_message = ""; if (count($users) == 0) { $this->instance->is_archive_loaded_friends = true; } foreach ($users as $u) { $user_to_update = new User($u, 'Friends'); $this->user_dao->updateUser($user_to_update); // add/update follow relationship $does_follow_exist = $follow_dao->followExists($user_to_update->user_id, $this->instance->network_user_id, 'twitter'); if ($does_follow_exist) { //update it if ($follow_dao->update($user_to_update->user_id, $this->instance->network_user_id, 'twitter')) { $updated_follow_count++; } } else { //insert it if ($follow_dao->insert($user_to_update->user_id, $this->instance->network_user_id, 'twitter')) { $inserted_follow_count++; } } } $this->logger->logSuccess("Cursor at " . strval($next_cursor), __METHOD__ . ',' . __LINE__); } } if ($updated_follow_count > 0 || $inserted_follow_count > 0) { $status_message = $updated_follow_count . " people " . $this->instance->network_username . " follows updated; " . $inserted_follow_count . " new follow(s) inserted."; $this->logger->logUserSuccess($status_message, __METHOD__ . ',' . __LINE__); } } }
/** * Convert parsed JSON of a profile or page's posts into ThinkUp posts and users * @param Object $stream * @param str $source The network for the post; by default 'facebook' */ private function processStream($stream, $network) { $thinkup_posts = array(); $total_added_posts = 0; $thinkup_users = array(); $total_added_users = 0; $thinkup_links = array(); $total_links_added = 0; $thinkup_likes = array(); $total_added_likes = 0; $profile = null; $post_dao = DAOFactory::getDAO('PostDAO'); $must_process_likes = true; $must_process_comments = true; foreach ($stream->data as $p) { $post_id = explode("_", $p->id); $post_id = $post_id[1]; if ($profile == null) { $profile = $this->fetchUserInfo($p->from->id, $network, 'Post stream'); } //assume profile comments are private and page posts are public $is_protected = $network == 'facebook' ? 1 : 0; //get likes count $likes_count = 0; if (isset($p->likes)) { if (is_int($p->likes)) { $likes_count = $p->likes; } elseif (isset($p->likes->count) && is_int($p->likes->count)) { $likes_count = $p->likes->count; } } //Figure out if we have to process likes and comments $post_in_storage = $post_dao->getPost($post_id, $network); if (isset($post_in_storage)) { if ($post_in_storage->favlike_count_cache >= $likes_count) { $must_process_likes = false; $this->logger->logInfo("Already have " . $likes_count . " likes for post ID " . $post_id . "; Skipping like processing this crawler run", __METHOD__ . ',' . __LINE__); } if (isset($p->comments->count)) { if ($post_in_storage->reply_count_cache >= $p->comments->count) { $must_process_comments = false; $this->logger->logInfo("Already have " . $p->comments->count . " comments for post ID " . $post_id . "; Skipping comments processing", __METHOD__ . ',' . __LINE__); } } } if (isset($profile) && !isset($post_in_storage)) { $posts_to_process = array("post_id" => $post_id, "author_username" => $profile->username, "author_fullname" => $profile->username, "author_avatar" => $profile->avatar, "author_user_id" => $p->from->id, "post_text" => isset($p->message) ? $p->message : '', "pub_date" => $p->created_time, "favlike_count_cache" => $likes_count, "in_reply_to_user_id" => '', "in_reply_to_post_id" => '', "source" => '', 'network' => $network, 'is_protected' => $is_protected, 'location' => $profile->location); array_push($thinkup_posts, $posts_to_process); $total_added_posts = $total_added_posts + $this->storePostsAndAuthors($thinkup_posts, "Owner stream"); //free up memory $thinkup_posts = array(); if (isset($p->source) || isset($p->link)) { // there's a link to store $link_url = isset($p->source) ? $p->source : $p->link; $link = new Link(array("url" => $link_url, "expanded_url" => $link_url, "image_src" => isset($p->picture) ? $p->picture : '', "caption" => isset($p->caption) ? $p->caption : '', "description" => isset($p->description) ? $p->description : '', "title" => isset($p->name) ? $p->name : '', "network" => $network, "post_id" => $post_id)); array_push($thinkup_links, $link); } $total_links_addded = $total_links_added + $this->storeLinks($thinkup_links); if ($total_links_added > 0) { $this->logger->logUserSuccess("Collected {$total_links_added} new links", __METHOD__ . ',' . __LINE__); } //free up memory $thinkup_links = array(); } if ($must_process_comments) { if (isset($p->comments)) { $comments_captured = 0; if (isset($p->comments->data)) { $post_comments = $p->comments->data; $post_comments_count = isset($post_comments) ? sizeof($post_comments) : 0; if (is_array($post_comments) && sizeof($post_comments) > 0) { foreach ($post_comments as $c) { if (isset($c->from)) { $comment_id = explode("_", $c->id); $comment_id = $comment_id[2]; //Get posts $posts_to_process = array("post_id" => $comment_id, "author_username" => $c->from->name, "author_fullname" => $c->from->name, "author_avatar" => 'https://graph.facebook.com/' . $c->from->id . '/picture', "author_user_id" => $c->from->id, "post_text" => $c->message, "pub_date" => $c->created_time, "in_reply_to_user_id" => $profile->user_id, "in_reply_to_post_id" => $post_id, "source" => '', 'network' => $network, 'is_protected' => $is_protected, 'location' => ''); array_push($thinkup_posts, $posts_to_process); $comments_captured = $comments_captured + 1; } } } } $total_added_posts = $total_added_posts + $this->storePostsAndAuthors($thinkup_posts, "Post stream comments"); //free up memory $thinkup_posts = array(); // collapsed comment thread if (isset($p->comments->count) && $p->comments->count > $comments_captured) { $api_call = 'https://graph.facebook.com/' . $p->from->id . '_' . $post_id . '/comments?access_token=' . $this->access_token; do { $comments_stream = FacebookGraphAPIAccessor::rawApiRequest($api_call); if (isset($comments_stream) && is_array($comments_stream->data)) { foreach ($comments_stream->data as $c) { if (isset($c->from)) { $comment_id = explode("_", $c->id); $comment_id = $comment_id[sizeof($comment_id) - 1]; //Get posts $posts_to_process = array("post_id" => $comment_id, "author_username" => $c->from->name, "author_fullname" => $c->from->name, "author_avatar" => 'https://graph.facebook.com/' . $c->from->id . '/picture', "author_user_id" => $c->from->id, "post_text" => $c->message, "pub_date" => $c->created_time, "in_reply_to_user_id" => $profile->user_id, "in_reply_to_post_id" => $post_id, "source" => '', 'network' => $network, 'is_protected' => $is_protected, 'location' => ''); array_push($thinkup_posts, $posts_to_process); } } $total_added_posts = $total_added_posts + $this->storePostsAndAuthors($thinkup_posts, "Posts stream comments collapsed"); //free up memory $thinkup_posts = array(); if (isset($comments_stream->paging->next)) { $api_call = str_replace('\\u00257C', '|', $comments_stream->paging->next); } } else { // no comments (pun intended) break; } } while (isset($comments_stream->paging->next)); } } } //process "likes" if ($must_process_likes) { if (isset($p->likes)) { $likes_captured = 0; if (isset($p->likes->data)) { $post_likes = $p->likes->data; $post_likes_count = isset($post_likes) ? sizeof($post_likes) : 0; if (is_array($post_likes) && sizeof($post_likes) > 0) { foreach ($post_likes as $l) { if (isset($l->name) && isset($l->id)) { //Get users $ttu = array("user_name" => $l->name, "full_name" => $l->name, "user_id" => $l->id, "avatar" => 'https://graph.facebook.com/' . $l->id . '/picture', "location" => '', "description" => '', "url" => '', "is_protected" => 1, "follower_count" => 0, "post_count" => 0, "joined" => '', "found_in" => "Likes", "network" => 'facebook'); //Users are always set to network=facebook array_push($thinkup_users, $ttu); $fav_to_add = array("favoriter_id" => $l->id, "network" => $network, "author_user_id" => $profile->user_id, "post_id" => $post_id); array_push($thinkup_likes, $fav_to_add); $likes_captured = $likes_captured + 1; } } } } $total_added_users = $total_added_users + $this->storeUsers($thinkup_users, "Likes"); $total_added_likes = $total_added_likes + $this->storeLikes($thinkup_likes); //free up memory $thinkup_users = array(); $thinkup_likes = array(); // collapsed likes if (isset($p->likes->count) && $p->likes->count > $likes_captured) { $api_call = 'https://graph.facebook.com/' . $p->from->id . '_' . $post_id . '/likes?access_token=' . $this->access_token; do { $likes_stream = FacebookGraphAPIAccessor::rawApiRequest($api_call); if (isset($likes_stream) && is_array($likes_stream->data)) { foreach ($likes_stream->data as $l) { if (isset($l->name) && isset($l->id)) { //Get users $ttu = array("user_name" => $l->name, "full_name" => $l->name, "user_id" => $l->id, "avatar" => 'https://graph.facebook.com/' . $l->id . '/picture', "location" => '', "description" => '', "url" => '', "is_protected" => 1, "follower_count" => 0, "post_count" => 0, "joined" => '', "found_in" => "Likes", "network" => 'facebook'); //Users are always set to network=facebook array_push($thinkup_users, $ttu); $fav_to_add = array("favoriter_id" => $l->id, "network" => $network, "author_user_id" => $p->from->id, "post_id" => $post_id); array_push($thinkup_likes, $fav_to_add); $likes_captured = $likes_captured + 1; } } $total_added_users = $total_added_users + $this->storeUsers($thinkup_users, "Likes"); $total_added_likes = $total_added_likes + $this->storeLikes($thinkup_likes); //free up memory $thinkup_users = array(); $thinkup_likes = array(); if (isset($likes_stream->paging->next)) { $api_call = str_replace('\\u00257C', '|', $likes_stream->paging->next); } } else { // no likes break; } } while (isset($likes_stream->paging->next)); } } //free up memory $thinkup_users = array(); $thinkup_likes = array(); } } if ($total_added_posts > 0) { $this->logger->logUserSuccess("Collected {$total_added_posts} posts", __METHOD__ . ',' . __LINE__); } else { $this->logger->logUserInfo("No new posts found.", __METHOD__ . ',' . __LINE__); } if ($total_added_users > 0) { $this->logger->logUserSuccess("Collected {$total_added_users} users", __METHOD__ . ',' . __LINE__); } else { $this->logger->logUserInfo("No new users found.", __METHOD__ . ',' . __LINE__); } if ($total_added_likes > 0) { $this->logger->logUserSuccess("Collected {$total_added_likes} likes", __METHOD__ . ',' . __LINE__); } else { $this->logger->logUserInfo("No new likes found.", __METHOD__ . ',' . __LINE__); } }