Example #1
0
    /**
     * Fetch a save the posts and replies on a Facebook page.
     * @param int $pid Page ID
     */
    public function fetchPagePostsAndReplies($pid) {
        $stream = FacebookGraphAPIAccessor::apiRequest('/'.$pid.'/posts', $this->access_token);

        if (isset($stream->data) && is_array($stream->data) && sizeof($stream->data > 0)) {
            $this->logger->logSuccess(sizeof($stream->data)." Facebook posts found for page ID $pid.",
            __METHOD__.','.__LINE__);

            $thinkup_data = $this->parseStream($stream, 'facebook page');
            $posts = $thinkup_data["posts"];

            $post_dao = DAOFactory::getDAO('PostDAO');
            $added_posts = 0;
            foreach ($posts as $post) {
                if ($post['author_username']== "" && isset($post['author_user_id'])) {
                    $commenter_object = $this->fetchUserInfo($post['author_user_id'], 'facebook',
                    'Facebook page comments');
                    if (isset($commenter_object)) {
                        $post["author_username"] = $commenter_object->full_name;
                        $post["author_fullname"] = $commenter_object->full_name;
                        $post["author_avatar"] = $commenter_object->avatar;
                    }
                }

                $added_posts = $added_posts + $post_dao->addPost($post);
                $this->logger->logInfo("Added post ID ".$post["post_id"]." on ".$post["network"].
                " for ".$post["author_username"].":".$post["post_text"], __METHOD__.','.__LINE__);
            }

            $added_users = 0;
            $users = $thinkup_data["users"];
            if (count($users) > 0) {
                foreach ($users as $user) {
                    $user["post_count"] = $post_dao->getTotalPostsByUser($user['user_id'], $user['network']);
                    $found_in = 'Facebook page stream';
                    $user_object = new User($user, $found_in);
                    $user_dao = DAOFactory::getDAO('UserDAO');
                    $user_dao->updateUser($user_object);
                    $added_users = $added_users + 1;
                }
            }
            if ($added_posts > 0 || $added_users > 0) {
                $this->logger->logUserSuccess($added_posts." post(s) added; ".$added_users." user(s) updated.",
                __METHOD__.','.__LINE__);
            } else {
                $this->logger->logUserInfo("No new page posts found.", __METHOD__.','.__LINE__);
            }
        } else {
            $this->logger->logInfo("No Facebook posts found for page ID $pid", __METHOD__.','.__LINE__);
        }
    }
Example #2
0
 /**
  * Expand Bit.ly links and recheck click count on any links less than 2 days old.
  *
  * @param str bitly api key
  * @param str bitly login name
  */
 public function acquireBitlyClickStats($api_key, $bit_login)
 {
     $this->logger->setUsername(null);
     $api_accessor = new BitlyAPIAccessor($api_key, $bit_login);
     $bitly_urls = array('http://bit.ly/', 'http://bitly.com/', 'http://j.mp/');
     foreach ($bitly_urls as $bitly_url) {
         if ($this->link_limit != 0) {
             //all short links first seen in the last 48 hours
             $bitly_links_to_update = $this->short_link_dao->getLinksToUpdate($bitly_url);
             if (count($bitly_links_to_update) > 0) {
                 $this->logger->logUserInfo(count($bitly_links_to_update) . " {$bitly_url}" . " links to acquire click stats for.", __METHOD__ . ',' . __LINE__);
             } else {
                 $this->logger->logUserInfo("There are no " . $bitly_url . " links to fetch click stats for.", __METHOD__ . ',', __LINE__);
             }
             $total_links = 0;
             $total_errors = 0;
             $total_updated = 0;
             foreach ($bitly_links_to_update as $link) {
                 $this->logger->logInfo("Getting bit.ly click stats for " . ($total_updated + 1) . " of " . count($bitly_links_to_update) . " " . $bitly_url . " links (" . $link->short_url . ")", __METHOD__ . ',' . __LINE__);
                 $link_data = $api_accessor->getBitlyLinkData($link->short_url);
                 if ($link_data["clicks"] != '') {
                     //save click total here
                     $this->short_link_dao->saveClickCount($link->short_url, $link_data["clicks"]);
                     // Save title to links table
                     if ($link_data["title"] != '') {
                         $this->link_dao->updateTitle($link->link_id, $link_data["title"]);
                     }
                     $total_links = $total_links + 1;
                     $total_updated = $total_updated + 1;
                 } elseif ($link_data["error"] != '') {
                     $this->link_dao->saveExpansionError($link->short_url, $link_data["error"]);
                     $total_errors = $total_errors + 1;
                     $total_updated = $total_updated + 1;
                 }
             }
             $this->logger->logUserSuccess($total_links . " " . $bitly_url . " link click stats acquired (" . $total_errors . " errors)", __METHOD__ . ',' . __LINE__);
         }
     }
 }
 /**
  * cleanUpMissedFavsUnFavs  pages back through the older pages of favs, checking for favs that are not yet in
  * the database, as well as favs that were added to the db but are no longer returned by Twitter's API.
  * However, that latter calculation, for un-fav'd tweets, is currently not reliable due to a bug on Twitter's end,
  * and so such tweets are not currently removed from the database.
  * Due to the same issue with the API, it's not clear whether all favs of older tweets are going to be actually
  * returned from Twitter (that is, it is currently not returning some actually-favorited tweets in a given range).
  * So, we may miss some older tweets that were in fact favorited, until Twitter fixes this.
  * The number of pages to page back for each run of the crawler is set by favs_cleanup_pages option.
  */
 public function cleanUpMissedFavsUnFavs()
 {
     // first, check that we have the resources to do work
     if (!($this->api->available && $this->api->available_api_calls_for_crawler)) {
         $this->logger->logInfo("terminating cleanUpMissedFavsUnFavs-- no API calls available", __METHOD__ . ',' . __LINE__);
         return true;
     }
     $this->logger->logInfo("In cleanUpMissedFavsUnFavs", __METHOD__ . ',' . __LINE__);
     $this->logger->logInfo("User id: " . $this->user->user_id . "\n", __METHOD__ . ',' . __LINE__);
     $fcount = 0;
     $favs_cleanup_pages = 1;
     // default number of pages to process each time the crawler runs
     // get plugin option value if it exists & is positive int, otherwise use default
     $topt = $this->twitter_options;
     if (isset($topt['favs_cleanup_pages'])) {
         $conf_favs_cleanup_pages = $topt['favs_cleanup_pages']->option_value;
         $this->logger->logInfo("conf_favs_cleanup_pages: {$conf_favs_cleanup_pages} ", __METHOD__ . ',' . __LINE__);
         if (is_integer((int) $conf_favs_cleanup_pages) && $conf_favs_cleanup_pages > 0) {
             $favs_cleanup_pages = $conf_favs_cleanup_pages;
         }
     }
     $this->logger->logInfo("favs_cleanup_pages: {$favs_cleanup_pages} ", __METHOD__ . ',' . __LINE__);
     $fpd = DAOFactory::getDAO('FavoritePostDAO');
     $pagesize = 20;
     // number of favs per page retrieved from the API call... (tbd: any way to get
     //this from the API?)
     // get 'favs_older_pages' plugin option value if it exists & is pos. int.  Use it to calculate default start
     // page if set, otherwise use default value.
     $default_start_page = 2;
     $topt = $this->twitter_options;
     if (isset($topt['favs_older_pages'])) {
         $conf_older_favs_pages = $topt['favs_older_pages']->option_value;
         if (is_integer((int) $conf_older_favs_pages) && $conf_older_favs_pages > 0) {
             $default_start_page = $conf_older_favs_pages + 1;
         }
     }
     $this->logger->logInfo("default start page: {$default_start_page} ", __METHOD__ . ',' . __LINE__);
     $last_page_of_favs = round($this->api->archive_limit / $pagesize);
     $last_unfav_page_checked = $this->instance->last_unfav_page_checked;
     $start_page = $last_unfav_page_checked > 0 ? $last_unfav_page_checked + 1 : $default_start_page;
     $this->logger->logInfo("start page: {$start_page}, with {$favs_cleanup_pages} cleanup pages", __METHOD__ . ',' . __LINE__);
     $curr_favs_count = $this->user->favorites_count;
     $count = 0;
     $page = $start_page;
     while ($count < $favs_cleanup_pages && $this->api->available && $this->api->available_api_calls_for_crawler) {
         // get the favs from that page
         try {
             list($tweets, $cURL_status, $twitter_data) = $this->getFavsPage($page);
         } catch (APICallLimitExceededException $e) {
             break;
         }
         if ($cURL_status != 200 || $tweets == -1) {
             // todo - handle more informatively
             $this->logger->logInfo("in cleanUpMissedFavsUnFavs, error with: {$twitter_data}", __METHOD__ . ',' . __LINE__);
             throw new Exception("in cleanUpUnFavs: error parsing favs");
         }
         if (sizeof($tweets) == 0) {
             // then done paging backwards through the favs.
             // reset pointer so that we start at the recent favs again next time through.
             $this->instance->last_unfav_page_checked = 0;
             break;
         }
         $min_tweet = $tweets[sizeof($tweets) - 1]['post_id'];
         $max_tweet = $tweets[0]['post_id'];
         $this->logger->logInfo("in cleanUpUnFavs, page {$page} min and max: {$min_tweet}, {$max_tweet}", __METHOD__ . ',' . __LINE__);
         foreach ($tweets as $fav) {
             $fav['network'] = 'twitter';
             // check whether the tweet is in the db-- if not, add it.
             if ($fpd->addFavorite($this->user->user_id, $fav) > 0) {
                 URLProcessor::processPostURLs($fav['post_text'], $fav['post_id'], 'twitter', $this->logger);
                 $this->logger->logInfo("added fav " . $fav['post_id'], __METHOD__ . ',' . __LINE__);
                 $fcount++;
             } else {
                 $status_message = "have already stored fav " . $fav['post_id'];
                 $this->logger->logDebug($status_message, __METHOD__ . ',' . __LINE__);
             }
         }
         // now for each favorited tweet in the database within the fetched range, check whether it's still
         // favorited. This part of the method is currently disabled due to issues with the Twitter API, which
         // is not returning all of the favorited tweets any more.  So, the fact that a previously-archived
         // tweet is not returned, no longer indicates that it was un-fav'd.
         // The method still IDs the 'missing' tweets, but no longer deletes them.  We may want to get rid of
         //  this check altogether at some point.
         $fposts = $fpd->getAllFavoritePostsUpperBound($this->user->user_id, 'twitter', $pagesize, $max_tweet + 1);
         foreach ($fposts as $old_fav) {
             $old_fav_id = $old_fav->post_id;
             if ($old_fav_id < $min_tweet) {
                 $this->logger->logInfo("Old fav {$old_fav_id} out of range ", __METHOD__ . ',' . __LINE__);
                 break;
                 // all the rest will be out of range also then
             }
             // look for the old_fav_id in the array of fetched favs
             $found = false;
             foreach ($tweets as $tweet) {
                 if ($old_fav_id == $tweet['post_id']) {
                     $found = true;
                     break;
                 }
             }
             if (!$found) {
                 // if it's not there...
                 // 14/10 arghh -- Twitter is suddenly (temporarily?) not returning all fav'd tweets in a
                 // sequence.
                 // skipping the delete for now, keep tabs on it.  Can check before delete with extra API
                 // request, but the point of doing it this way was to avoid the additional API request.
                 $this->logger->logInfo("Twitter claims tweet not still favorited, but this is currently " . "broken, so not deleting: " . $old_fav_id, __METHOD__ . ',' . __LINE__);
                 // 'unfavorite' by removing from favorites table
                 // $fpd->unFavorite($old_fav_id, $this->user->user_id);
             }
         }
         $this->instance->last_unfav_page_checked = $page++;
         if ($page > $last_page_of_favs) {
             $page = 0;
             break;
         }
         $count++;
     }
     $this->logger->logUserSuccess("Added {$fcount} older missed favorites", __METHOD__ . ',' . __LINE__);
     return true;
 }
 /**
  * Convert parsed JSON of a profile or page's posts into ThinkUp posts and users
  * @param Object $stream
  * @param str $source The network for the post, either 'facebook' or 'facebook page'
  * @param int Page number being processed
  * @return int $total_added_posts How many posts (excluding comments) got added to the data store
  */
 private function processStream($stream, $network, $page_number)
 {
     $thinkup_posts = array();
     $total_added_posts = 0;
     $total_added_comments = 0;
     $thinkup_users = array();
     $total_added_users = 0;
     $thinkup_links = array();
     $total_links_added = 0;
     $thinkup_likes = array();
     $total_added_likes = 0;
     $profiles = array();
     //efficiency control vars
     $must_process_likes = true;
     $must_process_comments = true;
     $post_comments_added = 0;
     $post_likes_added = 0;
     $comments_difference = false;
     $likes_difference = false;
     $post_dao = DAOFactory::getDAO('PostDAO');
     foreach ($stream->data as $index => $p) {
         $post_id = explode("_", $p->id);
         $post_id = $post_id[1];
         $this->logger->logInfo("Beginning to process " . $post_id . ", post " . ($index + 1) . " of " . count($stream->data) . " on page " . $page_number, __METHOD__ . ',' . __LINE__);
         // stream can contain posts from multiple users.  get profile for this post
         $profile = null;
         if (!empty($profiles[$p->from->id])) {
             $profile = $profiles[$p->from->id];
         } else {
             $profile = $this->fetchUser($p->from->id, 'Post stream', true);
             $profiles[$p->from->id] = $profile;
         }
         //Assume profile comments are private and page posts are public
         $is_protected = $network == 'facebook' ? 1 : 0;
         //Get likes count
         $likes_count = 0;
         //Normalize likes to be one array
         if (isset($p->likes)) {
             $likes_count = $p->likes->summary->total_count;
             $p->likes = $this->normalizeLikes($p->likes);
         }
         // Normalize comments to be one array
         $comments_count = 0;
         if (isset($p->comments)) {
             $comments_count = $p->comments->summary->total_count;
             $p->comments = $this->normalizeComments($p->comments);
         }
         $post_in_storage = $post_dao->getPost($post_id, $network);
         //Figure out if we have to process likes and comments
         if (isset($post_in_storage)) {
             $this->logger->logInfo("Post " . $post_id . " already in storage", __METHOD__ . ',' . __LINE__);
             if ($post_in_storage->favlike_count_cache >= $likes_count) {
                 $must_process_likes = false;
                 $this->logger->logInfo("Already have " . $likes_count . " like(s) for post " . $post_id . " in storage; skipping like processing", __METHOD__ . ',' . __LINE__);
             } else {
                 $likes_difference = $likes_count - $post_in_storage->favlike_count_cache;
                 $this->logger->logInfo($likes_difference . " new like(s) to process for post " . $post_id, __METHOD__ . ',' . __LINE__);
             }
             if (isset($p->comments->summary->total_count)) {
                 if ($post_in_storage->reply_count_cache >= $p->comments->summary->total_count) {
                     $must_process_comments = false;
                     $this->logger->logInfo("Already have " . $post_in_storage->reply_count_cache . " comment(s) for post " . $post_id . "; skipping comment processing", __METHOD__ . ',' . __LINE__);
                 } else {
                     $comments_difference = $p->comments->summary->total_count - $post_in_storage->reply_count_cache;
                     $this->logger->logInfo($comments_difference . " new comment(s) of " . $p->comments->summary->total_count . " total to process for post " . $post_id, __METHOD__ . ',' . __LINE__);
                 }
             }
         } else {
             $this->logger->logInfo("Post " . $post_id . " not in storage", __METHOD__ . ',' . __LINE__);
         }
         if (!isset($profile)) {
             $this->logger->logError("No profile set", __METHOD__ . ',' . __LINE__);
         } else {
             if (!isset($post_in_storage)) {
                 $this->logger->logInfo("Post " . $post_id . " has " . $comments_count . " comments", __METHOD__ . ',' . __LINE__);
                 $post_to_process = array("post_id" => $post_id, "author_username" => $profile->username, "author_fullname" => $profile->username, "author_avatar" => $profile->avatar, "author_user_id" => $p->from->id, "post_text" => isset($p->message) ? $p->message : '', "pub_date" => $p->created_time, "favlike_count_cache" => $likes_count, "reply_count_cache" => $comments_count, "in_reply_to_user_id" => isset($p->to->data[0]->id) ? $p->to->data[0]->id : '', "in_reply_to_post_id" => '', "source" => '', 'network' => $network, 'is_protected' => $is_protected, 'location' => '');
                 $new_post_key = $this->storePostAndAuthor($post_to_process, "Owner stream");
                 if ($new_post_key !== false) {
                     $total_added_posts++;
                 }
                 if (isset($p->source) || isset($p->link)) {
                     // there's a link to store
                     $link_url = isset($p->source) ? $p->source : $p->link;
                     $link = new Link(array("url" => $link_url, "expanded_url" => '', "image_src" => isset($p->picture) ? $p->picture : '', "caption" => isset($p->caption) ? $p->caption : '', "description" => isset($p->description) ? $p->description : '', "title" => isset($p->name) ? $p->name : '', "post_key" => $new_post_key));
                     array_push($thinkup_links, $link);
                 }
                 $total_links_addded = $total_links_added + $this->storeLinks($thinkup_links);
                 if ($total_links_added > 0) {
                     $this->logger->logUserSuccess("Collected {$total_links_added} new links", __METHOD__ . ',' . __LINE__);
                 }
                 //free up memory
                 $thinkup_links = array();
             } else {
                 // post already exists in storage
                 if ($must_process_likes) {
                     //update its like count only
                     $post_dao->updateFavLikeCount($post_id, $network, $likes_count);
                     $this->logger->logInfo("Updated Like count for post " . $post_id . " to " . $likes_count, __METHOD__ . ',' . __LINE__);
                 }
             }
             if ($must_process_comments) {
                 if (isset($p->comments)) {
                     $comments_captured = 0;
                     if (isset($p->comments->data)) {
                         $post_comments = $p->comments->data;
                         $post_comments_count = isset($post_comments) ? sizeof($post_comments) : 0;
                         if (is_array($post_comments) && sizeof($post_comments) > 0) {
                             foreach ($post_comments as $c) {
                                 if (isset($c->from)) {
                                     // Sometimes the id is parent_poster_postId
                                     // sometimes it's just parent_postId
                                     $comment_id = explode("_", $c->id);
                                     if (count($comment_id) == 3) {
                                         $comment_id = $comment_id[2];
                                     } else {
                                         $comment_id = $comment_id[1];
                                     }
                                     //only add to queue if not already in storage
                                     $comment_in_storage = $post_dao->getPost($comment_id, $network);
                                     if (!isset($comment_in_storage)) {
                                         $comment_to_process = array("post_id" => $comment_id, "author_username" => $c->from->name, "author_fullname" => $c->from->name, "author_gender" => $c->from->gender, "author_birthday" => $c->from->birthday, "author_avatar" => 'https://graph.facebook.com/' . $c->from->id . '/picture', "author_user_id" => $c->from->id, "post_text" => $c->message, "pub_date" => $c->created_time, "in_reply_to_user_id" => $profile->user_id, "in_reply_to_post_id" => $post_id, "source" => '', 'network' => $network, 'is_protected' => $is_protected, 'location' => '');
                                         array_push($thinkup_posts, $comment_to_process);
                                         $comments_captured = $comments_captured + 1;
                                     }
                                 }
                             }
                         }
                     }
                     $post_comments_added = $post_comments_added + $this->storePostsAndAuthors($thinkup_posts, "Post stream comments");
                     //free up memory
                     $thinkup_posts = array();
                     if (is_int($comments_difference) && $post_comments_added >= $comments_difference) {
                         $must_process_comments = false;
                         if (isset($comments_stream->paging->next)) {
                             $this->logger->logInfo("Caught up on post " . $post_id . "'s balance of " . $comments_difference . " comments; stopping comment processing", __METHOD__ . ',' . __LINE__);
                         }
                     }
                     // collapsed comment thread
                     if (isset($p->comments->summary->total_count) && $p->comments->summary->total_count > $comments_captured && $must_process_comments) {
                         if (is_int($comments_difference)) {
                             $offset = $p->comments->summary->total_count - $comments_difference;
                             $offset_arr = array('offset' => $offset, 'limit' => $comments_difference);
                         } else {
                             $offset_arr = null;
                         }
                         $api_call = $p->from->id . '_' . $post_id . '/comments';
                         do {
                             $comments_stream = FacebookGraphAPIAccessor::apiRequest($api_call, $this->access_token, $offset_arr);
                             if (isset($comments_stream) && isset($comments_stream->data) && is_array($comments_stream->data)) {
                                 foreach ($comments_stream->data as $c) {
                                     if (isset($c->from)) {
                                         $comment_id = explode("_", $c->id);
                                         $comment_id = $comment_id[sizeof($comment_id) - 1];
                                         //only add to queue if not already in storage
                                         $comment_in_storage = $post_dao->getPost($comment_id, $network);
                                         if (!isset($comment_in_storage)) {
                                             $comment_to_process = array("post_id" => $comment_id, "author_username" => $c->from->name, "author_fullname" => $c->from->name, "author_avatar" => 'https://graph.facebook.com/' . $c->from->id . '/picture', "author_user_id" => $c->from->id, "post_text" => $c->message, "pub_date" => $c->created_time, "in_reply_to_user_id" => $profile->user_id, "in_reply_to_post_id" => $post_id, "source" => '', 'network' => $network, 'is_protected' => $is_protected, 'location' => '');
                                             array_push($thinkup_posts, $comment_to_process);
                                         }
                                     }
                                 }
                                 $post_comments_added = $post_comments_added + $this->storePostsAndAuthors($thinkup_posts, "Posts stream comments collapsed");
                                 if (is_int($comments_difference) && $post_comments_added >= $comments_difference) {
                                     $must_process_comments = false;
                                     if (isset($comments_stream->paging->next)) {
                                         $this->logger->logInfo("Caught up on post " . $post_id . "'s balance of " . $comments_difference . " comments; stopping comment processing", __METHOD__ . ',' . __LINE__);
                                     }
                                 }
                                 //free up memory
                                 $thinkup_posts = array();
                                 if (isset($comments_stream->paging->next)) {
                                     $api_call = str_replace('\\u00257C', '|', $comments_stream->paging->next);
                                 }
                             } else {
                                 // no comments (pun intended)
                                 break;
                             }
                         } while (isset($comments_stream->paging->next) && $must_process_comments);
                     }
                 }
                 if ($post_comments_added > 0) {
                     //let user know
                     $this->logger->logUserSuccess("Added " . $post_comments_added . " comment(s) for post " . $post_id, __METHOD__ . ',' . __LINE__);
                 } else {
                     $this->logger->logInfo("Added " . $post_comments_added . " comment(s) for post " . $post_id, __METHOD__ . ',' . __LINE__);
                 }
                 $total_added_comments = $total_added_comments + $post_comments_added;
             }
             //Inserting comments also increments the original post's reply_count_cache; reset it here
             $post_dao->updateReplyCount($post_id, $network, $comments_count);
             //process "likes"
             if ($must_process_likes) {
                 if (isset($p->likes)) {
                     $likes_captured = 0;
                     if (isset($p->likes->data)) {
                         $post_likes = $p->likes->data;
                         $post_likes_count = isset($post_likes) ? sizeof($post_likes) : 0;
                         if (is_array($post_likes) && sizeof($post_likes) > 0) {
                             foreach ($post_likes as $l) {
                                 if (isset($l->name) && isset($l->id)) {
                                     //Get users
                                     $user_to_add = array("user_name" => $l->name, "full_name" => $l->name, "user_id" => $l->id, "avatar" => 'https://graph.facebook.com/' . $l->id . '/picture', "location" => '', "description" => '', "url" => '', "is_protected" => 1, "follower_count" => 0, "post_count" => 0, "joined" => '', "found_in" => "Likes", "network" => 'facebook');
                                     //Users are always set to network=facebook
                                     array_push($thinkup_users, $user_to_add);
                                     $fav_to_add = array("favoriter_id" => $l->id, "network" => $network, "author_user_id" => $profile->user_id, "post_id" => $post_id);
                                     array_push($thinkup_likes, $fav_to_add);
                                     $likes_captured = $likes_captured + 1;
                                 }
                             }
                         }
                     }
                     $total_added_users = $total_added_users + $this->storeUsers($thinkup_users, "Likes");
                     $post_likes_added = $post_likes_added + $this->storeLikes($thinkup_likes);
                     //free up memory
                     $thinkup_users = array();
                     $thinkup_likes = array();
                     if (is_int($likes_difference) && $post_likes_added >= $likes_difference) {
                         $must_process_likes = false;
                         if (isset($likes_stream->paging->next)) {
                             $this->logger->logInfo("Caught up on post " . $post_id . "'s balance of " . $likes_difference . " likes; stopping like processing", __METHOD__ . ',' . __LINE__);
                         }
                     }
                     // collapsed likes
                     if (isset($p->likes->count) && $p->likes->count > $likes_captured && $must_process_likes) {
                         if (is_int($likes_difference)) {
                             $offset = $p->likes->count - $likes_difference;
                             $offset_arr = array('offset' => $offset);
                         } else {
                             $offset_arr = null;
                         }
                         $api_call = $p->from->id . '_' . $post_id . '/likes';
                         do {
                             $likes_stream = FacebookGraphAPIAccessor::apiRequest($api_call, $this->access_token, $offset_arr);
                             if (isset($likes_stream) && is_array($likes_stream->data)) {
                                 foreach ($likes_stream->data as $l) {
                                     if (isset($l->name) && isset($l->id)) {
                                         //Get users
                                         $user_to_add = array("user_name" => $l->name, "full_name" => $l->name, "user_id" => $l->id, "avatar" => 'https://graph.facebook.com/' . $l->id . '/picture', "is_protected" => 1, "location" => '', "description" => '', "url" => '', "follower_count" => 0, "post_count" => 0, "joined" => '', "found_in" => "Likes", "network" => 'facebook');
                                         //Users are always set to network=facebook
                                         array_push($thinkup_users, $user_to_add);
                                         $fav_to_add = array("favoriter_id" => $l->id, "network" => $network, "author_user_id" => $p->from->id, "post_id" => $post_id);
                                         array_push($thinkup_likes, $fav_to_add);
                                         $likes_captured = $likes_captured + 1;
                                     }
                                 }
                                 $total_added_users = $total_added_users + $this->storeUsers($thinkup_users, "Likes");
                                 $post_likes_added = $post_likes_added + $this->storeLikes($thinkup_likes);
                                 //free up memory
                                 $thinkup_users = array();
                                 $thinkup_likes = array();
                                 if (is_int($likes_difference) && $post_likes_added >= $likes_difference) {
                                     $must_process_likes = false;
                                     if (isset($likes_stream->paging->next)) {
                                         $this->logger->logInfo("Caught up on post " . $post_id . "'s balance of " . $likes_difference . " likes; stopping like processing", __METHOD__ . ',' . __LINE__);
                                     }
                                 }
                                 if (isset($likes_stream->paging->next)) {
                                     $api_call = str_replace('\\u00257C', '|', $likes_stream->paging->next);
                                 }
                             } else {
                                 // no likes
                                 break;
                             }
                         } while (isset($likes_stream->paging->next) && $must_process_likes);
                     }
                 }
                 $this->logger->logInfo("Added " . $post_likes_added . " like(s) for post " . $post_id, __METHOD__ . ',' . __LINE__);
                 $total_added_likes = $total_added_likes + $post_likes_added;
             }
             //free up memory
             $thinkup_users = array();
             $thinkup_likes = array();
         }
         //reset control vars for next post
         $must_process_likes = true;
         $must_process_comments = true;
         $post_comments_added = 0;
         $post_likes_added = 0;
         $comments_difference = false;
         $likes_difference = false;
     }
     $this->logger->logUserSuccess("On page " . $page_number . ", captured " . $total_added_posts . " post(s), " . $total_added_comments . " comment(s), " . $total_added_users . " user(s) and " . $total_added_likes . " like(s)", __METHOD__ . ',' . __LINE__);
     return $total_added_posts;
 }
 /**
  * Retrieve tweets in search results for a keyword/hashtag.
  * @param InstanceHashtag $instance_hashtag
  * @return void
  */
 public function fetchInstanceHashtagTweets($instance_hashtag)
 {
     if (isset($this->instance)) {
         $status_message = "";
         $continue_fetching = true;
         $since_id = 0;
         $max_id = 0;
         $instance_hashtag_dao = DAOFactory::getDAO('InstanceHashtagDAO');
         $post_dao = DAOFactory::getDAO('PostDAO');
         $user_dao = DAOFactory::getDAO('UserDAO');
         $hashtagpost_dao = DAOFactory::getDAO('HashtagPostDAO');
         $hashtag_dao = DAOFactory::getDAO('HashtagDAO');
         //Get hashtag
         $hashtag = $hashtag_dao->getHashtagByID($instance_hashtag->hashtag_id);
         while ($continue_fetching) {
             $endpoint = $this->api->endpoints['search_tweets'];
             $args = array();
             $args["q"] = $hashtag->hashtag;
             $count_arg = isset($this->twitter_options['tweet_count_per_call']) ? $this->twitter_options['tweet_count_per_call']->option_value : 100;
             $args["count"] = $count_arg;
             $args["include_entities"] = "true";
             if ($since_id == 0) {
                 $since_id = $instance_hashtag->last_post_id;
             }
             if ($since_id > 0) {
                 $args["since_id"] = $since_id;
             }
             if ($max_id > $since_id) {
                 $args["max_id"] = $max_id;
             }
             try {
                 list($http_status, $payload) = $this->api->apiRequest($endpoint, $args);
             } catch (APICallLimitExceededException $e) {
                 $this->logger->logInfo($e->getMessage(), __METHOD__ . ',' . __LINE__);
                 break;
             }
             if ($http_status == 200) {
                 $this->logger->logDebug('Search tweets 200 ' . $endpoint->getPath(), __METHOD__ . ',' . __LINE__);
                 $count = 0;
                 $user_count = 0;
                 $tweets = $this->api->parseJSONTweetsFromSearch($payload);
                 foreach ($tweets as $tweet) {
                     $this->logger->logDebug('Processing ' . $tweet['post_id'], __METHOD__ . ',' . __LINE__);
                     $this->logger->logDebug('Processing ' . Utils::varDumpToString($tweet), __METHOD__ . ',' . __LINE__);
                     $inserted_post_key = $post_dao->addPost($tweet, $this->user, $this->logger);
                     //We need to check if post exists before add relationship between post and hashtag
                     if ($post_dao->isPostInDB($tweet['post_id'], 'twitter')) {
                         if (!$hashtagpost_dao->isHashtagPostInStorage($hashtag->id, $tweet['post_id'], 'twitter')) {
                             $count = $count + 1;
                             $hashtagpost_dao->insertHashtagPost($hashtag->hashtag, $tweet['post_id'], 'twitter');
                             $user = new User($tweet);
                             $rows_updated = $user_dao->updateUser($user);
                             if ($rows_updated > 0) {
                                 $user_count = $user_count + $rows_updated;
                             }
                             $this->logger->logDebug('User has been updated', __METHOD__ . ',' . __LINE__);
                             if (isset($tweet['retweeted_post']) && isset($tweet['retweeted_post']['content'])) {
                                 $this->logger->logDebug('Retweeted post info set', __METHOD__ . ',' . __LINE__);
                                 if (!$hashtagpost_dao->isHashtagPostInStorage($hashtag->id, $tweet['retweeted_post']['content']['post_id'], 'twitter')) {
                                     $this->logger->logDebug('Retweeted post not in storage', __METHOD__ . ',' . __LINE__);
                                     $count++;
                                     $hashtagpost_dao->insertHashtagPost($hashtag->hashtag, $tweet['retweeted_post']['content']['post_id'], 'twitter');
                                     $user_retweet = new User($tweet['retweeted_post']['content']);
                                     $rows_retweet_updated = $user_dao->updateUser($user_retweet);
                                     if ($rows_retweet_updated > 0) {
                                         $user_count = $user_count + $rows_retweet_updated;
                                     }
                                 } else {
                                     $this->logger->logDebug('Retweeted post in storage', __METHOD__ . ',' . __LINE__);
                                 }
                             } else {
                                 $this->logger->logDebug('Retweeted post info not set', __METHOD__ . ',' . __LINE__);
                             }
                             $this->logger->logDebug('About to process URLs', __METHOD__ . ',' . __LINE__);
                             URLProcessor::processPostURLs($tweet['post_text'], $tweet['post_id'], 'twitter', $this->logger);
                             $this->logger->logDebug('URLs have been processed', __METHOD__ . ',' . __LINE__);
                         }
                     }
                     if ($tweet['post_id'] > $instance_hashtag->last_post_id) {
                         $instance_hashtag->last_post_id = $tweet['post_id'];
                     }
                     if ($instance_hashtag->earliest_post_id == 0 || $tweet['post_id'] < $instance_hashtag->earliest_post_id) {
                         $instance_hashtag->earliest_post_id = $tweet['post_id'];
                     }
                     if ($max_id == 0 || $tweet['post_id'] < $max_id) {
                         $max_id = $tweet['post_id'];
                     }
                     $this->logger->logDebug('Instance hashtag markers updated', __METHOD__ . ',' . __LINE__);
                 }
                 //Status message for tweets and users
                 $status_message = ' ' . count($tweets) . " tweet(s) found and {$count} saved";
                 $this->logger->logUserSuccess($status_message, __METHOD__ . ',' . __LINE__);
                 $status_message = ' ' . count($tweets) . " tweet(s) found and {$user_count} users saved";
                 $this->logger->logUserSuccess($status_message, __METHOD__ . ',' . __LINE__);
                 //Save instance_hashtag important values
                 if ($instance_hashtag->last_post_id > 0) {
                     $instance_hashtag_dao->updateLastPostID($instance_hashtag->instance_id, $instance_hashtag->hashtag_id, $instance_hashtag->last_post_id);
                 }
                 if ($instance_hashtag->earliest_post_id > 0) {
                     $instance_hashtag_dao->updateEarliestPostID($instance_hashtag->instance_id, $instance_hashtag->hashtag_id, $instance_hashtag->earliest_post_id);
                 }
                 //Not to continue fetching if search not return the maxim number of tweets
                 if (count($tweets) < $count_arg) {
                     $continue_fetching = false;
                 }
             } else {
                 $status_message = "Stop fetching tweets. cURL_status = " . $cURL_status;
                 $this->logger->logUserSuccess($status_message, __METHOD__ . ',' . __LINE__);
                 $continue_fetching = false;
             }
         }
     }
 }
Example #6
0
 /**
  * Convert a collection of profile posts into ThinkUp posts and users
  * @param Object $posts
  * @param str $source The network for the post, always 'instagram'
  * @param int Page number being processed
  */
 private function processPosts(Instagram\Collection\MediaCollection $posts, $network, $page_number)
 {
     $thinkup_posts = array();
     $total_added_posts = 0;
     $thinkup_users = array();
     $total_added_users = 0;
     $thinkup_likes = array();
     $total_added_likes = 0;
     //efficiency control vars
     $must_process_likes = true;
     $must_process_comments = true;
     $post_comments_added = 0;
     $post_likes_added = 0;
     $comments_difference = false;
     $likes_difference = false;
     $post_dao = DAOFactory::getDAO('PostDAO');
     foreach ($posts as $index => $p) {
         $post_id = $p->getId();
         $this->logger->logInfo("Beginning to process " . $post_id . ", post " . ($index + 1) . " of " . count($posts->count()) . " on page " . $page_number, __METHOD__ . ',' . __LINE__);
         // stream can contain posts from multiple users.  get profile for this post
         $profile = $p->getUser();
         $is_protected = 0;
         //Get likes count
         $likes_count = $p->getLikesCount();
         $comments = $p->getComments();
         $post_in_storage = $post_dao->getPost($post_id, $network);
         //Figure out if we have to process likes and comments
         if (isset($post_in_storage)) {
             $this->logger->logInfo("Post " . $post_id . " already in storage", __METHOD__ . ',' . __LINE__);
             if ($post_in_storage->favlike_count_cache >= $likes_count) {
                 $must_process_likes = false;
                 $this->logger->logInfo("Already have " . $likes_count . " like(s) for post " . $post_id . "in storage; skipping like processing", __METHOD__ . ',' . __LINE__);
             } else {
                 $likes_difference = $likes_count - $post_in_storage->favlike_count_cache;
                 $this->logger->logInfo($likes_difference . " new like(s) to process for post " . $post_id, __METHOD__ . ',' . __LINE__);
             }
             $commentsCount = $comments->count();
             if ($commentsCount > 0) {
                 if ($post_in_storage->reply_count_cache >= $commentsCount) {
                     $must_process_comments = false;
                     $this->logger->logInfo("Already have " . $commentsCount . " comment(s) for post " . $post_id . "; skipping comment processing", __METHOD__ . ',' . __LINE__);
                 } else {
                     $comments_difference = $commentsCount - $post_in_storage->reply_count_cache;
                     $this->logger->logInfo($comments_difference . " new comment(s) of " . $commentsCount . " total to process for post " . $post_id, __METHOD__ . ',' . __LINE__);
                 }
             }
         } else {
             $this->logger->logInfo("Post " . $post_id . " not in storage", __METHOD__ . ',' . __LINE__);
         }
         // If we dont already have this photo
         if (!isset($post_in_storage)) {
             // Photos may be posted without a caption
             // Note that if you post a photo without a caption and then reply to it with the first comment instagram
             // treats this as your caption.
             if (strlen($p->getCaption()) > 0) {
                 $text = $p->getCaption();
             } else {
                 $text = "";
             }
             $photo_to_process = array("post_id" => $post_id, "author_username" => $profile->getUserName(), "author_fullname" => $profile->getFullName(), "author_avatar" => $profile->getProfilePicture(), "author_user_id" => $profile->getId(), "post_text" => $text, "pub_date" => DateTime::createFromFormat('U', $p->getCreatedTime())->format('Y-m-d H:i'), "favlike_count_cache" => $likes_count, "in_reply_to_user_id" => '', "in_reply_to_post_id" => '', "source" => '', 'network' => $network, 'is_protected' => $is_protected, 'location' => '', 'permalink' => $p->getLink(), 'standard_resolution_url' => $p->getStandardRes()->url, 'low_resolution_url' => $p->getLowRes()->url, 'thumbnail_url' => $p->getThumbnail()->url, 'filter' => $p->getFilter());
             $new_photo_key = $this->storePhotoAndAuthor($photo_to_process, "Owner stream");
             if ($new_photo_key !== false) {
                 $total_added_posts++;
             }
         } else {
             // post already exists in storage
             if ($must_process_likes) {
                 //update its like count only
                 $post_dao->updateFavLikeCount($post_id, $network, $likes_count);
                 $this->logger->logInfo("Updated Like count for post " . $post_id . " to " . $likes_count, __METHOD__ . ',' . __LINE__);
             }
         }
         if ($must_process_comments) {
             if ($comments->count() > 0) {
                 $comments_captured = 0;
                 $post_comments = $comments;
                 if ($post_comments->count() > 0) {
                     foreach ($post_comments as $c) {
                         $comment_id = $c->getId();
                         //only add to queue if not already in storage
                         $comment_in_storage = $post_dao->getPost($comment_id, $network);
                         if (!isset($comment_in_storage)) {
                             $comment_author = $c->getUser();
                             $comment_to_process = array("post_id" => $comment_id, "author_username" => $comment_author->getUserName(), "author_fullname" => $comment_author->getFullName(), "author_avatar" => $comment_author->getProfilePicture(), "author_user_id" => $comment_author->getId(), "post_text" => $c->getText(), "pub_date" => DateTime::createFromFormat('U', $p->getCreatedTime())->format('Y-m-d H:i'), "in_reply_to_user_id" => $profile->getId(), "in_reply_to_post_id" => $post_id, "source" => '', 'network' => $network, 'is_protected' => $is_protected, 'location' => '');
                             array_push($thinkup_posts, $comment_to_process);
                             $comments_captured = $comments_captured + 1;
                         }
                     }
                 }
                 $post_comments_added = $post_comments_added + $this->storePostsAndAuthors($thinkup_posts, "Post stream comments");
                 //free up memory
                 $thinkup_posts = array();
                 if (is_int($comments_difference) && $post_comments_added >= $comments_difference) {
                     $must_process_comments = false;
                     $this->logger->logInfo("Caught up on post " . $post_id . "'s balance of " . $comments_difference . " comments; stopping comment processing", __METHOD__ . ',' . __LINE__);
                 }
             }
             if ($post_comments_added > 0) {
                 //let user know
                 $this->logger->logUserSuccess("Added " . $post_comments_added . " comment(s) for post " . $post_id, __METHOD__ . ',' . __LINE__);
             } else {
                 $this->logger->logInfo("Added " . $post_comments_added . " comment(s) for post " . $post_id, __METHOD__ . ',' . __LINE__);
             }
             $total_added_posts = $total_added_posts + $post_comments_added;
         }
         //process "likes"
         if ($must_process_likes) {
             if ($likes_count > 0) {
                 $likes_captured = 0;
                 $post_likes = $p->getLikes();
                 $post_likes_count = $likes_count;
                 if ($post_likes_count > 0) {
                     foreach ($post_likes as $l) {
                         //Get users
                         $user_to_add = array("user_name" => $l->getUserName(), "full_name" => $l->getFullName(), "user_id" => $l->getId(), "avatar" => $l->getProfilePicture(), "description" => '', "url" => '', "is_protected" => 0, "follower_count" => 0, "post_count" => 0, "joined" => '', "found_in" => "Likes", "network" => 'instagram');
                         //Users are always set to network=instagram
                         array_push($thinkup_users, $user_to_add);
                         $fav_to_add = array("favoriter_id" => $l->getId(), "network" => $network, "author_user_id" => $profile->getId(), "post_id" => $post_id);
                         array_push($thinkup_likes, $fav_to_add);
                         $likes_captured = $likes_captured + 1;
                     }
                 }
                 $total_added_users = $total_added_users + $this->calculateNumberOfUsersStored($thinkup_users, "Likes");
                 $post_likes_added = $post_likes_added + $this->storeLikes($thinkup_likes);
                 //free up memory
                 $thinkup_users = array();
                 $thinkup_likes = array();
                 if (is_int($likes_difference) && $post_likes_added >= $likes_difference) {
                     $must_process_likes = false;
                     $this->logger->logInfo("Caught up on post " . $post_id . "'s balance of " . $likes_difference . " likes; stopping like processing", __METHOD__ . ',' . __LINE__);
                 }
             }
             $this->logger->logInfo("Added " . $post_likes_added . " like(s) for post " . $post_id, __METHOD__ . ',' . __LINE__);
             $total_added_likes = $total_added_likes + $post_likes_added;
         }
         //free up memory
         $thinkup_users = array();
         $thinkup_likes = array();
         //reset control vars for next post
         $must_process_likes = true;
         $must_process_comments = true;
         $post_comments_added = 0;
         $post_likes_added = 0;
         $comments_difference = false;
         $likes_difference = false;
     }
     $this->logger->logUserSuccess("On page " . $page_number . ", captured " . $total_added_posts . " post(s), " . $total_added_users . " user(s) and " . $total_added_likes . " like(s)", __METHOD__ . ',' . __LINE__);
 }
 /**
  * Fetch the instance user's friends' tweets.
  */
 public function fetchInstanceUserFriends()
 {
     if (!isset($this->user)) {
         $this->fetchInstanceUserInfo();
     }
     if (isset($this->user)) {
         $follow_dao = DAOFactory::getDAO('FollowDAO');
         $this->instance->total_friends_in_system = $follow_dao->countTotalFriends($this->instance->network_user_id, 'twitter');
         if ($this->instance->total_friends_in_system < $this->user->friend_count) {
             $this->instance->is_archive_loaded_friends = false;
             $this->logger->logUserInfo($this->instance->total_friends_in_system . " friends in system, " . $this->user->friend_count . " friends according to Twitter; Friend archive is not loaded", __METHOD__ . ',' . __LINE__);
         } else {
             $this->instance->is_archive_loaded_friends = true;
             $this->logger->logInfo("Friend archive loaded", __METHOD__ . ',' . __LINE__);
         }
         $status_message = "";
         // Fetch friend pages
         $continue_fetching = true;
         $updated_follow_count = 0;
         $inserted_follow_count = 0;
         while ($continue_fetching && !$this->instance->is_archive_loaded_friends) {
             $endpoint = $this->api->endpoints['following'];
             $args = array();
             if (!isset($next_cursor)) {
                 $next_cursor = -1;
             }
             $args['cursor'] = strval($next_cursor);
             $args['skip_status'] = 'true';
             $args['include_user_entities'] = 'false';
             try {
                 list($http_status, $payload) = $this->api->apiRequest($endpoint, $args);
             } catch (APICallLimitExceededException $e) {
                 $this->logger->logInfo($e->getMessage(), __METHOD__ . ',' . __LINE__);
                 break;
             }
             if ($http_status > 200) {
                 $continue_fetching = false;
             } else {
                 $status_message = "Parsing JSON. ";
                 $status_message .= "Cursor " . $next_cursor . ":";
                 $users = $this->api->parseJSONUsers($payload);
                 $next_cursor = $this->api->getNextCursor();
                 $status_message .= count($users) . " friends queued to update. ";
                 $this->logger->logInfo($status_message, __METHOD__ . ',' . __LINE__);
                 $status_message = "";
                 if (count($users) == 0) {
                     $this->instance->is_archive_loaded_friends = true;
                 }
                 foreach ($users as $u) {
                     $user_to_update = new User($u, 'Friends');
                     $this->user_dao->updateUser($user_to_update);
                     // add/update follow relationship
                     $does_follow_exist = $follow_dao->followExists($user_to_update->user_id, $this->instance->network_user_id, 'twitter');
                     if ($does_follow_exist) {
                         //update it
                         if ($follow_dao->update($user_to_update->user_id, $this->instance->network_user_id, 'twitter')) {
                             $updated_follow_count++;
                         }
                     } else {
                         //insert it
                         if ($follow_dao->insert($user_to_update->user_id, $this->instance->network_user_id, 'twitter')) {
                             $inserted_follow_count++;
                         }
                     }
                 }
                 $this->logger->logSuccess("Cursor at " . strval($next_cursor), __METHOD__ . ',' . __LINE__);
             }
         }
         if ($updated_follow_count > 0 || $inserted_follow_count > 0) {
             $status_message = $updated_follow_count . " people " . $this->instance->network_username . " follows updated; " . $inserted_follow_count . " new follow(s) inserted.";
             $this->logger->logUserSuccess($status_message, __METHOD__ . ',' . __LINE__);
         }
     }
 }
 /**
  * Convert parsed JSON of a profile or page's posts into ThinkUp posts and users
  * @param Object $stream
  * @param str $source The network for the post; by default 'facebook'
  */
 private function processStream($stream, $network)
 {
     $thinkup_posts = array();
     $total_added_posts = 0;
     $thinkup_users = array();
     $total_added_users = 0;
     $thinkup_links = array();
     $total_links_added = 0;
     $thinkup_likes = array();
     $total_added_likes = 0;
     $profile = null;
     $post_dao = DAOFactory::getDAO('PostDAO');
     $must_process_likes = true;
     $must_process_comments = true;
     foreach ($stream->data as $p) {
         $post_id = explode("_", $p->id);
         $post_id = $post_id[1];
         if ($profile == null) {
             $profile = $this->fetchUserInfo($p->from->id, $network, 'Post stream');
         }
         //assume profile comments are private and page posts are public
         $is_protected = $network == 'facebook' ? 1 : 0;
         //get likes count
         $likes_count = 0;
         if (isset($p->likes)) {
             if (is_int($p->likes)) {
                 $likes_count = $p->likes;
             } elseif (isset($p->likes->count) && is_int($p->likes->count)) {
                 $likes_count = $p->likes->count;
             }
         }
         //Figure out if we have to process likes and comments
         $post_in_storage = $post_dao->getPost($post_id, $network);
         if (isset($post_in_storage)) {
             if ($post_in_storage->favlike_count_cache >= $likes_count) {
                 $must_process_likes = false;
                 $this->logger->logInfo("Already have " . $likes_count . " likes for post ID " . $post_id . "; Skipping like processing this crawler run", __METHOD__ . ',' . __LINE__);
             }
             if (isset($p->comments->count)) {
                 if ($post_in_storage->reply_count_cache >= $p->comments->count) {
                     $must_process_comments = false;
                     $this->logger->logInfo("Already have " . $p->comments->count . " comments for post ID " . $post_id . "; Skipping comments processing", __METHOD__ . ',' . __LINE__);
                 }
             }
         }
         if (isset($profile) && !isset($post_in_storage)) {
             $posts_to_process = array("post_id" => $post_id, "author_username" => $profile->username, "author_fullname" => $profile->username, "author_avatar" => $profile->avatar, "author_user_id" => $p->from->id, "post_text" => isset($p->message) ? $p->message : '', "pub_date" => $p->created_time, "favlike_count_cache" => $likes_count, "in_reply_to_user_id" => '', "in_reply_to_post_id" => '', "source" => '', 'network' => $network, 'is_protected' => $is_protected, 'location' => $profile->location);
             array_push($thinkup_posts, $posts_to_process);
             $total_added_posts = $total_added_posts + $this->storePostsAndAuthors($thinkup_posts, "Owner stream");
             //free up memory
             $thinkup_posts = array();
             if (isset($p->source) || isset($p->link)) {
                 // there's a link to store
                 $link_url = isset($p->source) ? $p->source : $p->link;
                 $link = new Link(array("url" => $link_url, "expanded_url" => $link_url, "image_src" => isset($p->picture) ? $p->picture : '', "caption" => isset($p->caption) ? $p->caption : '', "description" => isset($p->description) ? $p->description : '', "title" => isset($p->name) ? $p->name : '', "network" => $network, "post_id" => $post_id));
                 array_push($thinkup_links, $link);
             }
             $total_links_addded = $total_links_added + $this->storeLinks($thinkup_links);
             if ($total_links_added > 0) {
                 $this->logger->logUserSuccess("Collected {$total_links_added} new links", __METHOD__ . ',' . __LINE__);
             }
             //free up memory
             $thinkup_links = array();
         }
         if ($must_process_comments) {
             if (isset($p->comments)) {
                 $comments_captured = 0;
                 if (isset($p->comments->data)) {
                     $post_comments = $p->comments->data;
                     $post_comments_count = isset($post_comments) ? sizeof($post_comments) : 0;
                     if (is_array($post_comments) && sizeof($post_comments) > 0) {
                         foreach ($post_comments as $c) {
                             if (isset($c->from)) {
                                 $comment_id = explode("_", $c->id);
                                 $comment_id = $comment_id[2];
                                 //Get posts
                                 $posts_to_process = array("post_id" => $comment_id, "author_username" => $c->from->name, "author_fullname" => $c->from->name, "author_avatar" => 'https://graph.facebook.com/' . $c->from->id . '/picture', "author_user_id" => $c->from->id, "post_text" => $c->message, "pub_date" => $c->created_time, "in_reply_to_user_id" => $profile->user_id, "in_reply_to_post_id" => $post_id, "source" => '', 'network' => $network, 'is_protected' => $is_protected, 'location' => '');
                                 array_push($thinkup_posts, $posts_to_process);
                                 $comments_captured = $comments_captured + 1;
                             }
                         }
                     }
                 }
                 $total_added_posts = $total_added_posts + $this->storePostsAndAuthors($thinkup_posts, "Post stream comments");
                 //free up memory
                 $thinkup_posts = array();
                 // collapsed comment thread
                 if (isset($p->comments->count) && $p->comments->count > $comments_captured) {
                     $api_call = 'https://graph.facebook.com/' . $p->from->id . '_' . $post_id . '/comments?access_token=' . $this->access_token;
                     do {
                         $comments_stream = FacebookGraphAPIAccessor::rawApiRequest($api_call);
                         if (isset($comments_stream) && is_array($comments_stream->data)) {
                             foreach ($comments_stream->data as $c) {
                                 if (isset($c->from)) {
                                     $comment_id = explode("_", $c->id);
                                     $comment_id = $comment_id[sizeof($comment_id) - 1];
                                     //Get posts
                                     $posts_to_process = array("post_id" => $comment_id, "author_username" => $c->from->name, "author_fullname" => $c->from->name, "author_avatar" => 'https://graph.facebook.com/' . $c->from->id . '/picture', "author_user_id" => $c->from->id, "post_text" => $c->message, "pub_date" => $c->created_time, "in_reply_to_user_id" => $profile->user_id, "in_reply_to_post_id" => $post_id, "source" => '', 'network' => $network, 'is_protected' => $is_protected, 'location' => '');
                                     array_push($thinkup_posts, $posts_to_process);
                                 }
                             }
                             $total_added_posts = $total_added_posts + $this->storePostsAndAuthors($thinkup_posts, "Posts stream comments collapsed");
                             //free up memory
                             $thinkup_posts = array();
                             if (isset($comments_stream->paging->next)) {
                                 $api_call = str_replace('\\u00257C', '|', $comments_stream->paging->next);
                             }
                         } else {
                             // no comments (pun intended)
                             break;
                         }
                     } while (isset($comments_stream->paging->next));
                 }
             }
         }
         //process "likes"
         if ($must_process_likes) {
             if (isset($p->likes)) {
                 $likes_captured = 0;
                 if (isset($p->likes->data)) {
                     $post_likes = $p->likes->data;
                     $post_likes_count = isset($post_likes) ? sizeof($post_likes) : 0;
                     if (is_array($post_likes) && sizeof($post_likes) > 0) {
                         foreach ($post_likes as $l) {
                             if (isset($l->name) && isset($l->id)) {
                                 //Get users
                                 $ttu = array("user_name" => $l->name, "full_name" => $l->name, "user_id" => $l->id, "avatar" => 'https://graph.facebook.com/' . $l->id . '/picture', "location" => '', "description" => '', "url" => '', "is_protected" => 1, "follower_count" => 0, "post_count" => 0, "joined" => '', "found_in" => "Likes", "network" => 'facebook');
                                 //Users are always set to network=facebook
                                 array_push($thinkup_users, $ttu);
                                 $fav_to_add = array("favoriter_id" => $l->id, "network" => $network, "author_user_id" => $profile->user_id, "post_id" => $post_id);
                                 array_push($thinkup_likes, $fav_to_add);
                                 $likes_captured = $likes_captured + 1;
                             }
                         }
                     }
                 }
                 $total_added_users = $total_added_users + $this->storeUsers($thinkup_users, "Likes");
                 $total_added_likes = $total_added_likes + $this->storeLikes($thinkup_likes);
                 //free up memory
                 $thinkup_users = array();
                 $thinkup_likes = array();
                 // collapsed likes
                 if (isset($p->likes->count) && $p->likes->count > $likes_captured) {
                     $api_call = 'https://graph.facebook.com/' . $p->from->id . '_' . $post_id . '/likes?access_token=' . $this->access_token;
                     do {
                         $likes_stream = FacebookGraphAPIAccessor::rawApiRequest($api_call);
                         if (isset($likes_stream) && is_array($likes_stream->data)) {
                             foreach ($likes_stream->data as $l) {
                                 if (isset($l->name) && isset($l->id)) {
                                     //Get users
                                     $ttu = array("user_name" => $l->name, "full_name" => $l->name, "user_id" => $l->id, "avatar" => 'https://graph.facebook.com/' . $l->id . '/picture', "location" => '', "description" => '', "url" => '', "is_protected" => 1, "follower_count" => 0, "post_count" => 0, "joined" => '', "found_in" => "Likes", "network" => 'facebook');
                                     //Users are always set to network=facebook
                                     array_push($thinkup_users, $ttu);
                                     $fav_to_add = array("favoriter_id" => $l->id, "network" => $network, "author_user_id" => $p->from->id, "post_id" => $post_id);
                                     array_push($thinkup_likes, $fav_to_add);
                                     $likes_captured = $likes_captured + 1;
                                 }
                             }
                             $total_added_users = $total_added_users + $this->storeUsers($thinkup_users, "Likes");
                             $total_added_likes = $total_added_likes + $this->storeLikes($thinkup_likes);
                             //free up memory
                             $thinkup_users = array();
                             $thinkup_likes = array();
                             if (isset($likes_stream->paging->next)) {
                                 $api_call = str_replace('\\u00257C', '|', $likes_stream->paging->next);
                             }
                         } else {
                             // no likes
                             break;
                         }
                     } while (isset($likes_stream->paging->next));
                 }
             }
             //free up memory
             $thinkup_users = array();
             $thinkup_likes = array();
         }
     }
     if ($total_added_posts > 0) {
         $this->logger->logUserSuccess("Collected {$total_added_posts} posts", __METHOD__ . ',' . __LINE__);
     } else {
         $this->logger->logUserInfo("No new posts found.", __METHOD__ . ',' . __LINE__);
     }
     if ($total_added_users > 0) {
         $this->logger->logUserSuccess("Collected {$total_added_users} users", __METHOD__ . ',' . __LINE__);
     } else {
         $this->logger->logUserInfo("No new users found.", __METHOD__ . ',' . __LINE__);
     }
     if ($total_added_likes > 0) {
         $this->logger->logUserSuccess("Collected {$total_added_likes} likes", __METHOD__ . ',' . __LINE__);
     } else {
         $this->logger->logUserInfo("No new likes found.", __METHOD__ . ',' . __LINE__);
     }
 }