Example #1
0
 /**
  * For a given post, extract URLs and store them, including image_src if that's from a known source like Twitpic,
  * Twitgoo, Yfrog, Instagr.am.
  * @param str $post_text
  * @param int $post_id
  * @param str $network
  * @param Logger $logger
  * @param arr $urls Array of URLs, optionally set, defaults to null
  */
 public static function processPostURLs($post_text, $post_id, $network, $logger, $urls = null)
 {
     if (!$urls) {
         $urls = Post::extractURLs($post_text);
     }
     if ($urls) {
         $link_dao = DAOFactory::getDAO('LinkDAO');
         $post_dao = DAOFactory::getDAO('PostDAO');
         $post = $post_dao->getPost($post_id, $network);
         if (isset($post->id)) {
             foreach ($urls as $url) {
                 $logger->logInfo("Processing URL {$url}", __METHOD__ . ',' . __LINE__);
                 $image_src = self::getImageSource($url);
                 //if we have an image_src, the URL is a known image source not in need of expansion
                 $expanded_url = $image_src !== '' ? $url : '';
                 $link_array = array('url' => $url, 'expanded_url' => $expanded_url, "image_src" => $image_src, 'post_key' => $post->id);
                 $link = new Link($link_array);
                 try {
                     $link_dao->insert($link);
                     $logger->logSuccess("Inserted " . $url . " " . ($image_src == '' ? '' : "(thumbnail " . $image_src . ") ") . "into links table", __METHOD__ . ',' . __LINE__);
                 } catch (DuplicateLinkException $e) {
                     $logger->logInfo($url . " " . ($image_src == '' ? '' : "(thumbnail " . $image_src . ") ") . " already exists in links table", __METHOD__ . ',' . __LINE__);
                 } catch (DataExceedsColumnWidthException $e) {
                     $logger->logInfo($url . " " . ($image_src == '' ? '' : "(thumbnail " . $image_src . ") ") . " data exceeds table column width", __METHOD__ . ',' . __LINE__);
                 }
             }
         }
     }
 }
 /**
  * Update profiles of users who are friends of the instance user, and haven't been checked in 1 day.
  * @return void
  */
 public function updateFriendsProfiles()
 {
     if (!isset($this->user)) {
         $this->fetchInstanceUserInfo();
     }
     if (isset($this->user)) {
         //Get stalest friends
         $follow_dao = DAOFactory::getDAO('FollowDAO');
         $stalest_friends = $follow_dao->getStalestFriends($this->user->user_id, 'twitter', $number_days_old = 1, $limit = 25);
         $status_message = count($stalest_friends) . ' friends haven\'t been updated recently.';
         $this->logger->logInfo($status_message, __METHOD__ . ',' . __LINE__);
         while (isset($stalest_friends) && count($stalest_friends) > 0) {
             try {
                 foreach ($stalest_friends as $user) {
                     $this->fetchAndAddUser($user->user_id, "Friends stale update");
                 }
                 $stalest_friends = $follow_dao->getStalestFriends($this->user->user_id, 'twitter', $number_days_old = 1, $limit = 25);
                 $status_message = count($stalest_friends) . ' friends haven\'t been updated recently.';
                 $this->logger->logInfo($status_message, __METHOD__ . ',' . __LINE__);
             } catch (APICallLimitExceededException $e) {
                 $this->logger->logInfo($e->getMessage(), __METHOD__ . ',' . __LINE__);
                 break;
             }
         }
     }
 }
 /**
  * Update profiles of users who are friends of the instance user, and haven't been checked in 2 days.
  * @return void
  */
 public function updateStaleFriendsProfiles()
 {
     if (!isset($this->user)) {
         //Force-refresh instance user in data store
         $this->user = self::fetchUser($this->instance->network_user_id, 'Owner info', $this->instance->network_username, null, null, true);
     }
     if (isset($this->user)) {
         //Get stalest friends
         $follow_dao = DAOFactory::getDAO('FollowDAO');
         $stalest_friends = $follow_dao->getStalestFriends($this->user->user_id, 'instagram', $number_days_old = 2, $limit = 25);
         $status_message = count($stalest_friends) . ' friends haven\'t been updated recently.';
         $this->logger->logInfo($status_message, __METHOD__ . ',' . __LINE__);
         while (isset($stalest_friends) && count($stalest_friends) > 0) {
             try {
                 foreach ($stalest_friends as $user) {
                     $this->fetchUser($user->user_id, "Friends stale update", $user->username, $user->full_name, $user->avatar, true);
                 }
                 $stalest_friends = $follow_dao->getStalestFriends($this->user->user_id, 'instagram', $number_days_old = 2, $limit = 25);
                 $status_message = count($stalest_friends) . ' friends haven\'t been updated recently.';
                 $this->logger->logInfo($status_message, __METHOD__ . ',' . __LINE__);
             } catch (APICallLimitExceededException $e) {
                 $this->logger->logInfo($e->getMessage(), __METHOD__ . ',' . __LINE__);
                 break;
             }
         }
     }
 }
 /**
  * This helper method returns the parsed favorites from a given favorites page.
  * @param int $since_id
  * @return array ($tweets, $http_status, $payload)
  */
 private function getFavorites($since_id)
 {
     $endpoint = $this->api->endpoints['favorites'];
     $args = array();
     $args["screen_name"] = $this->user->username;
     $args["include_entities"] = "false";
     $args["count"] = 100;
     if ($since_id != "") {
         $args["since_id"] = $since_id;
     }
     try {
         list($http_status, $payload) = $this->api->apiRequest($endpoint, $args);
         if ($http_status == 200) {
             // Parse the JSON file
             $tweets = $this->api->parseJSONTweets($payload);
             if (!(isset($tweets) && sizeof($tweets) == 0) && $tweets == null) {
                 // arghh, empty array evals to null
                 $this->logger->logInfo("in getFavorites; could not extract any tweets from response", __METHOD__ . ',' . __LINE__);
                 throw new Exception("could not extract any tweets from response");
             }
         }
     } catch (APICallLimitExceededException $e) {
         $this->logger->logInfo($e->getMessage(), __METHOD__ . ',' . __LINE__);
     }
     return array($tweets, $http_status, $payload);
 }
 private function storePostsAndAuthors($posts, $posts_source)
 {
     $total_added_posts = 0;
     $added_posts = 0;
     $post_dao = DAOFactory::getDAO('PostDAO');
     foreach ($posts as $post) {
         if (isset($post['author_user_id'])) {
             $user_object = $this->fetchUserInfo($post['author_user_id'], 'facebook', $posts_source);
             if (isset($user_object)) {
                 $post["author_username"] = $user_object->full_name;
                 $post["author_fullname"] = $user_object->full_name;
                 $post["author_avatar"] = $user_object->avatar;
                 $post["location"] = $user_object->location;
             }
         }
         $added_posts = $post_dao->addPost($post);
         if ($added_posts == 0 && isset($post['favlike_count_cache'])) {
             //post already exists in storage, so update its like count only
             $post_dao->updateFavLikeCount($post['post_id'], $post['network'], $post['favlike_count_cache']);
             $this->logger->logInfo("Updated Like count for post ID " . $post["post_id"] . " to " . $post['favlike_count_cache'], __METHOD__ . ',' . __LINE__);
         }
         if ($added_posts > 0) {
             $this->logger->logInfo("Added post ID " . $post["post_id"] . " on " . $post["network"] . " for " . $post["author_username"] . ":" . $post["post_text"], __METHOD__ . ',' . __LINE__);
         }
         $total_added_posts = $total_added_posts + $added_posts;
         $added_posts = 0;
     }
     return $added_posts;
 }
 /**
  * Check the validity of G+'s OAuth token by requestig the instance user's details.
  * Fetch details from Google+ API for the current instance user and insert into the datastore.
  * @param str $client_id
  * @param str $client_secret
  * @param str $access_token
  * @param str $refresh_token
  * @param str $owner_id
  * @return User
  */
 public function initializeInstanceUser($client_id, $client_secret, $access_token, $refresh_token, $owner_id)
 {
     $network = 'google+';
     $user_dao = DAOFactory::getDAO('UserDAO');
     $user_object = null;
     // Get owner user details and save them to DB
     $fields = array('fields' => 'displayName,id,image,tagline');
     $user_details = $this->api_accessor->apiRequest('people/me', $this->access_token, $fields);
     if (isset($user_details->error->code) && $user_details->error->code == '401') {
         //Token has expired, fetch and save a new one
         $tokens = self::getOAuthTokens($client_id, $client_secret, $refresh_token, 'refresh_token');
         $owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
         $owner_instance_dao->updateTokens($owner_id, $this->instance->id, $access_token, $refresh_token);
         $this->access_token = $tokens->access_token;
         //try again
         $user_details = $this->api_accessor->apiRequest('people/me', $this->access_token, $fields);
     }
     $user_details->network = $network;
     $user = $this->parseUserDetails($user_details);
     if (isset($user)) {
         $user_object = new User($user, 'Owner initialization');
         $user_dao->updateUser($user_object);
     }
     if (isset($user_object)) {
         $this->logger->logSuccess("Successfully fetched " . $user_object->username . " " . $user_object->network . "'s details from Google+", __METHOD__ . ',' . __LINE__);
     } else {
         $this->logger->logInfo("Error fetching " . $user_id . " " . $network . "'s details from the Google+ API, " . "response was " . Utils::varDumpToString($user_details), __METHOD__ . ',' . __LINE__);
     }
     return $user_object;
 }
Example #7
0
 /**
  * For fetching details about a user who does not have a Gooogle Plus ID
  * If user doesn't exist in the datastore, fetch details from YouTube API and insert into the datastore.
  * If $reload_from_youtube is true, update existing user details in store with data from YouTube API.
  * @param int $user_id YouTube user ID
  * @param str $found_in Where the user was found
  * @param bool $reload_from_youtube Defaults to false; if true will query YouTube API and update existing user
  * @return User
  */
 public function fetchUserFromYouTube($user_id, $found_in, $force_reload_from_youtube = false)
 {
     $network = 'youtube';
     $user_dao = DAOFactory::getDAO('UserDAO');
     $user_object = null;
     if ($force_reload_from_youtube || !$user_dao->isUserInDB($user_id, $network)) {
         // Get owner user details and save them to DB
         $fields = array('alt' => 'json');
         if (isset($this->developer_key)) {
             $fields['key'] = $this->developer_key;
         }
         $user_details = $this->youtube_api_v2_accessor->apiRequest('users/' . $user_id_string[sizeof($user_id_string) - 1], $fields);
         $user = $this->parseYouTubeUserDetails($user_details->entry);
         if (isset($user)) {
             $user_object = new User($user, $found_in);
             $user_dao->updateUser($user_object);
         }
         if (isset($user_object)) {
             $this->logger->logSuccess("Successfully fetched " . $user_id . " " . $network . "'s details from YouTube", __METHOD__ . ',' . __LINE__);
         } else {
             $this->logger->logInfo("Error fetching " . $user_id . " " . $network . "'s details from the YouTube API, " . "response was " . Utils::varDumpToString($user_details), __METHOD__ . ',' . __LINE__);
         }
     }
     return $user_object;
 }
 /**
  * If link is an image (Twitpic/Twitgoo/Yfrog/Flickr for now), insert direct path to thumb as expanded url.
  * @TODO Move image thumbnail processng to Expand URLs plugin.
  * @param Logger $logger
  * @param str $tweet
  * @param Array $urls
  */
 public static function processTweetURLs($logger, $tweet, $urls = null)
 {
     $link_dao = DAOFactory::getDAO('LinkDAO');
     if (!$urls) {
         $urls = Post::extractURLs($tweet['post_text']);
     }
     foreach ($urls as $u) {
         $logger->logInfo("processing url: {$u}", __METHOD__ . ',' . __LINE__);
         $is_image = 0;
         $title = '';
         $eurl = '';
         if (substr($u, 0, strlen('http://twitpic.com/')) == 'http://twitpic.com/') {
             $eurl = 'http://twitpic.com/show/thumb/' . substr($u, strlen('http://twitpic.com/'));
             $is_image = 1;
         } elseif (substr($u, 0, strlen('http://yfrog.com/')) == 'http://yfrog.com/') {
             $eurl = $u . '.th.jpg';
             $is_image = 1;
         } elseif (substr($u, 0, strlen('http://twitgoo.com/')) == 'http://twitgoo.com/') {
             $eurl = 'http://twitgoo.com/show/thumb/' . substr($u, strlen('http://twitgoo.com/'));
             $is_image = 1;
         } elseif (substr($u, 0, strlen('http://picplz.com/')) == 'http://picplz.com/') {
             $eurl = $u . '/thumb/';
             $is_image = 1;
         } elseif (substr($u, 0, strlen('http://flic.kr/')) == 'http://flic.kr/') {
             $is_image = 1;
         } elseif (substr($u, 0, strlen('http://instagr.am/')) == 'http://instagr.am/') {
             $is_image = 1;
         }
         if ($link_dao->insert($u, $eurl, $title, $tweet['post_id'], 'twitter', $is_image)) {
             $logger->logSuccess("Inserted " . $u . " (" . $eurl . ", " . $is_image . "), into links table", __METHOD__ . ',' . __LINE__);
         } else {
             $logger->logError("Did NOT insert " . $u . " (" . $eurl . ") into links table", __METHOD__ . ',' . __LINE__);
         }
     }
 }
 /**
  * Make an API request.
  * @param str $type String representing the endpoint, 'user', 'followers', 'media', 'relationship'
  * @param arr $params API call params
  * @return Object
  */
 public function apiRequest($type, $params = array())
 {
     if ($this->total_api_calls >= $this->max_api_calls) {
         $this->logger->logInfo("Throw APICallLimitExceededException", __METHOD__ . ',' . __LINE__);
         throw new APICallLimitExceededException();
     } else {
         $this->total_api_calls++;
         $this->logger->logInfo("Made " . $this->total_api_calls . " API calls", __METHOD__ . ',' . __LINE__);
     }
     try {
         if ($type == 'user') {
             return $this->instagram->getUser($params['user_id']);
         } else {
             if ($type == 'followers') {
                 return $this->current_user->getFollowers($params);
             } else {
                 if ($type == 'friends') {
                     return $this->current_user->getFollows($params);
                 } else {
                     if ($type == 'media') {
                         return $this->current_user->getMedia($params);
                     } else {
                         if ($type == 'relationship') {
                             return $this->current_user->getRelationship($params['user_id']);
                         } else {
                             if ($type == 'likes') {
                                 return $this->current_user->getLikedMedia($params);
                             }
                         }
                     }
                 }
             }
         }
     } catch (Instagram\Core\ApiException $e) {
         if ($e->getMessage() == 'you cannot view this resource') {
             $this->logger->logInfo("Throw APICallPermissionDeniedException: " . $e->getMessage(), __METHOD__ . ',' . __LINE__);
             throw new APICallPermissionDeniedException($e->getMessage());
         } elseif (strpos($e->getMessage(), 'exceeded the maximum number of requests per hour') !== false) {
             $this->logger->logInfo("Throw APICallLimitExceededException: " . $e->getMessage(), __METHOD__ . ',' . __LINE__);
             throw new APICallLimitExceededException($e->getMessage());
         } else {
             $this->logger->logInfo("Throw APIErrorException: " . $e->getMessage(), __METHOD__ . ',' . __LINE__);
             throw new APIErrorException($e->getMessage());
         }
     }
 }
Example #10
0
 /**
  * Capture the current instance users's posts and store them in the database.
  * @return null
  */
 public function fetchInstanceUserPosts()
 {
     //For now only capture the most recent 20 posts
     //@TODO Page back through all the archives
     $fields = array('alt' => 'json', 'maxResults' => 20, 'pp' => 1);
     $user_posts = $this->api_accessor->apiRequest('people/' . $this->instance->network_user_id . '/activities/public', $this->access_token, $fields);
     if (isset($user_posts->items)) {
         $post_dao = DAOFactory::getDAO('PostDAO');
         $link_dao = DAOFactory::getDAO('LinkDAO');
         foreach ($user_posts->items as $item) {
             $should_capture_post = false;
             //For now we're only capturing posts and shares
             //@TODO Capture all types of posts
             if ($item->verb == "post") {
                 $post['post_text'] = $item->object->content;
                 $should_capture_post = true;
             } elseif ($item->verb == "share") {
                 $post['post_text'] = isset($item->annotation) ? $item->annotation : '';
                 $should_capture_post = true;
             }
             if ($should_capture_post) {
                 $post['post_id'] = $item->id;
                 $post['author_username'] = $item->actor->displayName;
                 $post['author_fullname'] = $item->actor->displayName;
                 $post['author_avatar'] = $item->actor->image->url;
                 $post['author_user_id'] = $item->actor->id;
                 $post['pub_date'] = $item->published;
                 $post['source'] = '';
                 $post['is_protected'] = false;
                 $post['network'] = 'google+';
                 $post['reply_count_cache'] = $item->object->replies->totalItems;
                 $post['favlike_count_cache'] = $item->object->plusoners->totalItems;
                 $post['retweet_count_cache'] = $item->object->resharers->totalItems;
                 $inserted_post_key = $post_dao->addPost($post);
                 //If no post was added, at least update reply/fave/reshare counts and post text
                 if ($inserted_post_key === false) {
                     $post_dao->updateFavLikeCount($post['post_id'], 'google+', $post['favlike_count_cache']);
                     $post_dao->updateReplyCount($post['post_id'], 'google+', $post['reply_count_cache']);
                     $post_dao->updateRetweetCount($post['post_id'], 'google+', $post['retweet_count_cache']);
                     $post_dao->updatePostText($post['post_id'], 'google+', $post['post_text']);
                 }
                 if (isset($item->object->attachments) && isset($item->object->attachments[0]->url)) {
                     $link_url = $item->object->attachments[0]->url;
                     $link = new Link(array("url" => $link_url, "expanded_url" => '', "image_src" => isset($item->object->attachments[0]->image->url) ? $item->object->attachments[0]->image->url : '', "caption" => '', "description" => isset($item->object->attachments[0]->content) ? $item->object->attachments[0]->content : '', "title" => isset($item->object->attachments[0]->displayName) ? $item->object->attachments[0]->displayName : '', "post_key" => $inserted_post_key));
                     try {
                         $added_links = $link_dao->insert($link);
                     } catch (DuplicateLinkException $e) {
                         $this->logger->logInfo($link->url . " already exists in links table", __METHOD__ . ',' . __LINE__);
                     } catch (DataExceedsColumnWidthException $e) {
                         $this->logger->logInfo($link->url . "  data exceeds table column width", __METHOD__ . ',' . __LINE__);
                     }
                 }
             }
             $post = null;
             $link = null;
         }
     }
 }
Example #11
0
 /**
  * Retrives all of a users friends from the Instagram API and stores them in the database
  * @return null
  */
 private function storeFriends()
 {
     if ($this->instance->network != 'instagram') {
         return;
     }
     //Retrieve friends via the Instagram API
     $user_id = $this->instance->network_user_id;
     $access_token = $this->access_token;
     $network = $user_id == $this->instance->network_user_id ? $this->instance->network : 'instagram';
     try {
         $friends = InstagramAPIAccessor::apiRequest('friends', $user_id, $access_token);
     } catch (Instagram\Core\ApiException $e) {
         $this->logger->logInfo(get_class($e) . " Error fetching friends from Instagram API, error was " . $e->getMessage(), __METHOD__ . ',' . __LINE__);
         return;
     }
     if (isset($friends)) {
         //store relationships in follows table
         $follows_dao = DAOFactory::getDAO('FollowDAO');
         $count_dao = DAOFactory::getDAO('CountHistoryDAO');
         $user_dao = DAOFactory::getDAO('UserDAO');
         foreach ($friends as $friend) {
             $follower_id = null;
             try {
                 $follower_id = $friend->getId();
             } catch (Instagram\Core\ApiException $e) {
                 $this->logger->logInfo(get_class($e) . " Error fetching " . Utils::varDumpToString($friend) . "'s details from Instagram API, error was " . $e->getMessage(), __METHOD__ . ',' . __LINE__);
             }
             if (isset($follower_id)) {
                 if ($follows_dao->followExists($user_id, $follower_id, $network)) {
                     // follow relationship already exists
                     $follows_dao->update($user_id, $follower_id, $network);
                 } else {
                     // follow relationship does not exist yet
                     $follows_dao->insert($user_id, $follower_id, $network);
                 }
                 $follower_details = $friend;
                 if (isset($follower_details)) {
                     $follower_details->network = $network;
                 }
                 $follower = $this->parseUserDetails($follower_details);
                 $follower_object = new User($follower);
                 if (isset($follower_object)) {
                     $user_dao->updateUser($follower_object);
                 }
             }
         }
         //totals in follower_count table
         $count_dao->insert($user_id, $network, $friends->count(), null, 'followers');
     } else {
         throw new Instagram\Core\ApiAuthException('Error retrieving friends');
     }
     $this->logger->logInfo("Ending", __METHOD__ . ',' . __LINE__);
 }
Example #12
0
    /**
     * Fetch a save the posts and replies on a Facebook page.
     * @param int $pid Page ID
     */
    public function fetchPagePostsAndReplies($pid) {
        $stream = FacebookGraphAPIAccessor::apiRequest('/'.$pid.'/posts', $this->access_token);

        if (isset($stream->data) && is_array($stream->data) && sizeof($stream->data > 0)) {
            $this->logger->logSuccess(sizeof($stream->data)." Facebook posts found for page ID $pid.",
            __METHOD__.','.__LINE__);

            $thinkup_data = $this->parseStream($stream, 'facebook page');
            $posts = $thinkup_data["posts"];

            $post_dao = DAOFactory::getDAO('PostDAO');
            $added_posts = 0;
            foreach ($posts as $post) {
                if ($post['author_username']== "" && isset($post['author_user_id'])) {
                    $commenter_object = $this->fetchUserInfo($post['author_user_id'], 'facebook',
                    'Facebook page comments');
                    if (isset($commenter_object)) {
                        $post["author_username"] = $commenter_object->full_name;
                        $post["author_fullname"] = $commenter_object->full_name;
                        $post["author_avatar"] = $commenter_object->avatar;
                    }
                }

                $added_posts = $added_posts + $post_dao->addPost($post);
                $this->logger->logInfo("Added post ID ".$post["post_id"]." on ".$post["network"].
                " for ".$post["author_username"].":".$post["post_text"], __METHOD__.','.__LINE__);
            }

            $added_users = 0;
            $users = $thinkup_data["users"];
            if (count($users) > 0) {
                foreach ($users as $user) {
                    $user["post_count"] = $post_dao->getTotalPostsByUser($user['user_id'], $user['network']);
                    $found_in = 'Facebook page stream';
                    $user_object = new User($user, $found_in);
                    $user_dao = DAOFactory::getDAO('UserDAO');
                    $user_dao->updateUser($user_object);
                    $added_users = $added_users + 1;
                }
            }
            if ($added_posts > 0 || $added_users > 0) {
                $this->logger->logUserSuccess($added_posts." post(s) added; ".$added_users." user(s) updated.",
                __METHOD__.','.__LINE__);
            } else {
                $this->logger->logUserInfo("No new page posts found.", __METHOD__.','.__LINE__);
            }
        } else {
            $this->logger->logInfo("No Facebook posts found for page ID $pid", __METHOD__.','.__LINE__);
        }
    }
Example #13
0
    public function testAddRmOldFavMaintSearch() {
        $this->logger->logInfo("in testAddRmOldFavMaintSearch", __METHOD__.','.__LINE__);
        //set plugin cfg values
        $namespace = OptionDAO::PLUGIN_OPTIONS . '-1';
        $builder2 = FixtureBuilder::build('options',
        array('namespace' => $namespace, 'option_name'=>'favs_older_pages','option_value'=>1));
        $builder3 = FixtureBuilder::build('options',
        array('namespace' => $namespace, 'option_name'=>'favs_cleanup_pages','option_value'=>3));

        $id = DAOFactory::getDAO('InstanceDAO');

        self::setUpInstanceUserAmygdala();
        $this->api->available_api_calls_for_crawler = 3;
        $this->api->to->setDataPath('webapp/plugins/twitter/tests/testdata/favs_tests/favs_stage3/');

        $tc = new TwitterCrawler($this->instance, $this->api);
        $tc->fetchInstanceUserInfo();
        $retval = $tc->cleanUpMissedFavsUnFavs();
        $this->assertEqual($retval, true);
        $this->assertEqual($this->instance->last_unfav_page_checked, 3);
        // Save instance
        if (isset($tc->user)) {
            $id->save($this->instance, $tc->user->post_count, $this->logger);
        }
        $this->instance = $id->getByUsernameOnNetwork("amygdala", "twitter");
        // check fav count
        $this->assertEqual($this->instance->owner_favs_in_system, 40);

        $this->logger->logInfo("in testAddRmOldFavMaintSearch, second traversal", __METHOD__.','.__LINE__ );
        // now add an additional older fav , remove one, and traverse again
        $this->api->available_api_calls_for_crawler = 3;
        $this->instance->last_unfav_page_checked = 2;
        $this->api->to->setDataPath('webapp/plugins/twitter/tests/testdata/favs_tests/favs_stage6/');
        $tc = new TwitterCrawler($this->instance, $this->api);
        $tc->fetchInstanceUserInfo();

        $retval = $tc->cleanUpMissedFavsUnFavs();
        $this->assertEqual($retval, true);
        // Save instance
        if (isset($tc->user)) {
            $id->save($this->instance, $tc->user->post_count, $this->logger);
        }
        $this->instance = $id->getByUsernameOnNetwork("amygdala", "twitter");
        // check fav count- should have removed 2 and added 21...
        // update: due to issue with TwitterAPI, not currently removing un-favs from database
        // $this->assertEqual($this->instance->owner_favs_in_system, 59);
        $this->assertEqual($this->instance->owner_favs_in_system, 61);
        $builder2 = null; $builder3 = null;
    }
 private function apiRequest($method, $encoded_params)
 {
     $api_call = $this->api_url . $method . implode('&', $encoded_params);
     //Don't log this for privacy reasons; it will output the API key to the log
     //$this->logger->logInfo("Bit.ly API call: $api_call", __METHOD__.','.__LINE__);
     $resp = Utils::getURLContents($api_call);
     //$this->logger->logInfo("Bit.ly API call response: ".$resp, __METHOD__.','.__LINE__);
     if ($resp != false) {
         $bit_link = json_decode($resp, true);
         return $bit_link;
     } else {
         $this->logger->logInfo("ERROR: No response from Bit.ly API", __METHOD__ . ',' . __LINE__);
         return array("expanded_url" => '', "title" => '', "clicks" => '', "error" => 'No response from Bit.ly API');
     }
 }
 /**
  * Store links.
  * @param  arr $links
  * @return int Total links stored
  */
 private function storeLinks($links)
 {
     $total_links_added = 0;
     $link_dao = DAOFactory::getDAO('LinkDAO');
     foreach ($links as $link) {
         try {
             $added_links = $link_dao->insert($link);
             $total_links_added = $total_links_added + ($added_links ? 1 : 0);
         } catch (DuplicateLinkException $e) {
             $this->logger->logInfo($link->url . " already exists in links table", __METHOD__ . ',' . __LINE__);
         } catch (DataExceedsColumnWidthException $e) {
             $this->logger->logInfo($link->url . "  data exceeds table column width", __METHOD__ . ',' . __LINE__);
         }
     }
     return $total_links_added;
 }
 private function storePostsAndAuthors($posts, $posts_source)
 {
     $total_added_posts = 0;
     $added_post = 0;
     foreach ($posts as $post) {
         $added_post_key = $this->storePostAndAuthor($post, $posts_source);
         if ($added_post !== false) {
             $this->logger->logInfo("Added post ID " . $post["post_id"] . " on " . $post["network"] . " for " . $post["author_username"] . ":" . substr($post["post_text"], 0, 20) . "...", __METHOD__ . ',' . __LINE__);
             $total_added_posts = $total_added_posts++;
         } else {
             $this->logger->logInfo("Didn't add post " . $post["post_id"] . " on " . $post["network"] . " for " . $post["author_username"] . ":" . substr($post["post_text"], 0, 20) . "...", __METHOD__ . ',' . __LINE__);
         }
         $added_post = 0;
     }
     return $total_added_posts;
 }
 /**
  * If link is an image (Twitpic/Twitgoo/Yfrog/Flickr for now), insert direct path to thumb as expanded url.
  * @TODO Move image thumbnail processng to Expand URLs plugin.
  * @param Logger $logger
  * @param str $tweet
  * @param Array $urls
  */
 public static function processTweetURLs($logger, $tweet, $urls = null)
 {
     $link_dao = DAOFactory::getDAO('LinkDAO');
     if (!$urls) {
         $urls = Post::extractURLs($tweet['post_text']);
     }
     foreach ($urls as $u) {
         $logger->logInfo("processing url: {$u}", __METHOD__ . ',' . __LINE__);
         $is_image = 0;
         $title = '';
         $eurl = '';
         if (substr($u, 0, strlen('http://twitpic.com/')) == 'http://twitpic.com/') {
             $eurl = 'http://twitpic.com/show/thumb/' . substr($u, strlen('http://twitpic.com/'));
             $is_image = 1;
         } elseif (substr($u, 0, strlen('http://yfrog.com/')) == 'http://yfrog.com/') {
             $eurl = $u . '.th.jpg';
             $is_image = 1;
         } elseif (substr($u, 0, strlen('http://twitgoo.com/')) == 'http://twitgoo.com/') {
             $eurl = 'http://twitgoo.com/show/thumb/' . substr($u, strlen('http://twitgoo.com/'));
             $is_image = 1;
         } elseif (substr($u, 0, strlen('http://picplz.com/')) == 'http://picplz.com/') {
             $eurl = $u . '/thumb/';
             $is_image = 1;
         } elseif (substr($u, 0, strlen('http://flic.kr/')) == 'http://flic.kr/') {
             $is_image = 1;
         } elseif (substr($u, 0, strlen('http://instagr.am/')) == 'http://instagr.am/') {
             // see: http://instagr.am/developer/embedding/ for reference
             // the following does a redirect to the actual jpg
             // make a check for an end slash in the url -- if it is there (likely) then adding a second
             // slash prior to the 'media' string will break the expanded url
             if ($u[strlen($u) - 1] == '/') {
                 $eurl = $u . 'media/';
             } else {
                 $eurl = $u . '/media/';
             }
             $logger->logDebug("expanded instagram URL to: " . $eurl, __METHOD__ . ',' . __LINE__);
             $is_image = 1;
         }
         if ($link_dao->insert($u, $eurl, $title, $tweet['post_id'], 'twitter', $is_image)) {
             $logger->logSuccess("Inserted " . $u . " (" . $eurl . ", " . $is_image . "), into links table", __METHOD__ . ',' . __LINE__);
         } else {
             $logger->logError("Did NOT insert " . $u . " (" . $eurl . ") into links table", __METHOD__ . ',' . __LINE__);
         }
     }
 }
Example #18
0
 /**
  * Expand Bit.ly links and recheck click count on any links less than 2 days old.
  *
  * @param str bitly api key
  * @param str bitly login name
  */
 public function acquireBitlyClickStats($api_key, $bit_login)
 {
     $this->logger->setUsername(null);
     $api_accessor = new BitlyAPIAccessor($api_key, $bit_login);
     $bitly_urls = array('http://bit.ly/', 'http://bitly.com/', 'http://j.mp/');
     foreach ($bitly_urls as $bitly_url) {
         if ($this->link_limit != 0) {
             //all short links first seen in the last 48 hours
             $bitly_links_to_update = $this->short_link_dao->getLinksToUpdate($bitly_url);
             if (count($bitly_links_to_update) > 0) {
                 $this->logger->logUserInfo(count($bitly_links_to_update) . " {$bitly_url}" . " links to acquire click stats for.", __METHOD__ . ',' . __LINE__);
             } else {
                 $this->logger->logUserInfo("There are no " . $bitly_url . " links to fetch click stats for.", __METHOD__ . ',', __LINE__);
             }
             $total_links = 0;
             $total_errors = 0;
             $total_updated = 0;
             foreach ($bitly_links_to_update as $link) {
                 $this->logger->logInfo("Getting bit.ly click stats for " . ($total_updated + 1) . " of " . count($bitly_links_to_update) . " " . $bitly_url . " links (" . $link->short_url . ")", __METHOD__ . ',' . __LINE__);
                 $link_data = $api_accessor->getBitlyLinkData($link->short_url);
                 if ($link_data["clicks"] != '') {
                     //save click total here
                     $this->short_link_dao->saveClickCount($link->short_url, $link_data["clicks"]);
                     // Save title to links table
                     if ($link_data["title"] != '') {
                         $this->link_dao->updateTitle($link->link_id, $link_data["title"]);
                     }
                     $total_links = $total_links + 1;
                     $total_updated = $total_updated + 1;
                 } elseif ($link_data["error"] != '') {
                     $this->link_dao->saveExpansionError($link->short_url, $link_data["error"]);
                     $total_errors = $total_errors + 1;
                     $total_updated = $total_updated + 1;
                 }
             }
             $this->logger->logUserSuccess($total_links . " " . $bitly_url . " link click stats acquired (" . $total_errors . " errors)", __METHOD__ . ',' . __LINE__);
         }
     }
 }
Example #19
0
 /**
  * For a given post, extract URLs and store them, including image_src if that's from a known source like Twitpic,
  * Twitgoo, Yfrog, Instagr.am.
  * @param str $post_text
  * @param int $post_id
  * @param str $network
  * @param Logger $logger
  * @param arr $urls Array of URLs, optionally set, defaults to null
  */
 public static function processPostURLs($post_text, $post_id, $network, $logger, $urls = null)
 {
     if (!$urls) {
         $urls = Post::extractURLs($post_text);
     }
     if ($urls) {
         $link_dao = DAOFactory::getDAO('LinkDAO');
         foreach ($urls as $url) {
             $logger->logInfo("Processing URL: {$url}", __METHOD__ . ',' . __LINE__);
             $image_src = self::getImageSource($url);
             //if we have an image_src, the URL is a known image source not in need of expansion
             $expanded_url = isset($image_src) ? $url : '';
             $link_array = array('url' => $url, 'expanded_url' => $expanded_url, "image_src" => $image_src, 'post_id' => $post_id, 'network' => $network);
             $link = new Link($link_array);
             if ($link_dao->insert($link)) {
                 $logger->logSuccess("Inserted " . $url . " (thumbnail " . $image_src . "), into links table", __METHOD__ . ',' . __LINE__);
             } else {
                 $logger->logError("Did NOT insert " . $url . " (thumbnail " . $image_src . ") into links table", __METHOD__ . ',' . __LINE__);
             }
         }
     }
 }
 private function apiRequest($method, $encoded_params)
 {
     $api_call = $this->api_url . $method . implode('&', $encoded_params);
     $this->logger->logInfo("Bit.ly API call: {$api_call}", __METHOD__ . ',' . __LINE__);
     //$resp = Utils::getURLContents($api_call);
     $api_call = str_replace('http://', '', $api_call);
     $api_call = str_replace('/', '_', $api_call);
     $api_call = str_replace('?', '-', $api_call);
     $api_call = str_replace('&', '-', $api_call);
     if ($this->debug) {
         echo "READING LOCAL DATA FILE: " . $this->test_data_path . $api_call . "\n";
     }
     if (file_exists($this->test_data_path . $api_call)) {
         $resp = file_get_contents($this->test_data_path . $api_call);
         $bit_link = json_decode($resp, true);
         $this->logger->logInfo("API RESPONSE DECODED: " . Utils::varDumpToString($bit_link), __METHOD__ . ',' . __LINE__);
         return $bit_link;
     } else {
         $this->logger->logInfo("ERROR: No response from Bit.ly API", __METHOD__ . ',' . __LINE__);
         return array("expanded_url" => '', "title" => '', "clicks" => '', "error" => 'No response from Bit.ly API');
     }
 }
 /**
  * @param  $content
  * @return array
  */
 private function parsePost($content)
 {
     $logger = Logger::getInstance('stream_log_location');
     $rt_string = "RT @";
     $post = array();
     $mentions = array();
     $urls = array();
     $hashtags = array();
     $entities = array();
     $user_array = array();
     try {
         $post['is_rt'] = false;
         $post['in_rt_of_user_id'] = '';
         $user = $content['user'];
         // parse info into user and post arrays
         $post['post_id'] = $content['id_str'];
         $post['author_user_id'] = $user['id_str'];
         $post['author_username'] = $user['screen_name'];
         $post['author_fullname'] = $user['name'];
         $post['author_avatar'] = $user['profile_image_url'];
         $post['author_follower_count'] = $user['followers_count'];
         $post['post_text'] = $content['text'];
         $post['is_protected'] = $user['protected'];
         $post['source'] = $content['source'];
         $post['location'] = $user['location'];
         $post['description'] = $user['description'];
         $post['url'] = $user['url'];
         $post['author_friends_count'] = $user['friends_count'];
         $post['author_post_count'] = $user['statuses_count'];
         $post['author_joined'] = gmdate("Y-m-d H:i:s", strToTime($user['created_at']));
         $post['favorited'] = $content['favorited'];
         $user_array['url'] = $user['url'];
         // for now, retain existing 'place' handling, where a place is set in the post.
         // Set new place_id field as well, and add point coord information if it exists
         $logger->logDebug("point coords: " . print_r($content['coordinates'], true), __METHOD__ . ',' . __LINE__);
         $place = $content['place'];
         if ($place != null) {
             $post['place'] = $place['full_name'];
             $post['place_id'] = $place['id'];
             if (isset($content['coordinates'])) {
                 $place['point_coords'] = $content['coordinates'];
             }
         } else {
             $post['place'] = null;
             $post['place_id'] = null;
             // it's possible for the point coords to be set even if the place is not.
             if (isset($content['coordinates'])) {
                 $place = array();
                 $place['point_coords'] = $content['coordinates'];
             }
         }
         $post['pub_date'] = gmdate("Y-m-d H:i:s", strToTime($content['created_at']));
         $post['in_reply_to_user_id'] = $content['in_reply_to_user_id_str'];
         $post['in_reply_to_post_id'] = $content['in_reply_to_status_id_str'];
         $post['network'] = 'twitter';
         $post['reply_count_cache'] = 0;
         if (isset($content['entities'])) {
             foreach ($content['entities']['user_mentions'] as $m) {
                 $mention_info = array();
                 $mention_info['user_id'] = $m['id_str'];
                 $mention_info['user_name'] = $m['screen_name'];
                 $mentions[] = $mention_info;
             }
             // get urls
             foreach ($content['entities']['urls'] as $u) {
                 // This block broken under 0.11
                 /*
                 $url_info = array();
                 $url_info['url']= $u['url'];
                 if (isset($u['expanded_url'])) {
                 $url_info['expanded_url'] = $u['expanded_url'];
                 print "expanded url for: " . $url_info['url'] . ": " . $url_info['expanded_url'] . "\n";
                 } else {
                 $url_info['expanded_url'] = '';
                 }
                 $urls[] = $url_info;
                 */
                 // just need an array of urls now...
                 if (isset($u['expanded_url'])) {
                     array_push($urls, $u['expanded_url']);
                 } else {
                     if (isset($u['url'])) {
                         array_push($urls, $u['url']);
                     }
                 }
             }
             // get hashtags
             foreach ($content['entities']['hashtags'] as $h) {
                 $hashtags[] = $h['text'];
             }
         }
         $logger->logDebug($post['post_text'] . " -- " . $post['author_username'], __METHOD__ . ',' . __LINE__);
         if (!isset($content['retweeted_status'])) {
             if (isset($content['retweet_count'])) {
                 // do this only for the original post (rt will have rt count too)
                 $retweet_count_api = $content['retweet_count'];
                 $pos = strrpos($content['retweet_count'], '+');
                 if ($pos != false) {
                     // remove '+', e.g. '100+' -- so currently 100 is max that can be indicated
                     $retweet_count_api = substr($content['retweet_count'], 0, $pos);
                 }
                 $post['retweet_count_api'] = $retweet_count_api;
                 $this->logger->logDebug($content['id_str'] . " is not a retweet but orig., count is: " . $content['retweet_count'] . "/ " . $retweet_count_api, __METHOD__ . ',' . __LINE__);
             }
             // // parse to see if 'old-style' retweet "RT @..." for first 'mention'-- if so, set that information
             if (sizeof($mentions) > 0) {
                 $first_mention = $mentions[0]['user_name'];
                 $logger->logDebug("first mention: {$first_mention}", __METHOD__ . ',' . __LINE__);
                 if (RetweetDetector::isRetweet($post['post_text'], $first_mention)) {
                     $post['is_rt'] = true;
                     $post['in_rt_of_user_id'] = $mentions[0]['user_id'];
                     $logger->logDebug("detected retweet of: " . $post['in_rt_of_user_id'] . ", " . $first_mention, __METHOD__ . ',' . __LINE__);
                 }
             }
         } else {
             // then this is a retweet.
             // Process its original too.
             $this->logger->logInfo("this is a retweet, will first process original post " . $content['retweeted_status']['id_str'] . " from user " . $content['retweeted_status']['user']['id_str'], __METHOD__ . ',' . __LINE__);
             list($orig_post, $orig_entities, $orig_user_array) = $this->parsePost($content['retweeted_status']);
             $rtp = array();
             $rtp['content'] = $orig_post;
             $rtp['entities'] = $orig_entities;
             $rtp['user_array'] = $orig_user_array;
             $post['retweeted_post'] = $rtp;
             $post['in_retweet_of_post_id'] = $content['retweeted_status']['id_str'];
             $post['in_rt_of_user_id'] = $content['retweeted_status']['user']['id_str'];
         }
         $user_array = $this->parseUser($user, $post['pub_date']);
     } catch (Exception $e) {
         $logger->logErrro("exception: {$e}", __METHOD__ . ',' . __LINE__);
     }
     $entities['urls'] = $urls;
     $entities['mentions'] = $mentions;
     $entities['hashtags'] = $hashtags;
     $entities['place'] = $place;
     // add 'place' object to entities array; may be null
     return array($post, $entities, $user_array);
 }
    function usunOpisyDlaZdjecia($katalog, $nazwaPlikuZdjecia) {   
		$zapytanie = "DELETE FROM komentarzDoZdjecia WHERE jestOpisem = 't' AND katalog = '$katalog' AND nazwaPlikuZdjecia = '$nazwaPlikuZdjecia'";

		$wynik = mysql_query($zapytanie, $this->polaczenie);
		if (!$wynik) {
			Logger::logMysqlError("MysqlKomentarzDoZdjecia->usunOpisyDlaZdjecia", $zapytanie, mysql_error($this->polaczenie));
			return false;
		} else {
			Logger::logInfo($zapytanie);
			return true;
		}
    }
Example #23
0
 /**
  * Retrieve tweets in search results for a keyword/hashtag.
  * @param InstanceHashtag $instance_hashtag
  * @return void
  */
 public function fetchInstanceHashtagTweets($instance_hashtag)
 {
     if (isset($this->instance)) {
         $status_message = "";
         $continue_fetching = true;
         $since_id = 0;
         $max_id = 0;
         $instance_hashtag_dao = DAOFactory::getDAO('InstanceHashtagDAO');
         $post_dao = DAOFactory::getDAO('PostDAO');
         $user_dao = DAOFactory::getDAO('UserDAO');
         $hashtagpost_dao = DAOFactory::getDAO('HashtagPostDAO');
         $hashtag_dao = DAOFactory::getDAO('HashtagDAO');
         //Get hashtag
         $hashtag = $hashtag_dao->getHashtagByID($instance_hashtag->hashtag_id);
         while ($continue_fetching) {
             $endpoint = $this->api->endpoints['search_tweets'];
             $args = array();
             $args["q"] = $hashtag->hashtag;
             $count_arg = isset($this->twitter_options['tweet_count_per_call']) ? $this->twitter_options['tweet_count_per_call']->option_value : 100;
             $args["count"] = $count_arg;
             $args["include_entities"] = "true";
             if ($since_id == 0) {
                 $since_id = $instance_hashtag->last_post_id;
             }
             if ($since_id > 0) {
                 $args["since_id"] = $since_id;
             }
             if ($max_id > $since_id) {
                 $args["max_id"] = $max_id;
             }
             try {
                 list($http_status, $payload) = $this->api->apiRequest($endpoint, $args);
             } catch (APICallLimitExceededException $e) {
                 $this->logger->logInfo($e->getMessage(), __METHOD__ . ',' . __LINE__);
                 break;
             }
             if ($http_status == 200) {
                 $this->logger->logDebug('Search tweets 200 ' . $endpoint->getPath(), __METHOD__ . ',' . __LINE__);
                 $count = 0;
                 $user_count = 0;
                 $tweets = $this->api->parseJSONTweetsFromSearch($payload);
                 foreach ($tweets as $tweet) {
                     $this->logger->logDebug('Processing ' . $tweet['post_id'], __METHOD__ . ',' . __LINE__);
                     $this->logger->logDebug('Processing ' . Utils::varDumpToString($tweet), __METHOD__ . ',' . __LINE__);
                     $inserted_post_key = $post_dao->addPost($tweet, $this->user, $this->logger);
                     //We need to check if post exists before add relationship between post and hashtag
                     if ($post_dao->isPostInDB($tweet['post_id'], 'twitter')) {
                         if (!$hashtagpost_dao->isHashtagPostInStorage($hashtag->id, $tweet['post_id'], 'twitter')) {
                             $count = $count + 1;
                             $hashtagpost_dao->insertHashtagPost($hashtag->hashtag, $tweet['post_id'], 'twitter');
                             $user = new User($tweet);
                             $rows_updated = $user_dao->updateUser($user);
                             if ($rows_updated > 0) {
                                 $user_count = $user_count + $rows_updated;
                             }
                             $this->logger->logDebug('User has been updated', __METHOD__ . ',' . __LINE__);
                             if (isset($tweet['retweeted_post']) && isset($tweet['retweeted_post']['content'])) {
                                 $this->logger->logDebug('Retweeted post info set', __METHOD__ . ',' . __LINE__);
                                 if (!$hashtagpost_dao->isHashtagPostInStorage($hashtag->id, $tweet['retweeted_post']['content']['post_id'], 'twitter')) {
                                     $this->logger->logDebug('Retweeted post not in storage', __METHOD__ . ',' . __LINE__);
                                     $count++;
                                     $hashtagpost_dao->insertHashtagPost($hashtag->hashtag, $tweet['retweeted_post']['content']['post_id'], 'twitter');
                                     $user_retweet = new User($tweet['retweeted_post']['content']);
                                     $rows_retweet_updated = $user_dao->updateUser($user_retweet);
                                     if ($rows_retweet_updated > 0) {
                                         $user_count = $user_count + $rows_retweet_updated;
                                     }
                                 } else {
                                     $this->logger->logDebug('Retweeted post in storage', __METHOD__ . ',' . __LINE__);
                                 }
                             } else {
                                 $this->logger->logDebug('Retweeted post info not set', __METHOD__ . ',' . __LINE__);
                             }
                             $this->logger->logDebug('About to process URLs', __METHOD__ . ',' . __LINE__);
                             URLProcessor::processPostURLs($tweet['post_text'], $tweet['post_id'], 'twitter', $this->logger);
                             $this->logger->logDebug('URLs have been processed', __METHOD__ . ',' . __LINE__);
                         }
                     }
                     if ($tweet['post_id'] > $instance_hashtag->last_post_id) {
                         $instance_hashtag->last_post_id = $tweet['post_id'];
                     }
                     if ($instance_hashtag->earliest_post_id == 0 || $tweet['post_id'] < $instance_hashtag->earliest_post_id) {
                         $instance_hashtag->earliest_post_id = $tweet['post_id'];
                     }
                     if ($max_id == 0 || $tweet['post_id'] < $max_id) {
                         $max_id = $tweet['post_id'];
                     }
                     $this->logger->logDebug('Instance hashtag markers updated', __METHOD__ . ',' . __LINE__);
                 }
                 //Status message for tweets and users
                 $status_message = ' ' . count($tweets) . " tweet(s) found and {$count} saved";
                 $this->logger->logUserSuccess($status_message, __METHOD__ . ',' . __LINE__);
                 $status_message = ' ' . count($tweets) . " tweet(s) found and {$user_count} users saved";
                 $this->logger->logUserSuccess($status_message, __METHOD__ . ',' . __LINE__);
                 //Save instance_hashtag important values
                 if ($instance_hashtag->last_post_id > 0) {
                     $instance_hashtag_dao->updateLastPostID($instance_hashtag->instance_id, $instance_hashtag->hashtag_id, $instance_hashtag->last_post_id);
                 }
                 if ($instance_hashtag->earliest_post_id > 0) {
                     $instance_hashtag_dao->updateEarliestPostID($instance_hashtag->instance_id, $instance_hashtag->hashtag_id, $instance_hashtag->earliest_post_id);
                 }
                 //Not to continue fetching if search not return the maxim number of tweets
                 if (count($tweets) < $count_arg) {
                     $continue_fetching = false;
                 }
             } else {
                 $status_message = "Stop fetching tweets. cURL_status = " . $cURL_status;
                 $this->logger->logUserSuccess($status_message, __METHOD__ . ',' . __LINE__);
                 $continue_fetching = false;
             }
         }
     }
 }
Example #24
0
 /**
  *V4.0 获取商户结果通知平台请求商户数据
  * @param $map 请求数据
  * @return	HashMap
  */
 public static function getSplitMerRefundNotifyReqData($map)
 {
     $log = new Logger();
     if ($map == null || $map->size() == 0) {
         die("获取通知数据失败:待解析的数据对象为空!");
     }
     $plain = self::getSplitMerRefundNotifyPlain($map);
     $plain = iconv("UTF-8", "GBK", $plain);
     $log->logInfo("getSplitMerRefundNotifyReqData plain=[" . $plain . "]");
     $sign = $map->get("sign");
     $log->logInfo("getSplitMerRefundNotifyReqData sign=" . $sign);
     //进行请求数据验签
     $checked = SignUtil::verify($plain, $sign);
     if (!$checked) {
         die("支付结果通知平台请求数据验签失败!");
     }
     return $map;
 }
Example #25
0
 public function onFinish($serv, $task_id, $data)
 {
     Logger::logInfo("tasking_num: [" . $serv->stats()['tasking_num'] . "] pid: [" . posix_getpid() . "] task_id: [" . $task_id . "] message: [" . $data . "]");
 }
	function usun($katalog) {
		$zapytanie = "DELETE FROM galeriaZdjec WHERE katalog = '$katalog'";

		$wynik = mysql_query($zapytanie, $this->polaczenie);
		if (!$wynik) {
			Logger::logMysqlError("MysqlGaleriaZdjec->usun", $zapytanie, mysql_error($this->polaczenie));
			return false;
		} else {
			Logger::logInfo($zapytanie);
			return true;
		}
	}
Example #27
0
 /**
  * 真正的校验逻辑,通过配置的正则表达式校验字段是否符合规范
  *
  * @param key
  *            待校验字段
  * @param dc
  * @param must
  *            字段是否是必须的
  * @throws Exception
  */
 protected static function check($key, $map, $must)
 {
     $log = new Logger();
     $value = $map->get($key);
     $tValue = $value;
     if (empty($value)) {
         // 必须字段为空,抛出异常
         if ($must) {
             die("The parameters of the request " . $key . " can't be empty");
         }
     } else {
         if (count(CheckReqDataAndEncrypt::$regexkeysarray) <= 0) {
             $log->logInfo("loading mapping_regex.ini");
             CheckReqDataAndEncrypt::$regexkeysarray = parse_ini_file("mapping_regex.ini");
         }
         $regex = CheckReqDataAndEncrypt::$regexkeysarray["Regex." . $key];
         if (empty($regex)) {
             $log->logInfo("The parameters of the request " . $key . " not configured regular validation rules");
             return;
         }
         // 不符合正则表达式
         if (!preg_match($regex, $tValue)) {
             $log->logInfo("Request parameters [" . $key . "(" . $tValue . ")] format or value can not match the regular rules [" . $regex . "]");
             die("Request parameters [" . $key . "(" . $tValue . ")] format or value is not correct");
         } else {
             $log->logInfo("Request parameters [" . $key . "(" . $tValue . ")] format or value match the regular rules [" . $regex . "]");
         }
     }
 }
 /**
  * cleanUpMissedFavsUnFavs  pages back through the older pages of favs, checking for favs that are not yet in
  * the database, as well as favs that were added to the db but are no longer returned by Twitter's API.
  * However, that latter calculation, for un-fav'd tweets, is currently not reliable due to a bug on Twitter's end,
  * and so such tweets are not currently removed from the database.
  * Due to the same issue with the API, it's not clear whether all favs of older tweets are going to be actually
  * returned from Twitter (that is, it is currently not returning some actually-favorited tweets in a given range).
  * So, we may miss some older tweets that were in fact favorited, until Twitter fixes this.
  * The number of pages to page back for each run of the crawler is set by favs_cleanup_pages option.
  */
 public function cleanUpMissedFavsUnFavs()
 {
     // first, check that we have the resources to do work
     if (!($this->api->available && $this->api->available_api_calls_for_crawler)) {
         $this->logger->logInfo("terminating cleanUpMissedFavsUnFavs-- no API calls available", __METHOD__ . ',' . __LINE__);
         return true;
     }
     $this->logger->logInfo("In cleanUpMissedFavsUnFavs", __METHOD__ . ',' . __LINE__);
     $this->logger->logInfo("User id: " . $this->user->user_id . "\n", __METHOD__ . ',' . __LINE__);
     $fcount = 0;
     $favs_cleanup_pages = 1;
     // default number of pages to process each time the crawler runs
     // get plugin option value if it exists & is positive int, otherwise use default
     $topt = $this->twitter_options;
     if (isset($topt['favs_cleanup_pages'])) {
         $conf_favs_cleanup_pages = $topt['favs_cleanup_pages']->option_value;
         $this->logger->logInfo("conf_favs_cleanup_pages: {$conf_favs_cleanup_pages} ", __METHOD__ . ',' . __LINE__);
         if (is_integer((int) $conf_favs_cleanup_pages) && $conf_favs_cleanup_pages > 0) {
             $favs_cleanup_pages = $conf_favs_cleanup_pages;
         }
     }
     $this->logger->logInfo("favs_cleanup_pages: {$favs_cleanup_pages} ", __METHOD__ . ',' . __LINE__);
     $fpd = DAOFactory::getDAO('FavoritePostDAO');
     $pagesize = 20;
     // number of favs per page retrieved from the API call... (tbd: any way to get
     //this from the API?)
     // get 'favs_older_pages' plugin option value if it exists & is pos. int.  Use it to calculate default start
     // page if set, otherwise use default value.
     $default_start_page = 2;
     $topt = $this->twitter_options;
     if (isset($topt['favs_older_pages'])) {
         $conf_older_favs_pages = $topt['favs_older_pages']->option_value;
         if (is_integer((int) $conf_older_favs_pages) && $conf_older_favs_pages > 0) {
             $default_start_page = $conf_older_favs_pages + 1;
         }
     }
     $this->logger->logInfo("default start page: {$default_start_page} ", __METHOD__ . ',' . __LINE__);
     $last_page_of_favs = round($this->api->archive_limit / $pagesize);
     $last_unfav_page_checked = $this->instance->last_unfav_page_checked;
     $start_page = $last_unfav_page_checked > 0 ? $last_unfav_page_checked + 1 : $default_start_page;
     $this->logger->logInfo("start page: {$start_page}, with {$favs_cleanup_pages} cleanup pages", __METHOD__ . ',' . __LINE__);
     $curr_favs_count = $this->user->favorites_count;
     $count = 0;
     $page = $start_page;
     while ($count < $favs_cleanup_pages && $this->api->available && $this->api->available_api_calls_for_crawler) {
         // get the favs from that page
         try {
             list($tweets, $cURL_status, $twitter_data) = $this->getFavsPage($page);
         } catch (APICallLimitExceededException $e) {
             break;
         }
         if ($cURL_status != 200 || $tweets == -1) {
             // todo - handle more informatively
             $this->logger->logInfo("in cleanUpMissedFavsUnFavs, error with: {$twitter_data}", __METHOD__ . ',' . __LINE__);
             throw new Exception("in cleanUpUnFavs: error parsing favs");
         }
         if (sizeof($tweets) == 0) {
             // then done paging backwards through the favs.
             // reset pointer so that we start at the recent favs again next time through.
             $this->instance->last_unfav_page_checked = 0;
             break;
         }
         $min_tweet = $tweets[sizeof($tweets) - 1]['post_id'];
         $max_tweet = $tweets[0]['post_id'];
         $this->logger->logInfo("in cleanUpUnFavs, page {$page} min and max: {$min_tweet}, {$max_tweet}", __METHOD__ . ',' . __LINE__);
         foreach ($tweets as $fav) {
             $fav['network'] = 'twitter';
             // check whether the tweet is in the db-- if not, add it.
             if ($fpd->addFavorite($this->user->user_id, $fav) > 0) {
                 URLProcessor::processPostURLs($fav['post_text'], $fav['post_id'], 'twitter', $this->logger);
                 $this->logger->logInfo("added fav " . $fav['post_id'], __METHOD__ . ',' . __LINE__);
                 $fcount++;
             } else {
                 $status_message = "have already stored fav " . $fav['post_id'];
                 $this->logger->logDebug($status_message, __METHOD__ . ',' . __LINE__);
             }
         }
         // now for each favorited tweet in the database within the fetched range, check whether it's still
         // favorited. This part of the method is currently disabled due to issues with the Twitter API, which
         // is not returning all of the favorited tweets any more.  So, the fact that a previously-archived
         // tweet is not returned, no longer indicates that it was un-fav'd.
         // The method still IDs the 'missing' tweets, but no longer deletes them.  We may want to get rid of
         //  this check altogether at some point.
         $fposts = $fpd->getAllFavoritePostsUpperBound($this->user->user_id, 'twitter', $pagesize, $max_tweet + 1);
         foreach ($fposts as $old_fav) {
             $old_fav_id = $old_fav->post_id;
             if ($old_fav_id < $min_tweet) {
                 $this->logger->logInfo("Old fav {$old_fav_id} out of range ", __METHOD__ . ',' . __LINE__);
                 break;
                 // all the rest will be out of range also then
             }
             // look for the old_fav_id in the array of fetched favs
             $found = false;
             foreach ($tweets as $tweet) {
                 if ($old_fav_id == $tweet['post_id']) {
                     $found = true;
                     break;
                 }
             }
             if (!$found) {
                 // if it's not there...
                 // 14/10 arghh -- Twitter is suddenly (temporarily?) not returning all fav'd tweets in a
                 // sequence.
                 // skipping the delete for now, keep tabs on it.  Can check before delete with extra API
                 // request, but the point of doing it this way was to avoid the additional API request.
                 $this->logger->logInfo("Twitter claims tweet not still favorited, but this is currently " . "broken, so not deleting: " . $old_fav_id, __METHOD__ . ',' . __LINE__);
                 // 'unfavorite' by removing from favorites table
                 // $fpd->unFavorite($old_fav_id, $this->user->user_id);
             }
         }
         $this->instance->last_unfav_page_checked = $page++;
         if ($page > $last_page_of_favs) {
             $page = 0;
             break;
         }
         $count++;
     }
     $this->logger->logUserSuccess("Added {$fcount} older missed favorites", __METHOD__ . ',' . __LINE__);
     return true;
 }
 /**
  *  Parse the JSON returned from foursquare and store them in the database
  *  @param JSON $checkins
  */
 private function parseResults($checkins)
 {
     $post_dao = DAOFactory::getDAO('PostDAO');
     $place_dao = DAOFactory::getDAO('PlaceDAO');
     $link_dao = DAOFactory::getDAO('LinkDAO');
     $number_stored = 0;
     // Check we actually got a set of checkins
     if (isset($checkins->response->checkins->items) && sizeof($checkins->response->checkins->items) > 0) {
         // Make a query out to the API for this users details, like name etc.
         $user = $this->api_accessor->apiRequest('users/self', $this->access_token);
         // For each checkin store it in the database
         foreach ($checkins->response->checkins->items as $item) {
             // The post ID, is the checkin ID foursquare provides
             $post['post_id'] = $item->id;
             // The post text is the text they enter when checking in
             $post['post_text'] = isset($item->shout) ? $item->shout : " ";
             // The author username is the users foursquare email address
             $post['author_username'] = $user->response->user->contact->email;
             // The author full name is the name they gave foursquare
             $post['author_fullname'] = $user->response->user->firstName . " " . $user->response->user->lastName;
             // The avatar is the one they have set on foursquare
             $post["author_avatar"] = $user->response->user->photo->prefix . "100x100" . $user->response->user->photo->suffix;
             // The author user id is there foursquare user ID
             $post['author_user_id'] = $user->response->user->id;
             // The date they checked in
             $post['pub_date'] = date('Y-m-d H:i:s', $item->createdAt);
             // Source of the checkin
             $post['source'] = $item->source->name;
             // Check if the checkin was marked as private
             if (isset($item->private)) {
                 $post['is_protected'] = true;
             } else {
                 $post['is_protected'] = false;
             }
             // Set the network to foursquare
             $post['network'] = 'foursquare';
             // Set place to the name of the place their checking into
             $post['place'] = $item->venue->name;
             // There are a few parameters that may or may not be set for location so let the method figure it out
             $post['location'] = $this->getLocation($item);
             //$post['location'] = $item->location->postalCode;
             // Place ID is an ID foursquare provides
             $post['place_id'] = $item->venue->id;
             // Set geo to the lat,lng that foursquare provides
             $post['geo'] = $item->venue->location->lat . "," . $item->venue->location->lng;
             // These parameters cant be null but we don't need them for our plugin so set them to any empty string
             $post['reply_count_cache'] = '';
             $post['favlike_count_cache'] = '';
             $post['retweet_count_cache'] = '';
             $post['author_follower_count'] = '';
             // Store the checkin details in the database
             $done = $post_dao->addPost($post);
             if ($done != null) {
                 $number_stored++;
             }
             // Check if any photos are attached to this checkin
             if ($item->photos->count > 0 && $done != null) {
                 foreach ($item->photos->items as $photo) {
                     $photo_store = new Link(array('url' => $photo->url, 'expanded_url' => $photo->url, 'title' => ' ', 'description' => ' ', 'image_src' => $photo->url, 'caption' => ' ', 'clicks' => 0, 'post_key' => $done, 'error' => 'none'));
                     // Insert the photo into the database
                     $link_dao->insert($photo_store);
                     // Delete the current photo info ready for the next one
                     $photo_store = null;
                 }
             }
             // If there are any comments on this checkin capture them
             if ($item->comments->count > 0) {
                 // Make a query out for the comments
                 $comments = $this->api_accessor->apiRequest('checkins/' . $item->id, $this->access_token);
                 foreach ($comments->response->checkin->comments->items as $comment) {
                     // The post ID, is the comment ID foursquare provides
                     $comment_store['post_id'] = $comment->id;
                     // The post text is the comment they made
                     $comment_store['post_text'] = $comment->text;
                     // The author username is the users foursquare email address (which we need to query for)
                     $name = $this->api_accessor->apiRequest('users/' . $comment->user->id, $this->access_token);
                     $user_name = $name->response->user->contact->email;
                     $comment_store['author_username'] = isset($user_name) ? $user_name : 'email address withheld';
                     // The author full name is the name they gave foursquare
                     $comment_store['author_fullname'] = $comment->user->firstName . " " . $comment->user->lastName;
                     // The avatar is the one they have set on foursquare
                     $comment_store["author_avatar"] = $comment->user->photo->prefix . "100x100" . $comment->user->photo->suffix;
                     // The author user id is there foursquare user ID
                     $comment_store['author_user_id'] = $comment->user->id;
                     // The date they posted the comment
                     $comment_store['pub_date'] = date('Y-m-d H:i:s', $comment->createdAt);
                     // Source of the comment
                     $comment_store['source'] = "";
                     // Comments can not be private
                     $comment_store['is_protected'] = false;
                     // Set the network to foursquare
                     $comment_store['network'] = 'foursquare';
                     // Set place to the name of the place the comment is about
                     $comment_store['place'] = $comments->response->checkin->venue->name;
                     // A few parameters may or may not be set for location so let the method do the work
                     $comment_store['location'] = $this->getLocation($item);
                     // Place ID is an ID foursquare provides
                     $comment_store['place_id'] = $comments->response->checkin->venue->id;
                     // Set geo to the lat,lng that foursquare provides
                     $comment_store['geo'] = $item->venue->location->lat . "," . $item->venue->location->lng;
                     // The ID of the author of the checkin
                     $comment_store['in_reply_to_user_id'] = $user->response->user->id;
                     // The ID of the checkin this is a reply to
                     $comment_store['in_reply_to_post_id'] = $item->id;
                     // The number of replies this checkin has
                     $comment_store['reply_count_cache'] = $item->comments->count;
                     // These parameters cant be null but we don't need them so set them to any empty string
                     $comment_store['reply_count_cache'] = '';
                     $comment_store['favlike_count_cache'] = '';
                     $comment_store['retweet_count_cache'] = '';
                     $comment_store['author_follower_count'] = '';
                     self::fetchUser($comment_store['author_user_id'], 'comment');
                     // Now store the comment in the database
                     $post_dao->addPost($comment_store);
                     $comment = null;
                 }
             }
             // Store the details about this place in the place table if it doesn't already exist
             // See if this place is already in the database
             $place_test = $place_dao->getPlaceByID($item->venue->id);
             // If it isn't already in the database
             if ($place_test == null) {
                 // Insert it
                 $places['id'] = $item->venue->id;
                 $places['place_type'] = $item->venue->categories[0]->name;
                 $places['name'] = $item->venue->name;
                 $places['full_name'] = $item->venue->name;
                 $places['icon'] = $item->venue->categories[0]->icon->prefix . '64' . $item->venue->categories[0]->icon->suffix;
                 $places['lat_lng'] = 'POINT(' . $item->venue->location->lat . " " . $item->venue->location->lng . ')';
                 $places['map_image'] = $this->generateMap($item);
                 $place_dao->insertGenericPlace($places, 'foursquare');
             }
             // Blank out the details ready for the next checkin
             $post = null;
             $places = null;
         }
     } else {
         $this->logger->logInfo("No checkins found " . Utils::varDumpToString($checkins));
     }
 }
Example #30
0
 /**
  * @param $data
  */
 public function batchUpdate($data = array())
 {
     if ($data && is_array($data)) {
         try {
             $docs =& $this->_getBatchBuff('update');
             if (count($docs) > self::MAXBATCHNUM) {
                 $c = $this->_getColctObj();
                 $batch = new MongoUpdateBatch($c);
                 foreach ($docs as $doc) {
                     $batch->add((object) $doc);
                 }
                 $batch->execute();
                 $docs = array();
             } else {
                 if (isset($data['userkey'])) {
                     $update = array('q' => array('userkey' => $data['userkey']), 'u' => array('$set' => $data), 'multi' => false, 'upsert' => true);
                     array_push($docs, $update);
                 }
             }
         } catch (MongoException $e) {
             Logger::logInfo('mongo in lib batchupdate failed, message [' . $e->getMessage() . '] code: [' . $e->getCode() . ']');
             return false;
         }
     } else {
         Logger::logInfo('data of batchupdate is invaild');
         return false;
     }
 }