/** * Make a Graph API request with the absolute URL. This URL needs to include the https://graph.facebook.com/ at * the start and all the query string parameters EXCEPT the acces token. * * This is for use in paging, when the API payload specifies the full URL for the next page. * * @param str $url * @param str $access_token * @return array Decoded JSON response */ public static function apiRequestFullURL($url, $access_token = null) { $params = array(); if (isset($access_token)) { //Add access_token $params['access_token'] = $access_token; $access_token_str = http_build_query($params); if (strpos($url, '?') === false) { $url = $url . '?' . $access_token_str; } else { $url = $url . '&' . $access_token_str; } } //DEBUG // if (php_sapi_name() == "cli") {//Crawler being run at the command line // Logger::getInstance()->logInfo("Graph API call: ".$url, __METHOD__.','.__LINE__); // } $result = Utils::getURLContents($url); try { // if (php_sapi_name() == "cli") {//Crawler being run at the command line // Logger::getInstance()->logInfo("Graph API call payload: ".$result, __METHOD__.','.__LINE__); // } return JSONDecoder::decode($result); } catch (JSONDecoderException $e) { return $result; } }
/** Make a foursquare API request. * @param str $endpoint API endpoint to access * @param str $access_token OAuth token for the user * @param arr $fields Array of URL parameters * @return array Decoded JSON response */ public function apiRequest($endpoint, $access_token, $fields = null) { // Add endpoint and OAuth token to the URL $url = $this->api_domain . $endpoint . '?oauth_token=' . $access_token; // If there are additional parameters passed in add them to the URL also if ($fields != null) { foreach ($fields as $key => $value) { $url = $url . '&' . $key . '=' . $value; } } // Foursquare requires us to add the date at the end of the request so get a date array $date = getdate(); // Add the year month and day at the end of the URL $url = $url . "&v=" . $date['year'] . $date['mon'] . $date['mday']; // Get any results returned from this request $result = Utils::getURLContents($url); // Return the results $parsed_result = json_decode($result); if (isset($parsed_result->meta->errorType)) { $exception_description = $parsed_result->meta->errorType; if (isset($parsed_result->meta->errorDetail)) { $exception_description .= ": " . $parsed_result->meta->errorDetail; } throw new Exception($exception_description); } else { return $parsed_result; } }
/** * Make a Graph API request with the absolute URL. This URL needs to include the https://graph.facebook.com/ at * the start and the access token at the end as well as everything in between. It is literally the raw URL that * needs to be passed in. * * @param str $path * @param book $decode_json Defaults to true, if true returns decoded JSON * @return array Decoded JSON response */ public static function rawApiRequest($path, $decode_json = true) { if ($decode_json) { $result = Utils::getURLContents($path); return json_decode($result); } else { return Utils::getURLContents($path); } }
/** * Make a Graph API request. * @param str $path * @param str $access_token * @return array Decoded JSON response */ public static function apiRequest($path, $access_token, $fields = null) { $api_domain = 'https://graph.facebook.com'; $url = $api_domain . $path . '?access_token=' . $access_token; if ($fields != null) { $url = $url . '&fields=' . $fields; } $result = Utils::getURLContents($url); return json_decode($result); }
/** * Make an API request. * @param str $path * @param str $access_token * @param arr $fields Array of URL parameters * @return array Decoded JSON response */ public function apiRequest($path, $access_token, $fields = null) { $url = $this->api_domain . $path . '?access_token=' . $access_token; if ($fields != null) { foreach ($fields as $key => $value) { $url = $url . '&' . $key . '=' . $value; } } $result = Utils::getURLContents($url); return json_decode($result); }
public function getFlickrPhotoSource($u) { if ($this->api_key != '') { $this->logger->logInfo("Flickr API key set", __METHOD__.','.__LINE__); $photo_short_id = substr($u, strlen('http://flic.kr/p/')); $photo_id = $this->base_decode($photo_short_id); $params = array('method'=>$this->method, 'photo_id'=>$photo_id, 'api_key'=>$this->api_key, 'format'=>$this->format, ); $encoded_params = array(); foreach ($params as $k=>$v) { $encoded_params[] = urlencode($k).'='.urlencode($v); } $api_call = $this->api_url.implode('&', $encoded_params); $this->logger->logInfo("Flickr API call: $api_call", __METHOD__.','.__LINE__); $resp = Utils::getURLContents($api_call); if ($resp != false) { $fphoto = unserialize($resp); if ($fphoto['stat'] == 'ok') { $src = ''; foreach ($fphoto['sizes']['size'] as $s) { if ($s['label'] == 'Small') { $src = $s['source']; } } return array("expanded_url"=>$src, "error"=>''); } else { $this->logger->logInfo("ERROR: '".$fphoto['message']."'", __METHOD__.','.__LINE__); return array("expanded_url"=>'', "error"=>$fphoto['message']); } } else { $this->logger->logInfo("ERROR: No response from Flickr API", __METHOD__.','.__LINE__); return array("expanded_url"=>'', "error"=>'No response from Flickr API'); } } else { $this->logger->logInfo("ERROR: Flickr API key is not set", __METHOD__.','.__LINE__); return array("expanded_url"=>'', "error"=>''); } }
/** * If link is an image (Twitpic/Twitgoo/Yfrog/Flickr for now), insert direct path to thumb as expanded url. * @TODO (from TwitterCrawler version): Abstract out this image thumbnail link expansion into a plugin * modeled after the Flickr Thumbnails plugin * @param Logger $logger * @param str $tweet * @param Array $urls */ public static function processTweetURLs($logger, $tweet, $urls = null) { $link_dao = DAOFactory::getDAO('LinkDAO'); if (!$urls) { $urls = Post::extractURLs($tweet['post_text']); } foreach ($urls as $u) { $logger->logInfo("processing url: $u", __METHOD__.','.__LINE__); $is_image = 0; $title = ''; $eurl = ''; if (substr($u, 0, strlen('http://twitpic.com/')) == 'http://twitpic.com/') { $eurl = 'http://twitpic.com/show/thumb/'.substr($u, strlen('http://twitpic.com/')); $is_image = 1; } elseif (substr($u, 0, strlen('http://yfrog.com/')) == 'http://yfrog.com/') { $eurl = $u.'.th.jpg'; $is_image = 1; } elseif (substr($u, 0, strlen('http://twitgoo.com/')) == 'http://twitgoo.com/') { $eurl = 'http://twitgoo.com/show/thumb/'.substr($u, strlen('http://twitgoo.com/')); $is_image = 1; } elseif (substr($u, 0, strlen('http://picplz.com/')) == 'http://picplz.com/') { $eurl = $u.'/thumb/'; $is_image = 1; } elseif (substr($u, 0, strlen('http://flic.kr/')) == 'http://flic.kr/') { $is_image = 1; } elseif (substr($u, 0, strlen('http://instagr.am/')) == 'http://instagr.am/') { $logger->logInfo("processing instagram url: $u", __METHOD__.','.__LINE__); $html = (string) Utils::getURLContents($u); list($eurl, $is_image) = self::extractInstagramImageURL($logger, $html); } if ($link_dao->insert($u, $eurl, $title, $tweet['post_id'], 'twitter', $is_image)) { $logger->logSuccess("Inserted ".$u." (".$eurl.", ".$is_image."), into links table", __METHOD__.','.__LINE__); } else { $logger->logError("Did NOT insert ".$u." (".$eurl.") into links table", __METHOD__.','.__LINE__); } } }
/** * Retrieve data for Reverse Geoencoding * @var float $latitude * @var float $longitude * @return string $filecontents */ public function getDataForReverseGeoencoding($latlng) { $latlng = explode(' ', $latlng, 2); $url = "http://maps.google.com/maps/api/geocode/json?latlng=$latlng[0],$latlng[1]&sensor=true"; $filecontents=Utils::getURLContents($url); return $filecontents; }
/** * Make a Graph API request with the absolute URL. This URL needs to * include the https://graph.facebook.com/ at the start and the * access token at the end as well as everything in between. It is * literally the raw URL that needs to be passed in. * * @param str $path * @return array Decoded JSON response */ public static function rawApiRequest($path) { $result = Utils::getURLContents($path); return json_decode($result); }
private function apiRequest($method, $encoded_params) { $api_call = $this->api_url . $method . implode('&', $encoded_params); //Don't log this for privacy reasons; it will output the API key to the log //$this->logger->logInfo("Bit.ly API call: $api_call", __METHOD__.','.__LINE__); $resp = Utils::getURLContents($api_call); //$this->logger->logInfo("Bit.ly API call response: ".$resp, __METHOD__.','.__LINE__); if ($resp != false) { $bit_link = json_decode($resp, true); return $bit_link; } else { $this->logger->logInfo("ERROR: No response from Bit.ly API", __METHOD__ . ',' . __LINE__); return array("expanded_url" => '', "title" => '', "clicks" => '', "error" => 'No response from Bit.ly API'); } }
/** * Abstraction for pulling data from a file or url * @throws exception * @param str $url * @return request response data */ private function fetchURLData($url) { if (strpos($url, "/") == 0 || strpos($url, ".") == 0) { // we are a file path, so use file_get_contents $contents = file_get_contents($url); } else { // else we are a url, so use our Util::getURLContents $contents = Utils::getURLContents(URLProcessor::getFinalURL($url)); } if (is_null($contents)) { $contents = false; } return $contents; }
/** * Retrieve data for reverse geoencoding * @var float $latitude * @var float $longitude * @return string $filecontents */ public function getDataForReverseGeoencoding($latlng) { $latlng = explode(' ', $latlng, 2); if (isset($latlng[0]) && isset($latlng[1])) { $url = "http://maps.google.com/maps/api/geocode/json?latlng={$latlng['0']},{$latlng['1']}&sensor=true"; $filecontents = Utils::getURLContents($url); return $filecontents; } else { return ''; } }
<?php require_once 'init.php'; //$adr = urldecode("Centrum plaza Gurgaon"); $url = "https://maps.googleapis.com/maps/api/geocode/json?address=Centrum+plaza+Gurgaon&sensor=false"; //$url = "https://www.google.co.in/?gfe_rd=cr&ei=S765VaSrGoLC8gelhquwDw&gws_rd=ssl#q=curl"; $result = Utils::getURLContents($url); print_r($result);
/** * Make an API request to the URL passed in, no processing of the URL is done * @param str $path * @return array Decoded JSON response */ public function basicApiRequest($url) { $result = Utils::getURLContents($url); return json_decode($result); }
/** * Save direct link to Instagr.am images in data store. */ public function expandInstagramImageURLs() { $logger = Logger::getInstance(); $link_dao = DAOFactory::getDAO('LinkDAO'); $insta_links_to_expand = $link_dao->getLinksToExpandByURL('http://instagr.am/'); if (count($insta_links_to_expand) > 0) { $logger->logUserInfo(count($insta_links_to_expand) . " Instagr.am links to expand.", __METHOD__ . ',' . __LINE__); } else { $logger->logUserInfo("There are no Instagr.am thumbnails to expand.", __METHOD__ . ',' . __LINE__); } $total_thumbnails = 0; $total_errors = 0; $eurl = ''; foreach ($insta_links_to_expand as $il) { $html = (string) Utils::getURLContents($il); if ($html) { preg_match('/<meta property="og:image" content="[^"]+"\\/>/i', $html, $matches); if (isset($matches[0])) { $eurl = substr($matches[0], 35, -3); $logger->logDebug("Got instagr.am expanded URL: {$eurl}", __METHOD__ . ',' . __LINE__); } if ($eurl != '') { $link_dao->saveExpandedUrl($il, $eurl, '', 1); $total_thumbnails = $total_thumbnails + 1; } else { $total_errors = $total_errors + 1; } } } $logger->logUserSuccess($total_thumbnails . " Instagr.am thumbnails expanded (" . $total_errors . " errors)", __METHOD__ . ',' . __LINE__); }
/** * Convert the post as it is returned from the database to how it looks when output by the Twitter API. * Also, add the replies into the post with the index "replies". * * If the $post parameter is not a Post object, the function returns null. * * @param Post $post The post object. * @return stdObject The post formatted to look like the Twitter API. */ private function convertPostToTweet($post) { if (!$post instanceof Post) { return null; } if ($this->include_replies) { /* * Get all replies to the post. The limit is set to 0 because if the count is not greater than 0, * the method returns all replies. */ $replies = $this->post_dao->getRepliesToPost($post->post_id, $post->network, $this->order_by, $this->unit, $this->is_public, 0); // if replies exist for this post if ($replies) { // recursively scan through the post replies, converting them foreach ($replies as $reply) { $reply = $this->convertPostToTweet($reply); } // add the replies to the post $post->replies = $replies; } } /* * Chop and changing the data fetched from the database to look more like the official Twitter API. */ $post->text = $post->post_text; $post->created_at = strftime('%a %b %d %T %z %Y', strtotime($post->pub_date)); $post->id = $post->post_id; $post->favorited = $post->favorited ? true : false; $post->annotations = null; // to be implemented at some point $post->truncated = false; // always false $post->protected = $post->is_protected == 0 ? false : true; if ($post->geo != null) { $coordinates = preg_split('/(, |,| )/', $post->geo); $post->geo = new stdClass(); $post->geo->coordinates = $coordinates; if (!isset($post->coordinates)) { $post->coordinates = new stdClass(); } $post->coordinates->coordinates = $coordinates; } /* * SET THINKUP METADATA */ $post->thinkup = new stdClass(); $post->thinkup->retweet_count_cache = $post->retweet_count_cache; $post->thinkup->retweet_count_api = $post->retweet_count_api; $post->thinkup->reply_count_cache = $post->reply_count_cache; $post->thinkup->old_retweet_count_cache = $post->old_retweet_count_cache; $post->thinkup->is_geo_encoded = $post->is_geo_encoded; $user = $this->user_dao->getUserByName($post->author_username, $post->network); /* * Occasionally you run into users you haven't fetched yet. Bypass this code if you find one of them. */ if ($user != null) { if (!$this->trim_user) { $post->user = $this->convertUserToStdClass($user); $post->user->id = $post->user->user_id; $post->user->followers_count = $post->user->follower_count; $post->user->profile_image_url = $post->user->avatar; $post->user->name = $post->user->full_name; $post->user->screen_name = $post->user->username; $post->user->statuses_count = $post->user->post_count; $post->user->created_at = strftime('%a %b %d %T %z %Y', strtotime($post->user->joined)); $post->user->favorites_count = $post->user->favorites_count; if (isset($post->user->other)) { if (isset($post->user->other['avg_tweets_per_day'])) { $post->user->avg_tweets_per_day = $post->user->other['avg_tweets_per_day']; } } $post->user->thinkup = new stdClass(); $post->user->thinkup->last_post = $post->user->last_post; $post->user->thinkup->last_post_id = $post->user->last_post_id; $post->user->thinkup->found_in = $post->user->found_in; } else { $post->user = new stdClass(); $post->user->id = $user->user_id; } } if ($this->include_entities) { /* * Gather hashtags and format them into a Tweet entity. */ $extracted_hashtags = Post::extractHashtags($post->text); if (!isset($post->entities)) { $post->entities = new stdClass(); } $post->entities->hashtags = array(); if (!empty($extracted_hashtags)) { foreach ($extracted_hashtags as $hashtag_text) { $hashtag = new stdClass(); $hashtag->text = str_replace('#', '', $hashtag_text); $hashtag->indices[] = stripos($post->text, $hashtag_text); $hashtag->indices[] = strlen($hashtag_text) + $hashtag->indices[0]; $post->entities->hashtags[] = $hashtag; } } /* * Gather mentions and format them into a Tweet entity. */ $mentions = Post::extractMentions($post->text); if (!isset($post->entities)) { $post->entities = new stdClass(); } $post->entities->user_mentions = array(); if (!empty($mentions)) { foreach ($mentions as $username) { $mentioned_user = $this->user_dao->getUserByName(str_replace('@', '', $username), $user->network); $mention = new stdClass(); if (is_null($mentioned_user)) { // skip this for now, probably not a good idea continue; /* * If the user is not in our own ThinkUp database, a Twitter API call needs to be * made to fill in the missing details. * * Not 100% sure if this is a good idea but it works. */ $user_api_call = json_decode(Utils::getURLContents('https://api.twitter.com/1/users/show.json?screen_name=' . $username)); $mention->name = $user_api_call->name; $mention->id = $user_api_call->id; $mention->screen_name = $user_api_call->screen_name; } else { $mention->name = $mentioned_user->full_name; $mention->id = $mentioned_user->user_id; $mention->screen_name = $mentioned_user->username; } $mention->indices = array(); $mention->indices[] = stripos($post->text, $username); $mention->indices[] = strlen($username) + $mention->indices[0]; $post->entities->user_mentions[] = $mention; } } } if ($post->in_retweet_of_post_id != null) { $post->retweeted_status = $this->post_dao->getPost($post->in_retweet_of_post_id, $user->network); $post->retweeted_status = $this->convertPostToTweet($post->retweeted_status); } /* * Unset no-longer-used variables in this post; mostly variables that have been moved to more * Twtter like locations / naming conventions. */ unset($post->post_id, $post->pub_date, $post->network, $post->post_text, $post->author, $post->author_fullname, $post->author_username, $post->author_user_id, $post->author_avatar, $post->adj_pub_date, $post->user->follower_count, $post->user->is_protected, $post->user->network, $post->user->avatar, $post->user->full_name, $post->user->username, $post->user->user_id, $post->user->post_count, $post->user->joined, $post->user->favorites_count, $post->user->other, $post->link, $post->in_retweet_of_post_id, $post->in_retweet_of_user_id, $post->retweet_count_cache, $post->reply_count_cache, $post->old_retweet_count_cache, $post->is_geo_encoded, $post->rt_threshold, $post->is_protected, $post->user->last_post, $post->user->last_post_id, $post->user->found_in); return $post; }
public static function processLocation($data) { $config = Config::getInstance(); if (isset($data)) { if ($data['location_type'] == 0) { $adr = $data['name']; $url = $config->getValue('gmapJson') . $adr . "&sensor=false"; $content = Utils::getURLContents($url); $arr = json_decode($content); var_dump($arr); } } }