/**
* Make a Graph API request with the absolute URL. This URL needs to include the https://graph.facebook.com/ at
* the start and all the query string parameters EXCEPT the acces token.
*
* This is for use in paging, when the API payload specifies the full URL for the next page.
*
* @param str $url
* @param str $access_token
* @return array Decoded JSON response
*/
public static function apiRequestFullURL($url, $access_token = null)
{
$params = array();
if (isset($access_token)) {
//Add access_token
$params['access_token'] = $access_token;
$access_token_str = http_build_query($params);
if (strpos($url, '?') === false) {
$url = $url . '?' . $access_token_str;
} else {
$url = $url . '&' . $access_token_str;
}
}
//DEBUG
// if (php_sapi_name() == "cli") {//Crawler being run at the command line
// Logger::getInstance()->logInfo("Graph API call: ".$url, __METHOD__.','.__LINE__);
// }
$result = Utils::getURLContents($url);
try {
// if (php_sapi_name() == "cli") {//Crawler being run at the command line
// Logger::getInstance()->logInfo("Graph API call payload: ".$result, __METHOD__.','.__LINE__);
// }
return JSONDecoder::decode($result);
} catch (JSONDecoderException $e) {
return $result;
}
}
/** Make a foursquare API request.
* @param str $endpoint API endpoint to access
* @param str $access_token OAuth token for the user
* @param arr $fields Array of URL parameters
* @return array Decoded JSON response
*/
public function apiRequest($endpoint, $access_token, $fields = null)
{
// Add endpoint and OAuth token to the URL
$url = $this->api_domain . $endpoint . '?oauth_token=' . $access_token;
// If there are additional parameters passed in add them to the URL also
if ($fields != null) {
foreach ($fields as $key => $value) {
$url = $url . '&' . $key . '=' . $value;
}
}
// Foursquare requires us to add the date at the end of the request so get a date array
$date = getdate();
// Add the year month and day at the end of the URL
$url = $url . "&v=" . $date['year'] . $date['mon'] . $date['mday'];
// Get any results returned from this request
$result = Utils::getURLContents($url);
// Return the results
$parsed_result = json_decode($result);
if (isset($parsed_result->meta->errorType)) {
$exception_description = $parsed_result->meta->errorType;
if (isset($parsed_result->meta->errorDetail)) {
$exception_description .= ": " . $parsed_result->meta->errorDetail;
}
throw new Exception($exception_description);
} else {
return $parsed_result;
}
}
/**
* Make a Graph API request with the absolute URL. This URL needs to include the https://graph.facebook.com/ at
* the start and the access token at the end as well as everything in between. It is literally the raw URL that
* needs to be passed in.
*
* @param str $path
* @param book $decode_json Defaults to true, if true returns decoded JSON
* @return array Decoded JSON response
*/
public static function rawApiRequest($path, $decode_json = true)
{
if ($decode_json) {
$result = Utils::getURLContents($path);
return json_decode($result);
} else {
return Utils::getURLContents($path);
}
}
/**
* Make a Graph API request.
* @param str $path
* @param str $access_token
* @return array Decoded JSON response
*/
public static function apiRequest($path, $access_token, $fields = null)
{
$api_domain = 'https://graph.facebook.com';
$url = $api_domain . $path . '?access_token=' . $access_token;
if ($fields != null) {
$url = $url . '&fields=' . $fields;
}
$result = Utils::getURLContents($url);
return json_decode($result);
}
/**
* Make an API request.
* @param str $path
* @param str $access_token
* @param arr $fields Array of URL parameters
* @return array Decoded JSON response
*/
public function apiRequest($path, $access_token, $fields = null)
{
$url = $this->api_domain . $path . '?access_token=' . $access_token;
if ($fields != null) {
foreach ($fields as $key => $value) {
$url = $url . '&' . $key . '=' . $value;
}
}
$result = Utils::getURLContents($url);
return json_decode($result);
}
public function getFlickrPhotoSource($u) {
if ($this->api_key != '') {
$this->logger->logInfo("Flickr API key set", __METHOD__.','.__LINE__);
$photo_short_id = substr($u, strlen('http://flic.kr/p/'));
$photo_id = $this->base_decode($photo_short_id);
$params = array('method'=>$this->method, 'photo_id'=>$photo_id, 'api_key'=>$this->api_key,
'format'=>$this->format, );
$encoded_params = array();
foreach ($params as $k=>$v) {
$encoded_params[] = urlencode($k).'='.urlencode($v);
}
$api_call = $this->api_url.implode('&', $encoded_params);
$this->logger->logInfo("Flickr API call: $api_call", __METHOD__.','.__LINE__);
$resp = Utils::getURLContents($api_call);
if ($resp != false) {
$fphoto = unserialize($resp);
if ($fphoto['stat'] == 'ok') {
$src = '';
foreach ($fphoto['sizes']['size'] as $s) {
if ($s['label'] == 'Small') {
$src = $s['source'];
}
}
return array("expanded_url"=>$src, "error"=>'');
} else {
$this->logger->logInfo("ERROR: '".$fphoto['message']."'", __METHOD__.','.__LINE__);
return array("expanded_url"=>'', "error"=>$fphoto['message']);
}
} else {
$this->logger->logInfo("ERROR: No response from Flickr API", __METHOD__.','.__LINE__);
return array("expanded_url"=>'', "error"=>'No response from Flickr API');
}
} else {
$this->logger->logInfo("ERROR: Flickr API key is not set", __METHOD__.','.__LINE__);
return array("expanded_url"=>'', "error"=>'');
}
}
/**
* If link is an image (Twitpic/Twitgoo/Yfrog/Flickr for now), insert direct path to thumb as expanded url.
* @TODO (from TwitterCrawler version): Abstract out this image thumbnail link expansion into a plugin
* modeled after the Flickr Thumbnails plugin
* @param Logger $logger
* @param str $tweet
* @param Array $urls
*/
public static function processTweetURLs($logger, $tweet, $urls = null) {
$link_dao = DAOFactory::getDAO('LinkDAO');
if (!$urls) {
$urls = Post::extractURLs($tweet['post_text']);
}
foreach ($urls as $u) {
$logger->logInfo("processing url: $u", __METHOD__.','.__LINE__);
$is_image = 0;
$title = '';
$eurl = '';
if (substr($u, 0, strlen('http://twitpic.com/')) == 'http://twitpic.com/') {
$eurl = 'http://twitpic.com/show/thumb/'.substr($u, strlen('http://twitpic.com/'));
$is_image = 1;
} elseif (substr($u, 0, strlen('http://yfrog.com/')) == 'http://yfrog.com/') {
$eurl = $u.'.th.jpg';
$is_image = 1;
} elseif (substr($u, 0, strlen('http://twitgoo.com/')) == 'http://twitgoo.com/') {
$eurl = 'http://twitgoo.com/show/thumb/'.substr($u, strlen('http://twitgoo.com/'));
$is_image = 1;
} elseif (substr($u, 0, strlen('http://picplz.com/')) == 'http://picplz.com/') {
$eurl = $u.'/thumb/';
$is_image = 1;
} elseif (substr($u, 0, strlen('http://flic.kr/')) == 'http://flic.kr/') {
$is_image = 1;
} elseif (substr($u, 0, strlen('http://instagr.am/')) == 'http://instagr.am/') {
$logger->logInfo("processing instagram url: $u", __METHOD__.','.__LINE__);
$html = (string) Utils::getURLContents($u);
list($eurl, $is_image) = self::extractInstagramImageURL($logger, $html);
}
if ($link_dao->insert($u, $eurl, $title, $tweet['post_id'], 'twitter', $is_image)) {
$logger->logSuccess("Inserted ".$u." (".$eurl.", ".$is_image."), into links table",
__METHOD__.','.__LINE__);
} else {
$logger->logError("Did NOT insert ".$u." (".$eurl.") into links table", __METHOD__.','.__LINE__);
}
}
}
/**
* Retrieve data for Reverse Geoencoding
* @var float $latitude
* @var float $longitude
* @return string $filecontents
*/
public function getDataForReverseGeoencoding($latlng) {
$latlng = explode(' ', $latlng, 2);
$url = "http://maps.google.com/maps/api/geocode/json?latlng=$latlng[0],$latlng[1]&sensor=true";
$filecontents=Utils::getURLContents($url);
return $filecontents;
}
/**
* Make a Graph API request with the absolute URL. This URL needs to
* include the https://graph.facebook.com/ at the start and the
* access token at the end as well as everything in between. It is
* literally the raw URL that needs to be passed in.
*
* @param str $path
* @return array Decoded JSON response
*/
public static function rawApiRequest($path) {
$result = Utils::getURLContents($path);
return json_decode($result);
}
private function apiRequest($method, $encoded_params)
{
$api_call = $this->api_url . $method . implode('&', $encoded_params);
//Don't log this for privacy reasons; it will output the API key to the log
//$this->logger->logInfo("Bit.ly API call: $api_call", __METHOD__.','.__LINE__);
$resp = Utils::getURLContents($api_call);
//$this->logger->logInfo("Bit.ly API call response: ".$resp, __METHOD__.','.__LINE__);
if ($resp != false) {
$bit_link = json_decode($resp, true);
return $bit_link;
} else {
$this->logger->logInfo("ERROR: No response from Bit.ly API", __METHOD__ . ',' . __LINE__);
return array("expanded_url" => '', "title" => '', "clicks" => '', "error" => 'No response from Bit.ly API');
}
}
/**
* Abstraction for pulling data from a file or url
* @throws exception
* @param str $url
* @return request response data
*/
private function fetchURLData($url)
{
if (strpos($url, "/") == 0 || strpos($url, ".") == 0) {
// we are a file path, so use file_get_contents
$contents = file_get_contents($url);
} else {
// else we are a url, so use our Util::getURLContents
$contents = Utils::getURLContents(URLProcessor::getFinalURL($url));
}
if (is_null($contents)) {
$contents = false;
}
return $contents;
}
/**
* Retrieve data for reverse geoencoding
* @var float $latitude
* @var float $longitude
* @return string $filecontents
*/
public function getDataForReverseGeoencoding($latlng)
{
$latlng = explode(' ', $latlng, 2);
if (isset($latlng[0]) && isset($latlng[1])) {
$url = "http://maps.google.com/maps/api/geocode/json?latlng={$latlng['0']},{$latlng['1']}&sensor=true";
$filecontents = Utils::getURLContents($url);
return $filecontents;
} else {
return '';
}
}
<?php
require_once 'init.php';
//$adr = urldecode("Centrum plaza Gurgaon");
$url = "https://maps.googleapis.com/maps/api/geocode/json?address=Centrum+plaza+Gurgaon&sensor=false";
//$url = "https://www.google.co.in/?gfe_rd=cr&ei=S765VaSrGoLC8gelhquwDw&gws_rd=ssl#q=curl";
$result = Utils::getURLContents($url);
print_r($result);
/**
* Make an API request to the URL passed in, no processing of the URL is done
* @param str $path
* @return array Decoded JSON response
*/
public function basicApiRequest($url)
{
$result = Utils::getURLContents($url);
return json_decode($result);
}
/**
* Save direct link to Instagr.am images in data store.
*/
public function expandInstagramImageURLs()
{
$logger = Logger::getInstance();
$link_dao = DAOFactory::getDAO('LinkDAO');
$insta_links_to_expand = $link_dao->getLinksToExpandByURL('http://instagr.am/');
if (count($insta_links_to_expand) > 0) {
$logger->logUserInfo(count($insta_links_to_expand) . " Instagr.am links to expand.", __METHOD__ . ',' . __LINE__);
} else {
$logger->logUserInfo("There are no Instagr.am thumbnails to expand.", __METHOD__ . ',' . __LINE__);
}
$total_thumbnails = 0;
$total_errors = 0;
$eurl = '';
foreach ($insta_links_to_expand as $il) {
$html = (string) Utils::getURLContents($il);
if ($html) {
preg_match('/<meta property="og:image" content="[^"]+"\\/>/i', $html, $matches);
if (isset($matches[0])) {
$eurl = substr($matches[0], 35, -3);
$logger->logDebug("Got instagr.am expanded URL: {$eurl}", __METHOD__ . ',' . __LINE__);
}
if ($eurl != '') {
$link_dao->saveExpandedUrl($il, $eurl, '', 1);
$total_thumbnails = $total_thumbnails + 1;
} else {
$total_errors = $total_errors + 1;
}
}
}
$logger->logUserSuccess($total_thumbnails . " Instagr.am thumbnails expanded (" . $total_errors . " errors)", __METHOD__ . ',' . __LINE__);
}
/**
* Convert the post as it is returned from the database to how it looks when output by the Twitter API.
* Also, add the replies into the post with the index "replies".
*
* If the $post parameter is not a Post object, the function returns null.
*
* @param Post $post The post object.
* @return stdObject The post formatted to look like the Twitter API.
*/
private function convertPostToTweet($post)
{
if (!$post instanceof Post) {
return null;
}
if ($this->include_replies) {
/*
* Get all replies to the post. The limit is set to 0 because if the count is not greater than 0,
* the method returns all replies.
*/
$replies = $this->post_dao->getRepliesToPost($post->post_id, $post->network, $this->order_by, $this->unit, $this->is_public, 0);
// if replies exist for this post
if ($replies) {
// recursively scan through the post replies, converting them
foreach ($replies as $reply) {
$reply = $this->convertPostToTweet($reply);
}
// add the replies to the post
$post->replies = $replies;
}
}
/*
* Chop and changing the data fetched from the database to look more like the official Twitter API.
*/
$post->text = $post->post_text;
$post->created_at = strftime('%a %b %d %T %z %Y', strtotime($post->pub_date));
$post->id = $post->post_id;
$post->favorited = $post->favorited ? true : false;
$post->annotations = null;
// to be implemented at some point
$post->truncated = false;
// always false
$post->protected = $post->is_protected == 0 ? false : true;
if ($post->geo != null) {
$coordinates = preg_split('/(, |,| )/', $post->geo);
$post->geo = new stdClass();
$post->geo->coordinates = $coordinates;
if (!isset($post->coordinates)) {
$post->coordinates = new stdClass();
}
$post->coordinates->coordinates = $coordinates;
}
/*
* SET THINKUP METADATA
*/
$post->thinkup = new stdClass();
$post->thinkup->retweet_count_cache = $post->retweet_count_cache;
$post->thinkup->retweet_count_api = $post->retweet_count_api;
$post->thinkup->reply_count_cache = $post->reply_count_cache;
$post->thinkup->old_retweet_count_cache = $post->old_retweet_count_cache;
$post->thinkup->is_geo_encoded = $post->is_geo_encoded;
$user = $this->user_dao->getUserByName($post->author_username, $post->network);
/*
* Occasionally you run into users you haven't fetched yet. Bypass this code if you find one of them.
*/
if ($user != null) {
if (!$this->trim_user) {
$post->user = $this->convertUserToStdClass($user);
$post->user->id = $post->user->user_id;
$post->user->followers_count = $post->user->follower_count;
$post->user->profile_image_url = $post->user->avatar;
$post->user->name = $post->user->full_name;
$post->user->screen_name = $post->user->username;
$post->user->statuses_count = $post->user->post_count;
$post->user->created_at = strftime('%a %b %d %T %z %Y', strtotime($post->user->joined));
$post->user->favorites_count = $post->user->favorites_count;
if (isset($post->user->other)) {
if (isset($post->user->other['avg_tweets_per_day'])) {
$post->user->avg_tweets_per_day = $post->user->other['avg_tweets_per_day'];
}
}
$post->user->thinkup = new stdClass();
$post->user->thinkup->last_post = $post->user->last_post;
$post->user->thinkup->last_post_id = $post->user->last_post_id;
$post->user->thinkup->found_in = $post->user->found_in;
} else {
$post->user = new stdClass();
$post->user->id = $user->user_id;
}
}
if ($this->include_entities) {
/*
* Gather hashtags and format them into a Tweet entity.
*/
$extracted_hashtags = Post::extractHashtags($post->text);
if (!isset($post->entities)) {
$post->entities = new stdClass();
}
$post->entities->hashtags = array();
if (!empty($extracted_hashtags)) {
foreach ($extracted_hashtags as $hashtag_text) {
$hashtag = new stdClass();
$hashtag->text = str_replace('#', '', $hashtag_text);
$hashtag->indices[] = stripos($post->text, $hashtag_text);
$hashtag->indices[] = strlen($hashtag_text) + $hashtag->indices[0];
$post->entities->hashtags[] = $hashtag;
}
}
/*
* Gather mentions and format them into a Tweet entity.
*/
$mentions = Post::extractMentions($post->text);
if (!isset($post->entities)) {
$post->entities = new stdClass();
}
$post->entities->user_mentions = array();
if (!empty($mentions)) {
foreach ($mentions as $username) {
$mentioned_user = $this->user_dao->getUserByName(str_replace('@', '', $username), $user->network);
$mention = new stdClass();
if (is_null($mentioned_user)) {
// skip this for now, probably not a good idea
continue;
/*
* If the user is not in our own ThinkUp database, a Twitter API call needs to be
* made to fill in the missing details.
*
* Not 100% sure if this is a good idea but it works.
*/
$user_api_call = json_decode(Utils::getURLContents('https://api.twitter.com/1/users/show.json?screen_name=' . $username));
$mention->name = $user_api_call->name;
$mention->id = $user_api_call->id;
$mention->screen_name = $user_api_call->screen_name;
} else {
$mention->name = $mentioned_user->full_name;
$mention->id = $mentioned_user->user_id;
$mention->screen_name = $mentioned_user->username;
}
$mention->indices = array();
$mention->indices[] = stripos($post->text, $username);
$mention->indices[] = strlen($username) + $mention->indices[0];
$post->entities->user_mentions[] = $mention;
}
}
}
if ($post->in_retweet_of_post_id != null) {
$post->retweeted_status = $this->post_dao->getPost($post->in_retweet_of_post_id, $user->network);
$post->retweeted_status = $this->convertPostToTweet($post->retweeted_status);
}
/*
* Unset no-longer-used variables in this post; mostly variables that have been moved to more
* Twtter like locations / naming conventions.
*/
unset($post->post_id, $post->pub_date, $post->network, $post->post_text, $post->author, $post->author_fullname, $post->author_username, $post->author_user_id, $post->author_avatar, $post->adj_pub_date, $post->user->follower_count, $post->user->is_protected, $post->user->network, $post->user->avatar, $post->user->full_name, $post->user->username, $post->user->user_id, $post->user->post_count, $post->user->joined, $post->user->favorites_count, $post->user->other, $post->link, $post->in_retweet_of_post_id, $post->in_retweet_of_user_id, $post->retweet_count_cache, $post->reply_count_cache, $post->old_retweet_count_cache, $post->is_geo_encoded, $post->rt_threshold, $post->is_protected, $post->user->last_post, $post->user->last_post_id, $post->user->found_in);
return $post;
}
public static function processLocation($data)
{
$config = Config::getInstance();
if (isset($data)) {
if ($data['location_type'] == 0) {
$adr = $data['name'];
$url = $config->getValue('gmapJson') . $adr . "&sensor=false";
$content = Utils::getURLContents($url);
$arr = json_decode($content);
var_dump($arr);
}
}
}
