public function testProcessTweetInstagramURLs() {
//instagr.am
$tweet["post_id"] = 105;
$tweet['post_text'] = "This is an Instagram post: http://instagr.am/p/oyQ6/ :)";
URLProcessor::processTweetURLs($this->logger, $tweet);
$link_dao = new LinkMySQLDAO();
$result = $link_dao->getLinkByUrl('http://instagr.am/p/oyQ6/');
$this->assertIsA($result, "Link");
$this->assertEqual($result->url, 'http://instagr.am/p/oyQ6/');
$this->assertEqual($result->expanded_url,
'http://distillery.s3.amazonaws.com/media/2010/12/20/f0f411210cc54353be07cf74ceb79f3b_7.jpg');
$this->assertEqual($result->title, '');
$this->assertEqual($result->post_id, 105);
$this->assertEqual($result->network, 'twitter');
$this->assertTrue($result->is_image);
// bad instagr.am URL
$tweet["post_id"] = 106;
$tweet['post_text'] = "This is an Instagram post with a bad URL: http://instagr.am/p/oyQ5/ :(";
URLProcessor::processTweetURLs($this->logger, $tweet);
$link_dao = new LinkMySQLDAO();
$result = $link_dao->getLinkByUrl('http://instagr.am/p/oyQ5/');
$this->assertIsA($result, "Link");
$this->assertEqual($result->url, 'http://instagr.am/p/oyQ5/');
$this->assertEqual($result->expanded_url, '');
$this->assertEqual($result->title, '');
$this->assertEqual($result->post_id, 106);
$this->assertEqual($result->network, 'twitter');
$this->assertFalse($result->is_image);
// test regexp extraction of image link from html
$api_call = $this->faux_data_path . "/instagr_am_p_oyQ6";
$resp = file_get_contents($api_call);
list($eurl, $is_image) = URLProcessor::extractInstagramImageURL($this->logger, $resp);
$this->assertEqual($eurl,
'http://distillery.s3.amazonaws.com/media/2010/12/20/f0f411210cc54353be07cf74ceb79f3b_7.jpg');
$this->assertTrue($is_image);
}
public function testProcessTweetURLs()
{
//Twitpic
$tweet["post_id"] = 100;
$tweet['post_text'] = "This is a Twitpic post http://twitpic.com/blah Yay!";
URLProcessor::processTweetURLs($this->logger, $tweet);
$link_dao = new LinkMySQLDAO();
$result = $link_dao->getLinkByUrl('http://twitpic.com/blah');
$this->assertIsA($result, "Link");
$this->assertEqual($result->url, 'http://twitpic.com/blah');
$this->assertEqual($result->expanded_url, 'http://twitpic.com/show/thumb/blah');
$this->assertEqual($result->title, '');
$this->assertEqual($result->post_id, 100);
$this->assertEqual($result->network, 'twitter');
$this->assertTrue($result->is_image);
//Yfrog
$tweet["post_id"] = 101;
$tweet['post_text'] = "This is a Yfrog post http://yfrog.com/blah Yay!";
URLProcessor::processTweetURLs($this->logger, $tweet);
$link_dao = new LinkMySQLDAO();
$result = $link_dao->getLinkByUrl('http://yfrog.com/blah');
$this->assertIsA($result, "Link");
$this->assertEqual($result->url, 'http://yfrog.com/blah');
$this->assertEqual($result->expanded_url, 'http://yfrog.com/blah.th.jpg');
$this->assertEqual($result->title, '');
$this->assertEqual($result->post_id, 101);
$this->assertEqual($result->network, 'twitter');
$this->assertTrue($result->is_image);
//Twitgoo
$tweet["post_id"] = 102;
$tweet['post_text'] = "This is a Twitgoo post http://twitgoo.com/blah Yay!";
URLProcessor::processTweetURLs($this->logger, $tweet);
$link_dao = new LinkMySQLDAO();
$result = $link_dao->getLinkByUrl('http://twitgoo.com/blah');
$this->assertIsA($result, "Link");
$this->assertEqual($result->url, 'http://twitgoo.com/blah');
$this->assertEqual($result->expanded_url, 'http://twitgoo.com/show/thumb/blah');
$this->assertEqual($result->title, '');
$this->assertEqual($result->post_id, 102);
$this->assertEqual($result->network, 'twitter');
$this->assertTrue($result->is_image);
//Picplz
$tweet["post_id"] = 103;
$tweet['post_text'] = "This is a Picplz post http://picplz.com/blah Yay!";
URLProcessor::processTweetURLs($this->logger, $tweet);
$link_dao = new LinkMySQLDAO();
$result = $link_dao->getLinkByUrl('http://picplz.com/blah');
$this->assertIsA($result, "Link");
$this->assertEqual($result->url, 'http://picplz.com/blah');
$this->assertEqual($result->expanded_url, 'http://picplz.com/blah/thumb/');
$this->assertEqual($result->title, '');
$this->assertEqual($result->post_id, 103);
$this->assertEqual($result->network, 'twitter');
$this->assertTrue($result->is_image);
// instagr.am
// check first with ending slash in URL (which the URLs 'should' include)
$tweet["post_id"] = 104;
$tweet['post_text'] = "This is an instagram post http:/instagr.am/blah/ Yay!";
URLProcessor::processTweetURLs($this->logger, $tweet);
$link_dao = new LinkMySQLDAO();
$result = $link_dao->getLinkByUrl('http://instagr.am/blah/');
$this->assertIsA($result, "Link");
$this->assertEqual($result->url, 'http://instagr.am/blah/');
$this->assertEqual($result->expanded_url, 'http://instagr.am/blah/media/');
$this->assertEqual($result->title, '');
$this->assertEqual($result->post_id, 104);
$this->assertEqual($result->network, 'twitter');
$this->assertTrue($result->is_image);
// check w/out ending slash also just in case
$tweet["post_id"] = 105;
$tweet['post_text'] = "This is an instagram post http:/instagr.am/blah Yay!";
URLProcessor::processTweetURLs($this->logger, $tweet);
$result = $link_dao->getLinkByUrl('http://instagr.am/blah');
$this->assertIsA($result, "Link");
$this->assertEqual($result->url, 'http://instagr.am/blah');
$this->assertEqual($result->expanded_url, 'http://instagr.am/blah/media/');
$this->assertEqual($result->title, '');
$this->assertEqual($result->post_id, 105);
$this->assertEqual($result->network, 'twitter');
$this->assertTrue($result->is_image);
//Flic.kr
$tweet["post_id"] = 106;
$tweet['post_text'] = "This is a Flickr post http://flic.kr/blah Yay!";
URLProcessor::processTweetURLs($this->logger, $tweet);
$link_dao = new LinkMySQLDAO();
$result = $link_dao->getLinkByUrl('http://flic.kr/blah');
$this->assertIsA($result, "Link");
$this->assertEqual($result->url, 'http://flic.kr/blah');
$this->assertEqual($result->expanded_url, '');
$this->assertEqual($result->title, '');
$this->assertEqual($result->post_id, 106);
$this->assertEqual($result->network, 'twitter');
$this->assertTrue($result->is_image);
}
/**
* Fetch instance users's friends tweets and friends.
*/
public function fetchFriendTweetsAndFriends()
{
if (!isset($this->user)) {
$this->fetchInstanceUserInfo();
}
if (isset($this->user)) {
$fd = DAOFactory::getDAO('FollowDAO');
$pd = DAOFactory::getDAO('PostDAO');
$continue_fetching = true;
while ($this->api->available && $this->api->available_api_calls_for_crawler > 0 && $continue_fetching) {
$stale_friend = $fd->getStalestFriend($this->user->user_id, 'twitter');
if ($stale_friend != null) {
$this->logger->logInfo($stale_friend->username . " is friend most need of update", __METHOD__ . ',' . __LINE__);
$stale_friend_tweets = str_replace("[id]", $stale_friend->username, $this->api->cURL_source['user_timeline']);
$args = array();
$count_arg = isset($this->twitter_options['tweet_count_per_call']) ? $this->twitter_options['tweet_count_per_call']->option_value : 100;
$args['count'] = $count_arg;
if ($stale_friend->last_post_id > 0) {
$args['since_id'] = $stale_friend->last_post_id;
}
list($cURL_status, $twitter_data) = $this->api->apiRequest($stale_friend_tweets, $args);
if ($cURL_status == 200) {
$count = 0;
$tweets = $this->api->parseXML($twitter_data);
if (count($tweets) > 0) {
$stale_friend_updated_from_tweets = false;
foreach ($tweets as $tweet) {
if ($pd->addPost($tweet, $stale_friend, $this->logger) > 0) {
$count++;
//expand and insert links contained in tweet
URLProcessor::processTweetURLs($this->logger, $tweet);
}
if (!$stale_friend_updated_from_tweets) {
//Update stale_friend values here
$stale_friend->full_name = $tweet['full_name'];
$stale_friend->avatar = $tweet['avatar'];
$stale_friend->location = $tweet['location'];
$stale_friend->description = $tweet['description'];
$stale_friend->url = $tweet['url'];
$stale_friend->is_protected = $tweet['is_protected'];
$stale_friend->follower_count = $tweet['follower_count'];
$stale_friend->friend_count = $tweet['friend_count'];
$stale_friend->post_count = $tweet['post_count'];
$stale_friend->joined = date_format(date_create($tweet['joined']), "Y-m-d H:i:s");
if ($tweet['post_id'] > $stale_friend->last_post_id) {
$stale_friend->last_post_id = $tweet['post_id'];
}
$this->user_dao->updateUser($stale_friend);
$stale_friend_updated_from_tweets = true;
}
}
} else {
$this->fetchAndAddUser($stale_friend->user_id, "Friends");
}
$this->logger->logInfo(count($tweets) . " tweet(s) found for " . $stale_friend->username . " and " . $count . " saved", __METHOD__ . ',' . __LINE__);
$this->fetchUserFriendsByIDs($stale_friend->user_id, $fd);
} elseif ($cURL_status == 401 || $cURL_status == 404) {
$e = $this->api->parseError($twitter_data);
$ued = DAOFactory::getDAO('UserErrorDAO');
$ued->insertError($stale_friend->user_id, $cURL_status, isset($e['error']) ? $e['error'] : $twitter_data, $this->user->user_id, 'twitter');
$this->logger->logInfo('User error saved', __METHOD__ . ',' . __LINE__);
}
} else {
$this->logger->logInfo('No friend staler than 1 day', __METHOD__ . ',' . __LINE__);
$continue_fetching = false;
}
}
}
}
public function addPostAndEntities($vals, $entities)
{
$urls = null;
// first add post
$retval = $this->addPost($vals);
// if post did not already exist
if ($retval) {
// then process entity information as available.
if (isset($entities) && isset($entities['urls'])) {
$urls = $entities['urls'];
}
// if $urls is null, will extract from tweet content.
URLProcessor::processTweetURLs($this->logger, $vals, $urls);
}
return $retval;
}
public function testProcessTweetURLs()
{
//Twitpic
$tweet["post_id"] = 100;
$tweet['post_text'] = "This is a Twitpic post http://twitpic.com/blah Yay!";
URLProcessor::processTweetURLs($this->logger, $tweet);
$link_dao = new LinkMySQLDAO();
$result = $link_dao->getLinkByUrl('http://twitpic.com/blah');
$this->assertIsA($result, "Link");
$this->assertEqual($result->url, 'http://twitpic.com/blah');
$this->assertEqual($result->expanded_url, 'http://twitpic.com/show/thumb/blah');
$this->assertEqual($result->title, '');
$this->assertEqual($result->post_id, 100);
$this->assertEqual($result->network, 'twitter');
$this->assertTrue($result->is_image);
//Yfrog
$tweet["post_id"] = 101;
$tweet['post_text'] = "This is a Yfrog post http://yfrog.com/blah Yay!";
URLProcessor::processTweetURLs($this->logger, $tweet);
$link_dao = new LinkMySQLDAO();
$result = $link_dao->getLinkByUrl('http://yfrog.com/blah');
$this->assertIsA($result, "Link");
$this->assertEqual($result->url, 'http://yfrog.com/blah');
$this->assertEqual($result->expanded_url, 'http://yfrog.com/blah.th.jpg');
$this->assertEqual($result->title, '');
$this->assertEqual($result->post_id, 101);
$this->assertEqual($result->network, 'twitter');
$this->assertTrue($result->is_image);
//Twitgoo
$tweet["post_id"] = 102;
$tweet['post_text'] = "This is a Twitgoo post http://twitgoo.com/blah Yay!";
URLProcessor::processTweetURLs($this->logger, $tweet);
$link_dao = new LinkMySQLDAO();
$result = $link_dao->getLinkByUrl('http://twitgoo.com/blah');
$this->assertIsA($result, "Link");
$this->assertEqual($result->url, 'http://twitgoo.com/blah');
$this->assertEqual($result->expanded_url, 'http://twitgoo.com/show/thumb/blah');
$this->assertEqual($result->title, '');
$this->assertEqual($result->post_id, 102);
$this->assertEqual($result->network, 'twitter');
$this->assertTrue($result->is_image);
//Picplz
$tweet["post_id"] = 103;
$tweet['post_text'] = "This is a Picplz post http://picplz.com/blah Yay!";
URLProcessor::processTweetURLs($this->logger, $tweet);
$link_dao = new LinkMySQLDAO();
$result = $link_dao->getLinkByUrl('http://picplz.com/blah');
$this->assertIsA($result, "Link");
$this->assertEqual($result->url, 'http://picplz.com/blah');
$this->assertEqual($result->expanded_url, 'http://picplz.com/blah/thumb/');
$this->assertEqual($result->title, '');
$this->assertEqual($result->post_id, 103);
$this->assertEqual($result->network, 'twitter');
$this->assertTrue($result->is_image);
//Flic.kr
$tweet["post_id"] = 104;
$tweet['post_text'] = "This is a Flickr post http://flic.kr/blah Yay!";
URLProcessor::processTweetURLs($this->logger, $tweet);
$link_dao = new LinkMySQLDAO();
$result = $link_dao->getLinkByUrl('http://flic.kr/blah');
$this->assertIsA($result, "Link");
$this->assertEqual($result->url, 'http://flic.kr/blah');
$this->assertEqual($result->expanded_url, '');
$this->assertEqual($result->title, '');
$this->assertEqual($result->post_id, 104);
$this->assertEqual($result->network, 'twitter');
$this->assertTrue($result->is_image);
}
public function addPostAndAssociatedInfo(array $vals, $entities = null, $user_array = null)
{
$urls = null;
// first add post
$retval = $this->addPost($vals);
// if post did not already exist
if ($retval) {
if ($user_array) {
$u = new User($user_array);
$user_dao = DAOFactory::getDAO('UserDAO');
$user_dao->setLoggerInstance($this->logger);
$user_dao->updateUser($u);
}
if ($entities && isset($entities['urls'])) {
$urls = $entities['urls'];
}
// if $urls is null, will extract from tweet content.
URLProcessor::processTweetURLs($this->logger, $vals, $urls);
if (isset($entities)) {
if (isset($entities['mentions'])) {
$mdao = DAOFactory::getDAO('MentionDAO');
$mdao->setLoggerInstance($this->logger);
$mdao->insertMentions($entities['mentions'], $vals['post_id'], $vals['author_user_id'], $vals['network']);
}
if (isset($entities['hashtags'])) {
$hdao = DAOFactory::getDAO('HashtagDAO');
$hdao->setLoggerInstance($this->logger);
$hdao->insertHashtags($entities['hashtags'], $vals['post_id'], $vals['network']);
}
if (isset($entities['place'])) {
$place = $entities['place'];
if ($place) {
$place_dao = DAOFactory::getDAO('PlaceDAO');
$place_dao->setLoggerInstance($this->logger);
$place_dao->insertPlace($place, $vals['post_id'], $vals['network']);
}
}
}
}
return $retval;
}
