function flickrthumbnails_crawl() { global $THINKTANK_CFG; global $db; global $conn; if (isset($THINKTANK_CFG['flickr_api_key']) && $THINKTANK_CFG['flickr_api_key'] != '') { $logger = new Logger($THINKTANK_CFG['log_location']); $fa = new FlickrAPIAccessor($THINKTANK_CFG['flickr_api_key'], $logger); $ldao = new LinkDAO($db, $logger); $flickrlinkstoexpand = $ldao->getLinksToExpandByURL('http://flic.kr/'); if (count($flickrlinkstoexpand > 0)) { $logger->logStatus(count($flickrlinkstoexpand) . " Flickr links to expand", "Flickr Plugin"); } else { $logger->logStatus("No Flickr links to expand", "Flickr Plugin"); } foreach ($flickrlinkstoexpand as $fl) { $eurl = $fa->getFlickrPhotoSource($fl); if ($eurl["expanded_url"] != '') { $ldao->saveExpandedUrl($fl, $eurl["expanded_url"], '', 1); } elseif ($eurl["error"] != '') { $ldao->saveExpansionError($fl, $eurl["error"]); } } $logger->close(); # Close logging } }
/** * Expand a given short URL * * @param str $tinyurl Shortened URL * @param LinkDAO $ldao * @return str Expanded URL */ private function untinyurl($tinyurl, $ldao) { $logger = Logger::getInstance(); $url = parse_url($tinyurl); $host = $url['host']; $port = isset($url['port']) ? $url['port'] : 80; $query = isset($url['query']) ? '?' . $url['query'] : ''; $fragment = isset($url['fragment']) ? '#' . $url['fragment'] : ''; if (empty($url['path'])) { $logger->logstatus("{$tinyurl} has no path", "Expand URLs Plugin"); $ldao->saveExpansionError($tinyurl, "Error expanding URL"); return ''; } else { $path = $url['path']; } $ch = curl_init(); curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); curl_setopt($ch, CURLOPT_URL, "http://{$host}:{$port}" . $path . $query . $fragment); curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 5); // seconds curl_setopt($ch, CURLOPT_HEADER, true); curl_setopt($ch, CURLOPT_NOBODY, true); curl_setopt($ch, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); $response = curl_exec($ch); if ($response === false) { $logger->logstatus("cURL error: " . curl_error($ch), "Expand URLs Plugin"); $ldao->saveExpansionError($tinyurl, "Error expanding URL"); $tinyurl = ''; } curl_close($ch); $lines = explode("\r\n", $response); foreach ($lines as $line) { if (stripos($line, 'Location:') === 0) { list(, $location) = explode(':', $line, 2); return ltrim($location); } } if (strpos($response, 'HTTP/1.1 404 Not Found') === 0) { $logger->logstatus("Short URL returned '404 Not Found'", "Expand URLs Plugin"); $ldao->saveExpansionError($tinyurl, "Error expanding URL"); return ''; } return $tinyurl; }
/** * Expand shortened Flickr links to image thumbnails if Flickr API key is set. * @param $api_key Flickr API key * @param $flickr_link Flickr URL */ public function expandFlickrThumbnail($api_key, $flickr_link, $original_link) { $flickr_api = new FlickrAPIAccessor($api_key); $photo_details = $flickr_api->getFlickrPhotoSource($flickr_link); if ($photo_details["image_src"] != '') { //@TODO Make another Flickr API call to get the photo title & description and save to tu_links $this->link_dao->saveExpandedUrl($original_link, $flickr_link, '', $photo_details["image_src"]); } elseif ($photo_details["error"] != '') { $this->link_dao->saveExpansionError($original_link, $photo_details["error"]); } }
/** * Return the expanded version of a given short URL or save an error for the $original_link in links table and * return an empty string. * * @param str $tinyurl Shortened URL * @param LinkDAO $link_dao * @param str $original_link * @param int $current_number Current link number * @param int $total_number Total links in group * @return str Expanded URL */ private function untinyurl($tinyurl, $link_dao, $original_link, $current_number, $total_number) { $error_log_prefix = $current_number . " of " . $total_number . " links: "; $logger = Logger::getInstance(); $url = parse_url($tinyurl); if (isset($url['host'])) { $host = $url['host']; } else { $error_msg = $tinyurl . ": No host found."; $logger->logError($error_log_prefix . $error_msg, __METHOD__ . ',' . __LINE__); $link_dao->saveExpansionError($original_link, $error_msg); return ''; } $port = isset($url['port']) ? ':' . $url['port'] : ''; $query = isset($url['query']) ? '?' . $url['query'] : ''; $fragment = isset($url['fragment']) ? '#' . $url['fragment'] : ''; if (empty($url['path'])) { $path = ''; } else { $path = $url['path']; } $scheme = isset($url['scheme']) ? $url['scheme'] : 'http'; $reconstructed_url = $scheme . "://{$host}{$port}" . $path . $query . $fragment; $ch = curl_init(); curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); curl_setopt($ch, CURLOPT_URL, $reconstructed_url); curl_setopt($ch, CURLOPT_TIMEOUT, 5); // seconds curl_setopt($ch, CURLOPT_HEADER, true); curl_setopt($ch, CURLOPT_NOBODY, true); curl_setopt($ch, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); $response = curl_exec($ch); if ($response === false) { $error_msg = $reconstructed_url . " cURL error: " . curl_error($ch); $logger->logError($error_log_prefix . $error_msg, __METHOD__ . ',' . __LINE__); $link_dao->saveExpansionError($original_link, $error_msg); $tinyurl = ''; } curl_close($ch); $lines = explode("\r\n", $response); foreach ($lines as $line) { if (stripos($line, 'Location:') === 0) { list(, $location) = explode(':', $line, 2); return ltrim($location); } } if (strpos($response, 'HTTP/1.1 404 Not Found') === 0) { $error_msg = $reconstructed_url . " returned '404 Not Found'"; $logger->logError($error_log_prefix . $error_msg, __METHOD__ . ',' . __LINE__); $link_dao->saveExpansionError($original_link, $error_msg); return ''; } return $tinyurl; }
function testSaveExpansionError() { $ldao = new LinkDAO($this->db, $this->logger); $linktogeterror = $ldao->getLinkById(10); $this->assertEqual($linktogeterror->error, ''); $ldao->saveExpansionError($linktogeterror->url, "This is expansion error text"); $linkthathaserror = $ldao->getLinkById(10); $this->assertEqual($linkthathaserror->error, "This is expansion error text"); }