public function collectData(array $param) { function ExtractFromDelimiters($string, $start, $end) { if (strpos($string, $start) !== false) { $section_retrieved = substr($string, strpos($string, $start) + strlen($start)); $section_retrieved = substr($section_retrieved, 0, strpos($section_retrieved, $end)); return $section_retrieved; } return false; } $html = $this->file_get_html($this->getURI()) or $this->returnError('Could not request EstCeQuonMetEnProd: ' . $this->getURI(), 500); $img = $html->find('img', 0); if (is_object($img)) { $img = $img->src; if ($img[0] == '/') { $img = substr($this->getURI(), 0, strlen($this->getURI()) - 1) . $img; } } $item = new \Item(); $item->uri = $this->getURI() . '#' . date('Y-m-d'); $item->thumbnailUri = $img; $item->title = $this->getName(); $item->author = 'Nicolas Hoffmann'; $item->timestamp = strtotime('today midnight'); $item->content = str_replace('src="/', 'src="' . $this->getURI(), trim(ExtractFromDelimiters($html->outertext, '<body role="document">', '<br /><br />'))); $this->items[] = $item; }
public function collectData(array $param) { function ExtractFromDelimiters($string, $start, $end) { if (strpos($string, $start) !== false) { $section_retrieved = substr($string, strpos($string, $start) + strlen($start)); $section_retrieved = substr($section_retrieved, 0, strpos($section_retrieved, $end)); return $section_retrieved; } return false; } function StripWithDelimiters($string, $start, $end) { while (strpos($string, $start) !== false) { $section_to_remove = substr($string, strpos($string, $start)); $section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end)); $string = str_replace($section_to_remove, '', $string); } return $string; } $category = $param['category']; if (empty($category)) { $category = 'all'; } if ($category !== preg_replace('/[^a-z-]+/', '', $category) || strlen($category > 32)) { $this->returnError('Invalid "category" parameter.', 400); } $url = $this->getURI() . 'rss/' . $category . '/'; $html = $this->file_get_html($url) or $this->returnError('Could not request Nextgov: ' . $url, 500); $limit = 0; foreach ($html->find('item') as $element) { if ($limit < 10) { $article_url = ExtractFromDelimiters($element->innertext, '<link>', '</link>'); $article_author = ExtractFromDelimiters($element->innertext, 'dc/elements/1.1/">', '</dc:creator>'); $article_title = $element->find('title', 0)->plaintext; $article_subtitle = $element->find('description', 0)->plaintext; $article_timestamp = strtotime($element->find('pubDate', 0)->plaintext); $article_thumbnail = ExtractFromDelimiters($element->innertext, '<media:content url="', '"'); $article = $this->file_get_html($article_url) or $this->returnError('Could not request Nextgov: ' . $article_url, 500); $contents = $article->find('div.wysiwyg', 0)->innertext; $contents = StripWithDelimiters($contents, '<div class="ad-container">', '</div>'); $contents = StripWithDelimiters($contents, '<div', '</div>'); //ad outer div $contents = StripWithDelimiters($contents, '<script', '</script>'); $contents = ($article_thumbnail == '' ? '' : '<p><img src="' . $article_thumbnail . '" /></p>') . '<p><b>' . $article_subtitle . '</b></p>' . trim($contents); if ($article_thumbnail == '') { $article_thumbnail = 'http://cdn.nextgov.com/nextgov/images/logo.png'; } $item = new \Item(); $item->uri = $article_url; $item->title = $article_title; $item->author = $article_author; $item->thumbnailUri = $article_thumbnail; $item->timestamp = $article_timestamp; $item->content = $contents; $this->items[] = $item; $limit++; } } }
public function collectData(array $param) { //Utility function for retrieving text based on start and end delimiters function ExtractFromDelimiters($string, $start, $end) { if (strpos($string, $start) !== false) { $section_retrieved = substr($string, strpos($string, $start) + strlen($start)); $section_retrieved = substr($section_retrieved, 0, strpos($section_retrieved, $end)); return $section_retrieved; } return false; } //Ensure proper parameters have been provided if (empty($param['search'])) { $this->returnError('You must specify a search criteria', 400); } //Retrieve torrent listing as truncated rss, which does not contain torrent description $url = 'http://www.t411.in/torrents/search/?' . $param['search'] . '&order=added&type=desc'; $html = file_get_html($url) or $this->returnError('Could not request t411: ' . $url, 500); $results = $html->find('table.results')[0] or $this->returnError('No results from t411: ' . $url, 500); $limit = 0; //Process each item individually foreach ($results->find('tr') as $element) { //Limit total amount of requests if ($limit < 5) { //Requests are rate-limited sleep(1); //So we need to wait //Retrieve data from RSS entry $item_uri = 'http://' . ExtractFromDelimiters($element->outertext, '<a href="//', '"'); $item_title = ExtractFromDelimiters($element->outertext, '" title="', '"'); $item_date = strtotime($element->find('dd', 0)->plaintext); //Retrieve full description from torrent page if ($item_html = file_get_html($item_uri)) { //Retrieve data from page contents $item_desc = $item_html->find('div.description', 0); $item_author = $item_html->find('a.profile', 0)->innertext; //Retrieve image for thumbnail or generic logo fallback $item_image = 'http://www.t411.in/themes/blue/images/logo.png'; foreach ($item_desc->find('img') as $img) { if (strpos($img->src, 'dreamprez') === false) { $item_image = $img->src; break; } } //Build and add final item $item = new \Item(); $item->uri = $item_uri; $item->title = $item_title; $item->author = $item_author; $item->timestamp = $item_date; $item->thumbnailUri = $item_image; $item->content = utf8_encode($item_desc->innertext); $this->items[] = $item; $limit++; } } } }
public function collectData(array $param) { function ExtractFromDelimiters($string, $start, $end) { if (strpos($string, $start) !== false) { $section_retrieved = substr($string, strpos($string, $start) + strlen($start)); $section_retrieved = substr($section_retrieved, 0, strpos($section_retrieved, $end)); return $section_retrieved; } return false; } function StripWithDelimiters($string, $start, $end) { while (strpos($string, $start) !== false) { $section_to_remove = substr($string, strpos($string, $start)); $section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end)); $string = str_replace($section_to_remove, '', $string); } return $string; } $feed = $this->getURI() . 'feed/'; $html = $this->file_get_html($feed) or $this->returnError('Could not request ' . $this->getName() . ': ' . $feed, 500); $limit = 0; foreach ($html->find('item') as $element) { if ($limit < 5) { $article_image = $element->find('image', 0)->plaintext; $article_url = ExtractFromDelimiters($element->innertext, '<link>', '</link>'); $article_summary = ExtractFromDelimiters($element->innertext, '<description><![CDATA[<p>', '</p>'); $article_html = file_get_contents($article_url) or $this->returnError('Could not request ' . $this->getName() . ': ' . $article_url, 500); if (substr($article_html, 0, 2) == "‹") { //http://www.gzip.org/zlib/rfc-gzip.html#header-trailer -> GZip ID1 $article_html = gzdecode($article_html); } //Response is GZipped even if we didn't accept GZip!? Let's decompress... $article_html = str_get_html($article_html); //Now we have our HTML data. But still, that's an important HTTP violation... $article_content = $article_html->find('div.wlistingsingletext', 0)->innertext; $article_content = StripWithDelimiters($article_content, '<script', '</script>'); $article_content = '<p><img src="' . $article_image . '" /></p>' . '<p><b>' . $article_summary . '</b></p>' . trim($article_content); $item = new \Item(); $item->uri = $article_url; $item->thumbnailUri = $article_image; $item->title = $element->find('title', 0)->plaintext; $item->author = $article_html->find('a[rel=author]', 0)->plaintext; $item->timestamp = strtotime($element->find('pubDate', 0)->plaintext); $item->content = $article_content; $this->items[] = $item; $limit++; } } }
public function collectData(array $param) { function ExtractFromDelimiters($string, $start, $end) { if (strpos($string, $start) !== false) { $section_retrieved = substr($string, strpos($string, $start) + strlen($start)); $section_retrieved = substr($section_retrieved, 0, strpos($section_retrieved, $end)); return $section_retrieved; } return false; } function StripWithDelimiters($string, $start, $end) { while (strpos($string, $start) !== false) { $section_to_remove = substr($string, strpos($string, $start)); $section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end)); $string = str_replace($section_to_remove, '', $string); } return $string; } function CleanArticle($article_html) { $article_html = '<p>' . substr($article_html, strpos($article_html, '<p>') + 3); $article_html = StripWithDelimiters($article_html, '<span class="credit">', '</span>'); $article_html = StripWithDelimiters($article_html, '<script', '</script>'); $article_html = StripWithDelimiters($article_html, '<div class="shortcode related-links', '</div>'); $article_html = StripWithDelimiters($article_html, '<a class="clickToEnlarge">', '</a>'); return $article_html; } if (!empty($param['topic'])) { $this->topicName = $param['topic']; } $pageUrl = 'http://www.cnet.com/' . (empty($this->topicName) ? '' : 'topics/' . $this->topicName . '/'); $html = $this->file_get_html($pageUrl) or $this->returnError('Could not request CNET: ' . $pageUrl, 500); $limit = 0; foreach ($html->find('div.assetBody') as $element) { if ($limit < 8) { $article_title = trim($element->find('h2', 0)->plaintext); $article_uri = 'http://www.cnet.com' . $element->find('a', 0)->href; $article_thumbnail = $element->parent()->find('img', 0)->src; $article_timestamp = strtotime($element->find('time.assetTime', 0)->plaintext); $article_author = trim($element->find('a[rel=author]', 0)->plaintext); if (!empty($article_title) && !empty($article_uri) && strpos($article_uri, '/news/') !== false) { $article_html = $this->file_get_html($article_uri) or $this->returnError('Could not request CNET: ' . $article_uri, 500); if (is_null($article_thumbnail)) { $article_thumbnail = $article_html->find('div.originalImage', 0); } if (is_null($article_thumbnail)) { $article_thumbnail = $article_html->find('span.imageContainer', 0); } if (is_object($article_thumbnail)) { $article_thumbnail = $article_thumbnail->find('img', 0)->src; } $article_content = trim(CleanArticle(ExtractFromDelimiters($article_html, '<div class="articleContent', '<footer>'))); $item = new \Item(); $item->uri = $article_uri; $item->thumbnailUri = $article_thumbnail; $item->title = $article_title; $item->author = $article_author; $item->timestamp = $article_timestamp; $item->content = $article_content; $this->items[] = $item; $limit++; } } } }
public function collectData(array $param) { function ExtractFromDelimiters($string, $start, $end) { if (strpos($string, $start) !== false) { $section_retrieved = substr($string, strpos($string, $start) + strlen($start)); $section_retrieved = substr($section_retrieved, 0, strpos($section_retrieved, $end)); return $section_retrieved; } return false; } function StripWithDelimiters($string, $start, $end) { while (strpos($string, $start) !== false) { $section_to_remove = substr($string, strpos($string, $start)); $section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end)); $string = str_replace($section_to_remove, '', $string); } return $string; } function CleanArticle($article_html) { $article_html = '<p>' . substr($article_html, strpos($article_html, '</script></div><p>') + 18); $article_html = StripWithDelimiters($article_html, '<script>', '</script>'); $article_html = StripWithDelimiters($article_html, '<div class="shortcode related-links', '</div>'); $article_html = StripWithDelimiters($article_html, '<a class="clickToEnlarge">', '</a>'); return $article_html; } if (!empty($param['topic'])) { $this->topicName = $param['topic']; } $pageUrl = 'http://www.cnet.com/' . (empty($this->topicName) ? '' : 'topics/' . $this->topicName . '/'); $html = file_get_html($pageUrl) or $this->returnError('Could not request CNET: ' . $pageUrl, 500); $limit = 0; foreach ($html->find('div.socialSharingSmall') as $element) { if ($limit < 4) { $article_meta = json_decode(ExtractFromDelimiters($element->outertext, 'data-social-counts-options=\'', '\'>')); $article_title = $article_meta->title; $article_uri = $article_meta->url; if (!empty($article_title) && !empty($article_uri) && strpos($article_uri, '/news/') !== false) { $article_html = file_get_html($article_uri) or $this->returnError('Could not request CNET: ' . $article_uri, 500); $article_timestamp = strtotime(ExtractFromDelimiters($article_html->innertext, '<time itemprop="datePublished" class="', '">')); $article_thumbnail = $article_html->find('div.originalImage', 0); if (is_null($article_thumbnail)) { $article_thumbnail = $article_html->find('span.imageContainer', 0); } if (!is_null($article_thumbnail)) { $article_thumbnail = $article_thumbnail->find('img', 0)->src; } $article_content = trim(CleanArticle(ExtractFromDelimiters($article_html, '</div></div></div><div class="col-8">', '<footer>'))); $article_author = trim($article_html->find('a.author', 0)->plaintext); $item = new \Item(); $item->uri = $article_uri; $item->thumbnailUri = $article_thumbnail; $item->title = $article_title; $item->author = $article_author; $item->timestamp = $article_timestamp; $item->content = $article_content; $this->items[] = $item; $limit++; } } } }
public function collectData(array $param) { //Extract a string using start and end delimiters function ExtractFromDelimiters($string, $start, $end) { if (strpos($string, $start) !== false) { $section_retrieved = substr($string, strpos($string, $start) + strlen($start)); $section_retrieved = substr($section_retrieved, 0, strpos($section_retrieved, $end)); return $section_retrieved; } return false; } //Utility function for cleaning a Facebook link $unescape_fb_link = function ($matches) { if (is_array($matches) && count($matches) > 1) { $link = $matches[1]; if (strpos($link, '/') === 0) { $link = 'https://www.facebook.com' . $link . '"'; } if (strpos($link, 'facebook.com/l.php?u=') !== false) { $link = urldecode(ExtractFromDelimiters($link, 'facebook.com/l.php?u=', '&')); } return ' href="' . $link . '"'; } }; //Utility function for converting facebook emoticons $unescape_fb_emote = function ($matches) { static $facebook_emoticons = array('smile' => ':)', 'frown' => ':(', 'tongue' => ':P', 'grin' => ':D', 'gasp' => ':O', 'wink' => ';)', 'pacman' => ':<', 'grumpy' => '>_<', 'unsure' => ':/', 'cry' => ':\'(', 'kiki' => '^_^', 'glasses' => '8-)', 'sunglasses' => 'B-)', 'heart' => '<3', 'devil' => ']:D', 'angel' => '0:)', 'squint' => '-_-', 'confused' => 'o_O', 'upset' => 'xD', 'colonthree' => ':3', 'like' => '👍'); $len = count($matches); if ($len > 1) { for ($i = 1; $i < $len; $i++) { foreach ($facebook_emoticons as $name => $emote) { if ($matches[$i] === $name) { return $emote; } } } } return $matches[0]; }; $html = null; //Handle captcha response sent by the viewer if (isset($_POST['captcha_response'])) { if (session_status() == PHP_SESSION_NONE) { session_start(); } if (isset($_SESSION['captcha_fields'], $_SESSION['captcha_action'])) { $captcha_action = $_SESSION['captcha_action']; $captcha_fields = $_SESSION['captcha_fields']; $captcha_fields['captcha_response'] = preg_replace("/[^a-zA-Z0-9]+/", "", $_POST['captcha_response']); $http_options = array('http' => array('method' => 'POST', 'user_agent' => ini_get('user_agent'), 'header' => array("Content-type: application/x-www-form-urlencoded\r\nReferer: {$captcha_action}\r\nCookie: noscript=1\r\n"), 'content' => http_build_query($captcha_fields))); $context = stream_context_create($http_options); $html = file_get_contents($captcha_action, false, $context); if ($html === FALSE) { $this->returnError('Failed to submit captcha response back to Facebook', 500); } unset($_SESSION['captcha_fields']); $html = str_get_html($html); } unset($_SESSION['captcha_fields']); unset($_SESSION['captcha_action']); } //Retrieve page contents if (is_null($html)) { if (isset($param['u'])) { if (!strpos($param['u'], "/")) { $html = $this->file_get_html('https://www.facebook.com/' . urlencode($param['u']) . '?_fb_noscript=1') or $this->returnError('No results for this query.', 404); } else { $html = $this->file_get_html('https://www.facebook.com/pages/' . $param['u'] . '?_fb_noscript=1') or $this->returnError('No results for this query.', 404); } } else { $this->returnError('You must specify a Facebook username.', 400); } } //Handle captcha form? $captcha = $html->find('div.captcha_interstitial', 0); if (!is_null($captcha)) { //Save form for submitting after getting captcha response if (session_status() == PHP_SESSION_NONE) { session_start(); } $captcha_fields = array(); foreach ($captcha->find('input, button') as $input) { $captcha_fields[$input->name] = $input->value; } $_SESSION['captcha_fields'] = $captcha_fields; $_SESSION['captcha_action'] = 'https://www.facebook.com' . $captcha->find('form', 0)->action; //Show captcha filling form to the viewer, proxying the captcha image $img = base64_encode(file_get_contents($captcha->find('img', 0)->src)); header('HTTP/1.1 500 ' . Http::getMessageForCode(500)); header('Content-Type: text/html'); die('<form method="post" action="?' . $_SERVER['QUERY_STRING'] . '">' . '<h2>Facebook captcha challenge</h2>' . '<p>Unfortunately, rss-bridge cannot fetch the requested page.<br />' . 'Facebook wants rss-bridge to resolve the following captcha:</p>' . '<p><img src="data:image/png;base64,' . $img . '" /></p>' . '<p><b>Response:</b> <input name="captcha_response" placeholder="please fill in" />' . '<input type="submit" value="Submit!" /></p>' . '</form>'); } //No captcha? We can carry on retrieving page contents :) $element = $html->find('#pagelet_timeline_main_column')[0]->children(0)->children(0)->children(0)->next_sibling()->children(0); if (isset($element)) { $author = str_replace(' | Facebook', '', $html->find('title#pageTitle', 0)->innertext); $profilePic = 'https://graph.facebook.com/' . $param['u'] . '/picture?width=200&height=200'; $this->name = $author; foreach ($element->children() as $post) { $item = new \Item(); if (count($post->find('abbr')) > 0) { //Retrieve post contents $content = preg_replace('/(?i)><div class=\\"clearfix([^>]+)>(.+?)div\\ class=\\"userContent\\"/i', '', $post); $content = preg_replace('/(?i)><div class=\\"_59tj([^>]+)>(.+?)<\\/div><\\/div><a/i', '', $content); $content = preg_replace('/(?i)><div class=\\"_3dp([^>]+)>(.+?)div\\ class=\\"[^u]+userContent\\"/i', '', $content); $content = preg_replace('/(?i)><div class=\\"_4l5([^>]+)>(.+?)<\\/div>/i', '', $content); //Remove html nodes, keep only img, links, basic formatting $content = strip_tags($content, '<a><img><i><u>'); //Adapt link hrefs: convert relative links into absolute links and bypass external link redirection $content = preg_replace_callback('/ href=\\"([^"]+)\\"/i', $unescape_fb_link, $content); //Clean useless html tag properties and fix link closing tags foreach (array('onmouseover', 'onclick', 'target', 'ajaxify', 'tabindex', 'class', 'style', 'data-[^=]*', 'aria-[^=]*', 'role', 'rel', 'id') as $property_name) { $content = preg_replace('/ ' . $property_name . '=\\"[^"]*\\"/i', '', $content); } $content = preg_replace('/<\\/a [^>]+>/i', '</a>', $content); //Convert textual representation of emoticons eg "<i><u>smile emoticon</u></i>" back to ASCII emoticons eg ":)" $content = preg_replace_callback('/<i><u>([^ <>]+) ([^<>]+)<\\/u><\\/i>/i', $unescape_fb_emote, $content); //Retrieve date of the post $date = $post->find("abbr")[0]; if (isset($date) && $date->hasAttribute('data-utime')) { $date = $date->getAttribute('data-utime'); } else { $date = 0; } //Build title from username and content $title = $author; if (strlen($title) > 24) { $title = substr($title, 0, strpos(wordwrap($title, 24), "\n")) . '...'; } $title = $title . ' | ' . strip_tags($content); if (strlen($title) > 64) { $title = substr($title, 0, strpos(wordwrap($title, 64), "\n")) . '...'; } //Use first image as thumbnail if available, or profile pic fallback $thumbnail = $post->find('img', 1); if (is_object($thumbnail)) { $thumbnail = $thumbnail->src; } else { $thumbnail = $profilePic; } //Build and add final item $item->uri = 'https://facebook.com' . $post->find('abbr')[0]->parent()->getAttribute('href'); $item->thumbnailUri = $thumbnail; $item->content = $content; $item->title = $title; $item->author = $author; $item->timestamp = $date; $this->items[] = $item; } } } }
public function collectData(array $param) { //Extract a string using start and end delimiters function ExtractFromDelimiters($string, $start, $end) { if (strpos($string, $start) !== false) { $section_retrieved = substr($string, strpos($string, $start) + strlen($start)); $section_retrieved = substr($section_retrieved, 0, strpos($section_retrieved, $end)); return $section_retrieved; } return false; } //Utility function for cleaning a Facebook link $unescape_fb_link = function ($matches) { if (is_array($matches) && count($matches) > 1) { $link = $matches[1]; if (strpos($link, '/') === 0) { $link = 'https://facebook.com' . $link . '"'; } if (strpos($link, 'facebook.com/l.php?u=') !== false) { $link = urldecode(ExtractFromDelimiters($link, 'facebook.com/l.php?u=', '&')); } return ' href="' . $link . '"'; } }; //Utility function for converting facebook emoticons $unescape_fb_emote = function ($matches) { static $facebook_emoticons = array('smile' => ':)', 'frown' => ':(', 'tongue' => ':P', 'grin' => ':D', 'gasp' => ':O', 'wink' => ';)', 'pacman' => ':<', 'grumpy' => '>_<', 'unsure' => ':/', 'cry' => ':\'(', 'kiki' => '^_^', 'glasses' => '8-)', 'sunglasses' => 'B-)', 'heart' => '<3', 'devil' => ']:D', 'angel' => '0:)', 'squint' => '-_-', 'confused' => 'o_O', 'upset' => 'xD', 'colonthree' => ':3', 'like' => '👍'); $len = count($matches); if ($len > 1) { for ($i = 1; $i < $len; $i++) { foreach ($facebook_emoticons as $name => $emote) { if ($matches[$i] === $name) { return $emote; } } } } return $matches[0]; }; $html = ''; if (isset($param['u'])) { if (!strpos($param['u'], "/")) { $html = file_get_html('https://facebook.com/' . urlencode($param['u']) . '?_fb_noscript=1') or $this->returnError('No results for this query.', 404); } else { $html = file_get_html('https://facebook.com/pages/' . $param['u'] . '?_fb_noscript=1') or $this->returnError('No results for this query.', 404); } } else { $this->returnError('You must specify a Facebook username.', 400); } $element = $html->find('[id^=PagePostsSectionPagelet-]')[0]->children(0)->children(0); if (isset($element)) { $author = str_replace(' | Facebook', '', $html->find('title#pageTitle', 0)->innertext); $profilePic = 'https://graph.facebook.com/' . $param['u'] . '/picture?width=200&height=200'; $this->name = $author; foreach ($element->children() as $post) { $item = new \Item(); if ($post->hasAttribute("data-time")) { //Retrieve post contents $content = preg_replace('/(?i)><div class=\\"clearfix([^>]+)>(.+?)div\\ class=\\"userContent\\"/i', '', $post); $content = preg_replace('/(?i)><div class=\\"_59tj([^>]+)>(.+?)<\\/div><\\/div><a/i', '', $content); $content = preg_replace('/(?i)><div class=\\"_3dp([^>]+)>(.+?)div\\ class=\\"[^u]+userContent\\"/i', '', $content); $content = preg_replace('/(?i)><div class=\\"_4l5([^>]+)>(.+?)<\\/div>/i', '', $content); //Remove html nodes, keep only img, links, basic formatting $content = strip_tags($content, '<a><img><i><u>'); //Adapt link hrefs: convert relative links into absolute links and bypass external link redirection $content = preg_replace_callback('/ href=\\"([^"]+)\\"/i', $unescape_fb_link, $content); //Clean useless html tag properties and fix link closing tags foreach (array('onmouseover', 'onclick', 'target', 'ajaxify', 'tabindex', 'class', 'style', 'data-[^=]*', 'aria-[^=]*', 'role', 'rel', 'id') as $property_name) { $content = preg_replace('/ ' . $property_name . '=\\"[^"]*\\"/i', '', $content); } $content = preg_replace('/<\\/a [^>]+>/i', '</a>', $content); //Convert textual representation of emoticons eg "<i><u>smile emoticon</u></i>" back to ASCII emoticons eg ":)" $content = preg_replace_callback('/<i><u>([^ <>]+) ([^<>]+)<\\/u><\\/i>/i', $unescape_fb_emote, $content); //Retrieve date of the post $date = $post->find("abbr")[0]; if (isset($date) && $date->hasAttribute('data-utime')) { $date = $date->getAttribute('data-utime'); } else { $date = 0; } //Build title from username and content $title = $author; if (strlen($title) > 24) { $title = substr($title, 0, strpos(wordwrap($title, 24), "\n")) . '...'; } $title = $title . ' | ' . strip_tags($content); if (strlen($title) > 64) { $title = substr($title, 0, strpos(wordwrap($title, 64), "\n")) . '...'; } //Use first image as thumbnail if available, or profile pic fallback $thumbnail = $post->find('img', 1)->src; if (strlen($thumbnail) == 0) { $thumbnail = $profilePic; } //Build and add final item $item->uri = 'https://facebook.com' . str_replace('&', '&', $post->find('abbr')[0]->parent()->getAttribute('href')); $item->thumbnailUri = $thumbnail; $item->content = $content; $item->title = $title; $item->author = $author; $item->timestamp = $date; $this->items[] = $item; } } } }
public function collectData(array $param) { function ExtractFromDelimiters($string, $start, $end) { if (strpos($string, $start) !== false) { $section_retrieved = substr($string, strpos($string, $start) + strlen($start)); $section_retrieved = substr($section_retrieved, 0, strpos($section_retrieved, $end)); return $section_retrieved; } return false; } function TypeToString($type) { switch ($type) { case 1: return '3DS Game'; case 4: return 'eShop'; default: return '??? (' . $type . ')'; } } function CardToString($card) { switch ($card) { case 1: return 'Regular (CARD1)'; case 2: return 'NAND (CARD2)'; default: return '??? (' . $card . ')'; } } $dataUrl = 'http://3dsdb.com/xml.php'; $xml = file_get_contents($dataUrl) or $this->returnError('Could not request 3dsdb: ' . $dataUrl, 500); $limit = 0; foreach (array_reverse(explode('<release>', $xml)) as $element) { if ($limit < 5) { if (strpos($element, '</release>') !== false) { $id = ExtractFromDelimiters($element, '<id>', '</id>'); $name = ExtractFromDelimiters($element, '<name>', '</name>'); $publisher = ExtractFromDelimiters($element, '<publisher>', '</publisher>'); $region = ExtractFromDelimiters($element, '<region>', '</region>'); $group = ExtractFromDelimiters($element, '<group>', '</group>'); $imagesize = ExtractFromDelimiters($element, '<imagesize>', '</imagesize>'); $serial = ExtractFromDelimiters($element, '<serial>', '</serial>'); $titleid = ExtractFromDelimiters($element, '<titleid>', '</titleid>'); $imgcrc = ExtractFromDelimiters($element, '<imgcrc>', '</imgcrc>'); $filename = ExtractFromDelimiters($element, '<filename>', '</filename>'); $releasename = ExtractFromDelimiters($element, '<releasename>', '</releasename>'); $trimmedsize = ExtractFromDelimiters($element, '<trimmedsize>', '</trimmedsize>'); $firmware = ExtractFromDelimiters($element, '<firmware>', '</firmware>'); $type = ExtractFromDelimiters($element, '<type>', '</type>'); $card = ExtractFromDelimiters($element, '<card>', '</card>'); if (!empty($releasename)) { //Retrieve cover art and short desc from IGN? $ignResult = false; $ignDescription = ''; $ignLink = ''; $ignDate = time(); $ignCoverArt = ''; $ignSearchUrl = 'http://www.ign.com/search?q=' . urlencode($name); if ($ignResult = file_get_html($ignSearchUrl)) { $ignCoverArt = $ignResult->find('div.search-item-media', 0)->find('img', 0)->src; $ignDesc = $ignResult->find('div.search-item-description', 0)->plaintext; $ignLink = $ignResult->find('div.search-item-sub-title', 0)->find('a', 1)->href; $ignDate = strtotime(trim($ignResult->find('span.publish-date', 0)->plaintext)); $ignDescription = '<div><img src="' . $ignCoverArt . '" /></div><div>' . $ignDesc . ' <a href="' . $ignLink . '">More at IGN</a></div>'; } //Main section : Release description from 3DS database $releaseDescription = '<h3>Release Details</h3>' . '<b>Release ID: </b>' . $id . '<br />' . '<b>Game Name: </b>' . $name . '<br />' . '<b>Publisher: </b>' . $publisher . '<br />' . '<b>Region: </b>' . $region . '<br />' . '<b>Group: </b>' . $group . '<br />' . '<b>Image size: </b>' . intval($imagesize) / 8 . 'MB<br />' . '<b>Serial: </b>' . $serial . '<br />' . '<b>Title ID: </b>' . $titleid . '<br />' . '<b>Image CRC: </b>' . $imgcrc . '<br />' . '<b>File Name: </b>' . $filename . '<br />' . '<b>Release Name: </b>' . $releasename . '<br />' . '<b>Trimmed size: </b>' . intval(intval($trimmedsize) / 1048576) . 'MB<br />' . '<b>Firmware: </b>' . $firmware . '<br />' . '<b>Type: </b>' . TypeToString($type) . '<br />' . '<b>Card: </b>' . CardToString($card) . '<br />'; //Build search links section to facilitate release search using search engines $releaseNameEncoded = urlencode(str_replace(' ', '+', $releasename)); $searchLinkGoogle = 'https://google.com/?q=' . $releaseNameEncoded; $searchLinkDuckDuckGo = 'https://duckduckgo.com/?q=' . $releaseNameEncoded; $searchLinkQwant = 'https://lite.qwant.com/?q=' . $releaseNameEncoded . '&t=web'; $releaseSearchLinks = '<h3>Search this release</h3><ul>' . '<li><a href="' . $searchLinkGoogle . '">Search using Google</a></li>' . '<li><a href="' . $searchLinkDuckDuckGo . '">Search using DuckDuckGo</a></li>' . '<li><a href="' . $searchLinkQwant . '">Search using Qwant</a></li>' . '</ul>'; //Build and add final item with the above three sections $item = new \Item(); $item->title = $name; $item->author = $publisher; $item->timestamp = $ignDate; $item->thumbnailUri = $ignCoverArt; $item->uri = empty($ignLink) ? $searchLinkDuckDuckGo : $ignLink; $item->content = $ignDescription . $releaseDescription . $releaseSearchLinks; $this->items[] = $item; $limit++; } } } } }
public function collectData(array $param) { function StripCDATA($string) { $string = str_replace('<![CDATA[', '', $string); $string = str_replace(']]>', '', $string); return trim($string); } function ExtractFromDelimiters($string, $start, $end) { if (strpos($string, $start) !== false) { $section_retrieved = substr($string, strpos($string, $start) + strlen($start)); $section_retrieved = substr($section_retrieved, 0, strpos($section_retrieved, $end)); return $section_retrieved; } return false; } function StripWithDelimiters($string, $start, $end) { while (strpos($string, $start) !== false) { $section_to_remove = substr($string, strpos($string, $start)); $section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end)); $string = str_replace($section_to_remove, '', $string); } return $string; } function StripRecursiveHTMLSection($string, $tag_name, $tag_start) { $open_tag = '<' . $tag_name; $close_tag = '</' . $tag_name . '>'; $close_tag_length = strlen($close_tag); if (strpos($tag_start, $open_tag) === 0) { while (strpos($string, $tag_start) !== false) { $max_recursion = 100; $section_to_remove = null; $section_start = strpos($string, $tag_start); $search_offset = $section_start; do { $max_recursion--; $section_end = strpos($string, $close_tag, $search_offset); $search_offset = $section_end + $close_tag_length; $section_to_remove = substr($string, $section_start, $section_end - $section_start + $close_tag_length); $open_tag_count = substr_count($section_to_remove, $open_tag); $close_tag_count = substr_count($section_to_remove, $close_tag); } while ($open_tag_count > $close_tag_count && $max_recursion > 0); $string = str_replace($section_to_remove, '', $string); } } return $string; } $baseUri = $this->getURI(); $feed = $param['feed']; if (empty($feed)) { $this->returnError('Please select a feed to display.', 400); } if (strpos($feed, 'downloads!') !== false) { $feed = str_replace('downloads!', '', $feed); $baseUri = str_replace('www.', 'downloads.', $baseUri); } if ($feed !== preg_replace('/[^a-zA-Z0-9-\\/]+/', '', $feed) || substr_count($feed, '/') > 1 || strlen($feed > 64)) { $this->returnError('Invalid "feed" parameter.', 400); } $url = $baseUri . trim($feed, '/') . '/rss.xml'; $html = $this->file_get_html($url) or $this->returnError('Could not request ZDNet: ' . $url, 500); $limit = 0; foreach ($html->find('item') as $element) { if ($limit < 10) { $article_url = preg_replace('/([^#]+)#ftag=.*/', '$1', StripCDATA(ExtractFromDelimiters($element->innertext, '<link>', '</link>'))); $article_author = StripCDATA(ExtractFromDelimiters($element->innertext, 'role="author">', '<')); $article_title = StripCDATA($element->find('title', 0)->plaintext); $article_subtitle = StripCDATA($element->find('description', 0)->plaintext); $article_timestamp = strtotime(StripCDATA($element->find('pubDate', 0)->plaintext)); $article = $this->file_get_html($article_url) or $this->returnError('Could not request ZDNet: ' . $article_url, 500); if (!empty($article_author)) { $author = $article_author; } else { $author = $article->find('meta[name=author]', 0); if (is_object($author)) { $author = $author->content; } else { $author = 'ZDNet'; } } $thumbnail = $article->find('meta[itemprop=image]', 0); if (is_object($thumbnail)) { $thumbnail = $thumbnail->content; } else { $thumbnail = ''; } $contents = $article->find('article', 0)->innertext; foreach (array('<div class="shareBar"', '<div class="shortcodeGalleryWrapper"', '<div class="relatedContent', '<div class="downloadNow', '<div data-shortcode', '<div id="sharethrough', '<div id="inpage-video') as $div_start) { $contents = StripRecursiveHTMLSection($contents, 'div', $div_start); } $contents = StripWithDelimiters($contents, '<script', '</script>'); $contents = StripWithDelimiters($contents, '<meta itemprop="image"', '>'); $contents = trim(StripWithDelimiters($contents, '<section class="sharethrough-top', '</section>')); $content_img = strpos($contents, '<img'); //Look for first image if ($content_img !== false && $content_img < 512 || $thumbnail == '') { $content_img = ''; } else { $content_img = '<p><img src="' . $thumbnail . '" /></p>'; } //Include thumbnail $contents = $content_img . '<p><b>' . $article_subtitle . '</b></p>' . $contents; if ($thumbnail == '') { $thumbnail = 'http://zdnet1.cbsistatic.com/fly/bundles/zdnetcss/images/logos/logo-192x192.png'; } $item = new \Item(); $item->author = $author; $item->uri = $article_url; $item->title = $article_title; $item->thumbnailUri = $thumbnail; $item->timestamp = $article_timestamp; $item->content = $contents; $this->items[] = $item; $limit++; } } }
public function collectData(array $param) { //Utility function for retrieving text based on start and end delimiters function ExtractFromDelimiters($string, $start, $end) { if (strpos($string, $start) !== false) { $section_retrieved = substr($string, strpos($string, $start) + strlen($start)); $section_retrieved = substr($section_retrieved, 0, strpos($section_retrieved, $end)); return $section_retrieved; } return false; } //Ensure proper parameters have been provided if (empty($param['search'])) { $this->returnError('You must specify a search criteria', 400); } //Retrieve torrent listing from search results, which does not contain torrent description $url = $this->getURI() . 'torrents/search/?' . $param['search'] . '&order=added&type=desc'; $html = $this->file_get_html($url) or $this->returnError('Could not request t411: ' . $url, 500); $results = $html->find('table.results', 0); if (is_null($results)) { $this->returnError('No results from t411: ' . $url, 500); } $limit = 0; //Process each item individually foreach ($results->find('tr') as $element) { //Limit total amount of requests and ignore table header if ($limit < 10 && !is_object($element->find('th', 0))) { //Requests are rate-limited usleep(500000); //So we need to wait (500ms) //Retrieve data from RSS entry $item_uri = $this->getURI() . 'torrents/details/?id=' . ExtractFromDelimiters($element->find('a.nfo', 0)->outertext, '?id=', '"'); $item_title = ExtractFromDelimiters($element->outertext, '" title="', '"'); $item_date = strtotime($element->find('dd', 0)->plaintext); //Retrieve full description from torrent page if ($item_html = $this->file_get_html($item_uri)) { //Retrieve data from page contents $item_desc = $item_html->find('div.description', 0); $item_author = $item_html->find('a.profile', 0)->innertext; //Retrieve image for thumbnail or generic logo fallback $item_image = $this->getURI() . 'themes/blue/images/logo.png'; foreach ($item_desc->find('img') as $img) { if (strpos($img->src, 'prez') === false && strpos($img->src, '/ad/') === false) { $item_image = $img->src; break; } } //Cleanup advertisments $divs = explode('<div class="align-center">', $item_desc->innertext); $item_desc = ''; foreach ($divs as $text) { if (strpos($text, 'adprovider.adlure.net') === false) { $item_desc = $item_desc . '<div class="align-center">' . $text; } } $item_desc = preg_replace('/<h2 class="align-center">LIENS DE T..?L..?CHARGEMENT<\\/h2>/i', '', $item_desc); //Build and add final item $item = new \Item(); $item->uri = $item_uri; $item->title = $item_title; $item->author = $item_author; $item->timestamp = $item_date; $item->thumbnailUri = $item_image; $item->content = $item_desc; $this->items[] = $item; $limit++; } } } }
public function collectData(array $param) { function ExtractFromDelimiters($string, $start, $end) { if (strpos($string, $start) !== false) { $section_retrieved = substr($string, strpos($string, $start) + strlen($start)); $section_retrieved = substr($section_retrieved, 0, strpos($section_retrieved, $end)); return $section_retrieved; } return false; } function StripWithDelimiters($string, $start, $end) { while (strpos($string, $start) !== false) { $section_to_remove = substr($string, strpos($string, $start)); $section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end)); $string = str_replace($section_to_remove, '', $string); } return $string; } function build_item($uri, $title, $author, $timestamp, $thumnail, $content) { $item = new \Item(); $item->uri = $uri; $item->title = $title; $item->author = $author; $item->timestamp = $timestamp; $item->thumbnailUri = $thumnail; $item->content = $content; return $item; } function cleanup_post_content($content, $site_url) { $content = str_replace(':arrow:', '➤', $content); $content = str_replace('href="attachments/', 'href="' . $site_url . 'attachments/', $content); $content = StripWithDelimiters($content, '<script', '</script>'); return $content; } function fetch_post_content($uri, $site_url) { $html = $this->file_get_html($uri) or $this->returnError('Could not request GBAtemp: ' . $uri, 500); $content = $html->find('div.messageContent', 0)->innertext; return cleanup_post_content($content, $site_url); } $typeFilter = ''; if (!empty($param['type'])) { if ($param['type'] == 'N' || $param['type'] == 'R' || $param['type'] == 'T' || $param['type'] == 'F') { $typeFilter = $param['type']; if ($typeFilter == 'N') { $this->filter = 'News'; } if ($typeFilter == 'R') { $this->filter = 'Review'; } if ($typeFilter == 'T') { $this->filter = 'Tutorial'; } if ($typeFilter == 'F') { $this->filter = 'Forum'; } } else { $this->returnError('The provided type filter is invalid. Expecting N, R, T, or F.', 400); } } else { $this->returnError('Please provide a type filter. Expecting N, R, T, or F.', 400); } $html = $this->file_get_html($this->getURI()) or $this->returnError('Could not request GBAtemp.', 500); if ($typeFilter == 'N') { foreach ($html->find('li[class=news_item full]') as $newsItem) { $url = $this->getURI() . $newsItem->find('a', 0)->href; $img = $this->getURI() . $newsItem->find('img', 0)->src; $time = intval(ExtractFromDelimiters($newsItem->find('abbr.DateTime', 0)->outertext, 'data-time="', '"')); $author = $newsItem->find('a.username', 0)->plaintext; $title = $newsItem->find('a', 1)->plaintext; $content = fetch_post_content($url, $this->getURI()); $this->items[] = build_item($url, $title, $author, $time, $img, $content); } } else { if ($typeFilter == 'R') { foreach ($html->find('li.portal_review') as $reviewItem) { $url = $this->getURI() . $reviewItem->find('a', 0)->href; $img = $this->getURI() . ExtractFromDelimiters($reviewItem->find('a', 0)->style, 'image:url(', ')'); $title = $reviewItem->find('span.review_title', 0)->plaintext; $content = $this->file_get_html($url) or $this->returnError('Could not request GBAtemp: ' . $uri, 500); $author = $content->find('a.username', 0)->plaintext; $time = intval(ExtractFromDelimiters($content->find('abbr.DateTime', 0)->outertext, 'data-time="', '"')); $intro = '<p><b>' . $content->find('div#review_intro', 0)->plaintext . '</b></p>'; $review = $content->find('div#review_main', 0)->innertext; $subheader = '<p><b>' . $content->find('div.review_subheader', 0)->plaintext . '</b></p>'; $procons = $content->find('table.review_procons', 0)->outertext; $scores = $content->find('table.reviewscores', 0)->outertext; $content = cleanup_post_content($intro . $review . $subheader . $procons . $scores, $this->getURI()); $this->items[] = build_item($url, $title, $author, $time, $img, $content); } } else { if ($typeFilter == 'T') { foreach ($html->find('li.portal-tutorial') as $tutorialItem) { $url = $this->getURI() . $tutorialItem->find('a', 0)->href; $title = $tutorialItem->find('a', 0)->plaintext; $time = intval(ExtractFromDelimiters($tutorialItem->find('abbr.DateTime', 0)->outertext, 'data-time="', '"')); $author = $tutorialItem->find('a.username', 0)->plaintext; $content = fetch_post_content($url, $this->getURI()); $this->items[] = build_item($url, $title, $author, $time, '', $content); } } else { if ($typeFilter == 'F') { foreach ($html->find('li.rc_item') as $postItem) { $url = $this->getURI() . $postItem->find('a', 1)->href; $title = $postItem->find('a', 1)->plaintext; $time = intval(ExtractFromDelimiters($postItem->find('abbr.DateTime', 0)->outertext, 'data-time="', '"')); $author = $postItem->find('a.username', 0)->plaintext; $content = fetch_post_content($url, $this->getURI()); $this->items[] = build_item($url, $title, $author, $time, '', $content); } } } } } }