public function collectData(array $param)
 {
     function StripCDATA($string)
     {
         $string = str_replace('<![CDATA[', '', $string);
         $string = str_replace(']]>', '', $string);
         return $string;
     }
     function StripWithDelimiters($string, $start, $end)
     {
         while (strpos($string, $start) !== false) {
             $section_to_remove = substr($string, strpos($string, $start));
             $section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end));
             $string = str_replace($section_to_remove, '', $string);
         }
         return $string;
     }
     function CleanArticle($article_html)
     {
         $article_html = StripWithDelimiters($article_html, '<script', '</script>');
         $article_html = StripWithDelimiters($article_html, '<h1 class="cleanprint-title"', '</h1>');
         return $article_html;
     }
     $feedUrl = 'http://www.lemondeinformatique.fr/rss/rss.xml';
     $html = $this->file_get_html($feedUrl) or $this->returnError('Could not request LeMondeInformatique: ' . $feedUrl, 500);
     $limit = 0;
     foreach ($html->find('item') as $element) {
         if ($limit < 5) {
             //Retrieve article details
             $article_uri = $element->innertext;
             $article_uri = substr($article_uri, strpos($article_uri, '<link>') + 6);
             $article_uri = substr($article_uri, 0, strpos($article_uri, '</link>'));
             $article_html = $this->file_get_html($article_uri) or $this->returnError('Could not request LeMondeInformatique: ' . $article_uri, 500);
             $thumbnailUri = $article_html->find('div#article', 0)->find('img#illustration', 0)->src;
             $article_content = CleanArticle($article_html->find('div#article', 0)->innertext);
             $article_title = $article_html->find('h1.cleanprint-title', 0)->plaintext;
             //Build and add final item
             $item = new \Item();
             $item->uri = $article_uri;
             $item->thumbnailUri = $thumbnailUri;
             $item->title = $article_title;
             $item->author = StripCDATA($element->find('dc:creator', 0)->innertext);
             $item->timestamp = strtotime($element->find('dc:date', 0)->plaintext);
             $item->content = $article_content;
             $this->items[] = $item;
             $limit++;
         }
     }
 }
Ejemplo n.º 2
0
 public function collectData(array $param)
 {
     function ExtractFromDelimiters($string, $start, $end)
     {
         if (strpos($string, $start) !== false) {
             $section_retrieved = substr($string, strpos($string, $start) + strlen($start));
             $section_retrieved = substr($section_retrieved, 0, strpos($section_retrieved, $end));
             return $section_retrieved;
         }
         return false;
     }
     function StripWithDelimiters($string, $start, $end)
     {
         while (strpos($string, $start) !== false) {
             $section_to_remove = substr($string, strpos($string, $start));
             $section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end));
             $string = str_replace($section_to_remove, '', $string);
         }
         return $string;
     }
     function CleanArticle($article_html)
     {
         $article_html = '<p>' . substr($article_html, strpos($article_html, '<p>') + 3);
         $article_html = StripWithDelimiters($article_html, '<span class="credit">', '</span>');
         $article_html = StripWithDelimiters($article_html, '<script', '</script>');
         $article_html = StripWithDelimiters($article_html, '<div class="shortcode related-links', '</div>');
         $article_html = StripWithDelimiters($article_html, '<a class="clickToEnlarge">', '</a>');
         return $article_html;
     }
     if (!empty($param['topic'])) {
         $this->topicName = $param['topic'];
     }
     $pageUrl = 'http://www.cnet.com/' . (empty($this->topicName) ? '' : 'topics/' . $this->topicName . '/');
     $html = $this->file_get_html($pageUrl) or $this->returnError('Could not request CNET: ' . $pageUrl, 500);
     $limit = 0;
     foreach ($html->find('div.assetBody') as $element) {
         if ($limit < 8) {
             $article_title = trim($element->find('h2', 0)->plaintext);
             $article_uri = 'http://www.cnet.com' . $element->find('a', 0)->href;
             $article_thumbnail = $element->parent()->find('img', 0)->src;
             $article_timestamp = strtotime($element->find('time.assetTime', 0)->plaintext);
             $article_author = trim($element->find('a[rel=author]', 0)->plaintext);
             if (!empty($article_title) && !empty($article_uri) && strpos($article_uri, '/news/') !== false) {
                 $article_html = $this->file_get_html($article_uri) or $this->returnError('Could not request CNET: ' . $article_uri, 500);
                 if (is_null($article_thumbnail)) {
                     $article_thumbnail = $article_html->find('div.originalImage', 0);
                 }
                 if (is_null($article_thumbnail)) {
                     $article_thumbnail = $article_html->find('span.imageContainer', 0);
                 }
                 if (is_object($article_thumbnail)) {
                     $article_thumbnail = $article_thumbnail->find('img', 0)->src;
                 }
                 $article_content = trim(CleanArticle(ExtractFromDelimiters($article_html, '<div class="articleContent', '<footer>')));
                 $item = new \Item();
                 $item->uri = $article_uri;
                 $item->thumbnailUri = $article_thumbnail;
                 $item->title = $article_title;
                 $item->author = $article_author;
                 $item->timestamp = $article_timestamp;
                 $item->content = $article_content;
                 $this->items[] = $item;
                 $limit++;
             }
         }
     }
 }
Ejemplo n.º 3
0
 public function collectData(array $param)
 {
     function ExtractFromDelimiters($string, $start, $end)
     {
         if (strpos($string, $start) !== false) {
             $section_retrieved = substr($string, strpos($string, $start) + strlen($start));
             $section_retrieved = substr($section_retrieved, 0, strpos($section_retrieved, $end));
             return $section_retrieved;
         }
         return false;
     }
     function StripWithDelimiters($string, $start, $end)
     {
         while (strpos($string, $start) !== false) {
             $section_to_remove = substr($string, strpos($string, $start));
             $section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end));
             $string = str_replace($section_to_remove, '', $string);
         }
         return $string;
     }
     function CleanArticle($article_html)
     {
         $article_html = '<p>' . substr($article_html, strpos($article_html, '</script></div><p>') + 18);
         $article_html = StripWithDelimiters($article_html, '<script>', '</script>');
         $article_html = StripWithDelimiters($article_html, '<div class="shortcode related-links', '</div>');
         $article_html = StripWithDelimiters($article_html, '<a class="clickToEnlarge">', '</a>');
         return $article_html;
     }
     if (!empty($param['topic'])) {
         $this->topicName = $param['topic'];
     }
     $pageUrl = 'http://www.cnet.com/' . (empty($this->topicName) ? '' : 'topics/' . $this->topicName . '/');
     $html = file_get_html($pageUrl) or $this->returnError('Could not request CNET: ' . $pageUrl, 500);
     $limit = 0;
     foreach ($html->find('div.socialSharingSmall') as $element) {
         if ($limit < 4) {
             $article_meta = json_decode(ExtractFromDelimiters($element->outertext, 'data-social-counts-options=\'', '\'>'));
             $article_title = $article_meta->title;
             $article_uri = $article_meta->url;
             if (!empty($article_title) && !empty($article_uri) && strpos($article_uri, '/news/') !== false) {
                 $article_html = file_get_html($article_uri) or $this->returnError('Could not request CNET: ' . $article_uri, 500);
                 $article_timestamp = strtotime(ExtractFromDelimiters($article_html->innertext, '<time itemprop="datePublished" class="', '">'));
                 $article_thumbnail = $article_html->find('div.originalImage', 0);
                 if (is_null($article_thumbnail)) {
                     $article_thumbnail = $article_html->find('span.imageContainer', 0);
                 }
                 if (!is_null($article_thumbnail)) {
                     $article_thumbnail = $article_thumbnail->find('img', 0)->src;
                 }
                 $article_content = trim(CleanArticle(ExtractFromDelimiters($article_html, '</div></div></div><div class="col-8">', '<footer>')));
                 $article_author = trim($article_html->find('a.author', 0)->plaintext);
                 $item = new \Item();
                 $item->uri = $article_uri;
                 $item->thumbnailUri = $article_thumbnail;
                 $item->title = $article_title;
                 $item->author = $article_author;
                 $item->timestamp = $article_timestamp;
                 $item->content = $article_content;
                 $this->items[] = $item;
                 $limit++;
             }
         }
     }
 }