protected function findModel($id)
 {
     if (($model = Settings::findOne($id)) !== null) {
         return $model;
     } else {
         throw new NotFoundHttpException('The requested page does not exist.');
     }
 }
예제 #2
0
 public function __construct(PendingNews $pn)
 {
     if ($contentWeight = Settings::findOne(['name' => 'similar_weight'])) {
         $this->contentSimilar = $contentWeight->value;
     }
     if ($titleWeight = Settings::findOne(['name' => 'title_similar_weight'])) {
         $this->titleSimilar = $titleWeight->value;
     }
     $this->news = $pn;
 }
예제 #3
0
 public function run()
 {
     echo "Try parse `{$this->url}`\n";
     if ($html = PageLoaderComponent::load($this->url)) {
         preg_match('/<meta.*?charset=("?|\\")(.*?)("|\\")/i', $html, $matches);
         if (isset($matches[2])) {
             if ($charset = $matches[2]) {
                 $html = mb_convert_encoding($html, "UTF-8", $charset);
             } else {
                 echo "ERROR ON ENCODING DETECTING";
             }
         } else {
             if ($defaultEncoding = SourcesSettings::findOne(['source_id' => $this->source->id, 'name' => 'default_encoding'])) {
                 $html = mb_convert_encoding($html, "UTF-8", $defaultEncoding->value);
             } else {
                 $html = mb_convert_encoding($html, "UTF-8");
             }
         }
         try {
             $html = $this->stripTagWithContent($html, "script");
             //                    $htmlToDetect = $this->processExcludeElements( $html );
             //                    $content      = $this->tryContentDetect( $htmlToDetect );
             $readability = new Readabillity($this->url);
             if ($readability) {
                 $title = "test title";
                 $title = $this->processTitleStopWords($title);
                 //                        if ( ! $content) {
                 $content = $readability->getContent();
                 //                        die("content: ".$content);
                 //                        }
                 $content = $this->processContentStopWords($content);
                 $content = preg_replace('/\\n/', ' ', $content);
                 //                        $content                    = strip_tags( $content,
                 //                            "<p><div><img><span><br><ul><li><embed><iframe><strong><h1><h2><h3><h4>" );
                 $content = $this->fixUrls($content);
                 $content = $this->processExcludeElements($content);
                 if ($date = $this->processPublishDate($content)) {
                     if (!(date("Y-m-d") == date("Y-m-d", $date))) {
                         throw new Exception("Old post");
                     }
                 }
                 if ($searchContent = trim(strip_tags($content))) {
                     $searchContent = preg_replace('/\\n/', ' ', $searchContent);
                     if (count(explode(" ", $searchContent)) >= Settings::findOne(['name' => 'news_min_length'])->value) {
                         if ($this->pendingNews) {
                             $this->pendingNews->content = $content;
                             $this->pendingNews->search_content = $searchContent;
                             $this->pendingNews->status = PendingNews::STATUS_NEW;
                             if (!$this->pendingNews->thumb_src) {
                                 if ($thumbUrl = $this->detectThumb($html, $content)) {
                                     $this->pendingNews->thumb_src = $thumbUrl;
                                 }
                             }
                             if ($this->pendingNews->save()) {
                                 try {
                                     PendingNews::fillTags($this->pendingNews->search_content, $this->pendingNews->id);
                                 } catch (\Exception $e) {
                                     print_r($e->getMessage());
                                 }
                                 $mq = new RabbitMQComponent();
                                 $mq->postMessage("compile", "compile", json_encode(["pn_id" => $this->pendingNews->id]));
                                 $this->parserQueue->status = ParserQueue::STATUS_DONE;
                                 $this->parserQueue->save();
                                 return true;
                             } else {
                                 print_r($this->pendingNews->getErrors());
                                 $this->parserQueue->status = ParserQueue::STATUS_FAIL;
                                 $this->parserQueue->save();
                             }
                         } else {
                             echo PHP_EOL . "NEWS CREATION" . PHP_EOL;
                             $pn = new PendingNews();
                             $pn->source_id = $this->source->id;
                             $pn->title = $title;
                             $pn->content = $content;
                             $pn->search_content = $searchContent;
                             $pn->status = PendingNews::STATUS_NEW;
                             $pn->group_hash = md5(time());
                             $pn->thumb_src = $this->detectThumb($html, $content);
                             $pn->pq_id = $this->parserQueue->id;
                             $pn->created_at = new \yii\db\Expression("NOW()");
                             if ($pn->save()) {
                                 $this->parserQueue->status = ParserQueue::STATUS_DONE;
                                 $this->parserQueue->save();
                                 return true;
                             } else {
                                 echo PHP_EOL . "ERROR" . PHP_EOL;
                                 print_r($pn->getErrors());
                                 $this->parserQueue->status = ParserQueue::STATUS_FAIL;
                                 $this->parserQueue->save();
                             }
                         }
                     } else {
                         $this->parserQueue->status = ParserQueue::STATUS_FAIL;
                         $this->parserQueue->save();
                     }
                 }
             } else {
                 throw new Exception('Looks like we couldn\'t find the content. :(');
             }
         } catch (Exception $e) {
             print_r($e);
             $this->parserQueue->status = ParserQueue::STATUS_FAIL;
             $this->parserQueue->save();
         }
     } else {
         $this->parserQueue->status = ParserQueue::STATUS_FAIL;
         $this->parserQueue->save();
     }
 }
예제 #4
0
 public static function add(Sources $source, $title, $content, $image_src, $status = PendingNews::STATUS_NEW, ParserQueue $parser_queue = null, $data = [])
 {
     if ($searchContent = trim(strip_tags($content))) {
         $searchContent = preg_replace("/[^а-яa-z ]/ui", "", $searchContent);
         if (count(explode(" ", $searchContent)) >= Settings::findOne(['name' => 'news_min_length'])->value) {
             $pn = new PendingNews();
             $pn->source_id = $source->id;
             $pn->title = $title;
             $pn->content = $content;
             $pn->search_content = $searchContent;
             $pn->status = $status;
             $pn->group_hash = md5(microtime());
             $pn->thumb_src = $image_src;
             if ($parser_queue) {
                 $pn->pq_id = $parser_queue->id;
             }
             if (!empty($data)) {
                 $pn->additonal_data = json_encode($data);
             }
             $pn->created_at = new \yii\db\Expression("NOW()");
             $pn->update_at = new \yii\db\Expression("NOW()");
             if ($pn->save()) {
                 if ($parser_queue) {
                     $parser_queue->status = ParserQueue::STATUS_DONE;
                     $parser_queue->save();
                 }
             } else {
                 if ($parser_queue) {
                     $parser_queue->status = ParserQueue::STATUS_FAIL;
                     $parser_queue->save();
                 }
             }
         }
     }
 }
예제 #5
0
 protected function resolveRequestUri()
 {
     $selected = Settings::findOne('backend_language')->value;
     Languages::setCurrent($selected ? $selected : Languages::getDefaultLanguage()->url);
     return parent::resolveRequestUri();
 }