protected function findModel($id)
{
if (($model = Settings::findOne($id)) !== null) {
return $model;
} else {
throw new NotFoundHttpException('The requested page does not exist.');
}
}
public function __construct(PendingNews $pn)
{
if ($contentWeight = Settings::findOne(['name' => 'similar_weight'])) {
$this->contentSimilar = $contentWeight->value;
}
if ($titleWeight = Settings::findOne(['name' => 'title_similar_weight'])) {
$this->titleSimilar = $titleWeight->value;
}
$this->news = $pn;
}
public function run()
{
echo "Try parse `{$this->url}`\n";
if ($html = PageLoaderComponent::load($this->url)) {
preg_match('/<meta.*?charset=("?|\\")(.*?)("|\\")/i', $html, $matches);
if (isset($matches[2])) {
if ($charset = $matches[2]) {
$html = mb_convert_encoding($html, "UTF-8", $charset);
} else {
echo "ERROR ON ENCODING DETECTING";
}
} else {
if ($defaultEncoding = SourcesSettings::findOne(['source_id' => $this->source->id, 'name' => 'default_encoding'])) {
$html = mb_convert_encoding($html, "UTF-8", $defaultEncoding->value);
} else {
$html = mb_convert_encoding($html, "UTF-8");
}
}
try {
$html = $this->stripTagWithContent($html, "script");
// $htmlToDetect = $this->processExcludeElements( $html );
// $content = $this->tryContentDetect( $htmlToDetect );
$readability = new Readabillity($this->url);
if ($readability) {
$title = "test title";
$title = $this->processTitleStopWords($title);
// if ( ! $content) {
$content = $readability->getContent();
// die("content: ".$content);
// }
$content = $this->processContentStopWords($content);
$content = preg_replace('/\\n/', ' ', $content);
// $content = strip_tags( $content,
// "<p><div><img><span><br><ul><li><embed><iframe><strong><h1><h2><h3><h4>" );
$content = $this->fixUrls($content);
$content = $this->processExcludeElements($content);
if ($date = $this->processPublishDate($content)) {
if (!(date("Y-m-d") == date("Y-m-d", $date))) {
throw new Exception("Old post");
}
}
if ($searchContent = trim(strip_tags($content))) {
$searchContent = preg_replace('/\\n/', ' ', $searchContent);
if (count(explode(" ", $searchContent)) >= Settings::findOne(['name' => 'news_min_length'])->value) {
if ($this->pendingNews) {
$this->pendingNews->content = $content;
$this->pendingNews->search_content = $searchContent;
$this->pendingNews->status = PendingNews::STATUS_NEW;
if (!$this->pendingNews->thumb_src) {
if ($thumbUrl = $this->detectThumb($html, $content)) {
$this->pendingNews->thumb_src = $thumbUrl;
}
}
if ($this->pendingNews->save()) {
try {
PendingNews::fillTags($this->pendingNews->search_content, $this->pendingNews->id);
} catch (\Exception $e) {
print_r($e->getMessage());
}
$mq = new RabbitMQComponent();
$mq->postMessage("compile", "compile", json_encode(["pn_id" => $this->pendingNews->id]));
$this->parserQueue->status = ParserQueue::STATUS_DONE;
$this->parserQueue->save();
return true;
} else {
print_r($this->pendingNews->getErrors());
$this->parserQueue->status = ParserQueue::STATUS_FAIL;
$this->parserQueue->save();
}
} else {
echo PHP_EOL . "NEWS CREATION" . PHP_EOL;
$pn = new PendingNews();
$pn->source_id = $this->source->id;
$pn->title = $title;
$pn->content = $content;
$pn->search_content = $searchContent;
$pn->status = PendingNews::STATUS_NEW;
$pn->group_hash = md5(time());
$pn->thumb_src = $this->detectThumb($html, $content);
$pn->pq_id = $this->parserQueue->id;
$pn->created_at = new \yii\db\Expression("NOW()");
if ($pn->save()) {
$this->parserQueue->status = ParserQueue::STATUS_DONE;
$this->parserQueue->save();
return true;
} else {
echo PHP_EOL . "ERROR" . PHP_EOL;
print_r($pn->getErrors());
$this->parserQueue->status = ParserQueue::STATUS_FAIL;
$this->parserQueue->save();
}
}
} else {
$this->parserQueue->status = ParserQueue::STATUS_FAIL;
$this->parserQueue->save();
}
}
} else {
throw new Exception('Looks like we couldn\'t find the content. :(');
}
} catch (Exception $e) {
print_r($e);
$this->parserQueue->status = ParserQueue::STATUS_FAIL;
$this->parserQueue->save();
}
} else {
$this->parserQueue->status = ParserQueue::STATUS_FAIL;
$this->parserQueue->save();
}
}
public static function add(Sources $source, $title, $content, $image_src, $status = PendingNews::STATUS_NEW, ParserQueue $parser_queue = null, $data = [])
{
if ($searchContent = trim(strip_tags($content))) {
$searchContent = preg_replace("/[^а-яa-z ]/ui", "", $searchContent);
if (count(explode(" ", $searchContent)) >= Settings::findOne(['name' => 'news_min_length'])->value) {
$pn = new PendingNews();
$pn->source_id = $source->id;
$pn->title = $title;
$pn->content = $content;
$pn->search_content = $searchContent;
$pn->status = $status;
$pn->group_hash = md5(microtime());
$pn->thumb_src = $image_src;
if ($parser_queue) {
$pn->pq_id = $parser_queue->id;
}
if (!empty($data)) {
$pn->additonal_data = json_encode($data);
}
$pn->created_at = new \yii\db\Expression("NOW()");
$pn->update_at = new \yii\db\Expression("NOW()");
if ($pn->save()) {
if ($parser_queue) {
$parser_queue->status = ParserQueue::STATUS_DONE;
$parser_queue->save();
}
} else {
if ($parser_queue) {
$parser_queue->status = ParserQueue::STATUS_FAIL;
$parser_queue->save();
}
}
}
}
}
protected function resolveRequestUri()
{
$selected = Settings::findOne('backend_language')->value;
Languages::setCurrent($selected ? $selected : Languages::getDefaultLanguage()->url);
return parent::resolveRequestUri();
}