public function __construct(RssSources $source)
{
echo "Start parsing: {$source->source->label}\n";
$this->source = $source;
$this->itemPatterns = SourcesSettings::findAll(['source_id' => $this->source->source_id, 'name' => 'rss_news_item_pattern']);
if ($titlePattern = SourcesSettings::findOne(['source_id' => $this->source->source_id, 'name' => 'rss_title'])) {
$this->titlePattern = $titlePattern->value;
}
if ($contentPattern = SourcesSettings::findOne(['source_id' => $this->source->source_id, 'name' => 'rss_content'])) {
$this->contentPattern = $contentPattern->value;
}
if ($linkPattern = SourcesSettings::findOne(['source_id' => $this->source->source_id, 'name' => 'rss_link'])) {
$this->linkPattern = $linkPattern->value;
}
if ($imagePattern = SourcesSettings::findOne(['source_id' => $this->source->source_id, 'name' => 'rss_image'])) {
$this->imagePattern = $imagePattern->value;
}
if ($categoryPattern = SourcesSettings::findOne(['source_id' => $this->source->source_id, 'name' => 'rss_category'])) {
$this->categoryPattern = $categoryPattern->value;
}
}
private function tryContentDetect($html)
{
if ($patterns = SourcesSettings::findAll(['source_id' => $this->source->id, 'name' => 'content_pattern'])) {
$doc = new \DOMDocument("1.0", "utf-8");
$doc->preserveWhiteSpace = false;
libxml_use_internal_errors(true);
$doc->loadHTML($html);
$xpath = new \DOMXpath($doc);
$contentResult = false;
foreach ($patterns as $pattern) {
if ($content = $xpath->evaluate($pattern->value)) {
$contentResult .= $doc->saveHTML($content->item(0));
}
}
return $contentResult;
}
return false;
}
private function processCombine($xpath)
{
$itemPatterns = SourcesSettings::findAll(['source_id' => $this->source->source_id, 'name' => 'rss_news_item_pattern']);
if ($titlePattern = SourcesSettings::findOne(['source_id' => $this->source->source_id, 'name' => 'rss_title'])) {
$titlePattern = $titlePattern->value;
}
if ($contentPattern = SourcesSettings::findOne(['source_id' => $this->source->source_id, 'name' => 'rss_content'])) {
$contentPattern = $contentPattern->value;
}
if ($linkPattern = SourcesSettings::findOne(['source_id' => $this->source->source_id, 'name' => 'rss_link'])) {
$linkPattern = $linkPattern->value;
}
if ($imagePattern = SourcesSettings::findOne(['source_id' => $this->source->source_id, 'name' => 'rss_image'])) {
$imagePattern = $imagePattern->value;
}
if ($categoryPattern = SourcesSettings::findOne(['source_id' => $this->source->source_id, 'name' => 'rss_category'])) {
$categoryPattern = $categoryPattern->value;
}
foreach ($itemPatterns as $pattern) {
if ($newsList = $xpath->query($pattern->value)) {
for ($i = 0; $i < $newsList->length; $i++) {
$news = $newsList->item($i);
$newsParams = array();
$newsParams['data'] = [];
if ($this->source->category_id) {
$newsParams['data']['category_id'] = $this->source->category_id;
}
$newsParams['source'] = $this->source->source;
foreach ($news->childNodes as $node) {
if ($titlePattern == $node->nodeName) {
$newsParams['title'] = $node->nodeValue;
}
if ($categoryPattern) {
if ($categoryPattern == $node->nodeName) {
$newsParams['data']['category'] = $node->nodeValue;
}
} else {
if ("category" == strtolower($node->nodeName)) {
$newsParams['data']['category'] = $node->nodeValue;
}
}
if ($linkPattern == $node->nodeName) {
$newsParams['link'] = str_replace($this->source->source->url, '', $node->nodeValue);
// $newsParams['link'] = $node->nodeValue;
}
if ($imagePattern == $node->nodeName) {
if (preg_match_all('/(https?:\\/\\/[a-z0-9\\/_а-я\\-\\.]*\\.(?:png|jpg))/i', $node->nodeValue, $images)) {
$newsParams['image_src'] = $images[1][0];
}
}
if (!isset($newsParams['image_src'])) {
if ($node->nodeName == 'enclosure') {
if (preg_match_all('/(https?:\\/\\/[a-z0-9\\/_а-я\\-\\.]*\\.(?:png|jpg))/i', $node->getAttribute('url'), $images)) {
$newsParams['image_src'] = $images[1][0];
}
}
}
}
try {
if (array_key_exists("link", $newsParams)) {
$pqItem = new ParserQueue();
$pqItem->source_id = $this->source->source_id;
$pqItem->url = $newsParams['link'];
$pqItem->status = ParserQueue::STATUS_INPROCESS;
$pqItem->created_at = new \yii\db\Expression('NOW()');
$pqItem->updated_at = new \yii\db\Expression('NOW()');
if ($pqItem->save()) {
$pn = new PendingNews();
$pn->content = ' ';
$pn->search_content = ' ';
$pn->source_id = $this->source->source_id;
if (isset($newsParams['title'])) {
$pn->title = $newsParams['title'];
}
$pn->status = PendingNews::STATUS_SUSPENDED;
$pn->group_hash = md5(time());
if (isset($newsParams['image_src'])) {
$pn->thumb_src = $newsParams['image_src'];
}
if ($pqItem) {
$pn->pq_id = $pqItem->id;
}
if (!empty($newsParams['data'])) {
$pn->additonal_data = json_encode($newsParams['data']);
}
$pn->created_at = new \yii\db\Expression("NOW()");
$pn->update_at = new \yii\db\Expression("NOW()");
if ($pn->save()) {
if ($pqItem) {
$pqItem->status = ParserQueue::STATUS_DONE;
$pqItem->save();
$this->mq->postMessage("parse", "parse_rss", json_encode(["pn_id" => $pn->id, "pq_id" => $pqItem->id]));
}
} else {
if ($pqItem) {
$pqItem->status = ParserQueue::STATUS_FAIL;
$pqItem->save();
}
}
} else {
// print_r( $pqItem->errors );
}
}
} catch (\yii\db\Exception $e) {
// print_r( $e->getMessage() );
}
}
}
}
}
