/** * Creates data provider instance with search query applied * * @param array $params * * @return ActiveDataProvider */ public function search($params) { $query = SourcesSettings::find(); $dataProvider = new ActiveDataProvider(['query' => $query]); if ($this->load($params) && !$this->validate()) { return $dataProvider; } $query->andFilterWhere(['id' => $this->id, 'source_id' => $this->source_id]); $query->andFilterWhere(['like', 'name', $this->name])->andFilterWhere(['like', 'value', $this->value]); return $dataProvider; }
/** * Updates an existing RssSources model. * If update is successful, the browser will be redirected to the 'view' page. * * @param integer $id * * @return mixed */ public function actionUpdate($id) { $model = $this->findModel($id); $postData = Yii::$app->request->post(); if ($model->load($postData) && $model->save()) { SourcesSettings::deleteAll(['source_id' => $model->source_id]); if (isset($postData['RssSources']['settings'])) { foreach ($postData['RssSources']['settings'] as $name => $value) { $sourcesSettings = new SourcesSettings(); $sourcesSettings->source_id = $model->source_id; $sourcesSettings->name = $name; $sourcesSettings->value = $value; $sourcesSettings->save(); } } return $this->redirect(['view', 'id' => $model->id]); } else { return $this->render('update', ['model' => $model]); } }
public function __construct(RssSources $source) { echo "Start parsing: {$source->source->label}\n"; $this->source = $source; $this->itemPatterns = SourcesSettings::findAll(['source_id' => $this->source->source_id, 'name' => 'rss_news_item_pattern']); if ($titlePattern = SourcesSettings::findOne(['source_id' => $this->source->source_id, 'name' => 'rss_title'])) { $this->titlePattern = $titlePattern->value; } if ($contentPattern = SourcesSettings::findOne(['source_id' => $this->source->source_id, 'name' => 'rss_content'])) { $this->contentPattern = $contentPattern->value; } if ($linkPattern = SourcesSettings::findOne(['source_id' => $this->source->source_id, 'name' => 'rss_link'])) { $this->linkPattern = $linkPattern->value; } if ($imagePattern = SourcesSettings::findOne(['source_id' => $this->source->source_id, 'name' => 'rss_image'])) { $this->imagePattern = $imagePattern->value; } if ($categoryPattern = SourcesSettings::findOne(['source_id' => $this->source->source_id, 'name' => 'rss_category'])) { $this->categoryPattern = $categoryPattern->value; } }
private function tryContentDetect($html) { if ($patterns = SourcesSettings::findAll(['source_id' => $this->source->id, 'name' => 'content_pattern'])) { $doc = new \DOMDocument("1.0", "utf-8"); $doc->preserveWhiteSpace = false; libxml_use_internal_errors(true); $doc->loadHTML($html); $xpath = new \DOMXpath($doc); $contentResult = false; foreach ($patterns as $pattern) { if ($content = $xpath->evaluate($pattern->value)) { $contentResult .= $doc->saveHTML($content->item(0)); } } return $contentResult; } return false; }
?> <?php echo $form->field($model, 'active')->checkbox(); ?> <?php echo $form->field($model, 'is_full')->checkbox(); ?> <?php $settingsFields = ["rss_news_item_pattern" => ["label" => "Item pattern", "defaultValue" => "//item"], "rss_title" => ["label" => "Title", "defaultValue" => "title"], "rss_link" => ["label" => "Link", "defaultValue" => "link"], "rss_image" => ["label" => "Image", "defaultValue" => "enclosure"], "rss_content" => ["label" => "Content", "defaultValue" => "yandex:full-text"], "rss_category" => ["label" => "RSS Category", "defaultValue" => "category"]]; foreach ($settingsFields as $key => $data) { $placeholder = $data['defaultValue']; $pattern = false; if ($settings = \common\models\SourcesSettings::findOne(['source_id' => $model->source_id, "name" => $key])) { $pattern = $settings->value; } ?> <div class="form-group field-<?php echo $key; ?> "> <label class="control-label" for="<?php echo $key; ?> "><?php echo $data['label']; ?> </label> <input type="text" id="<?php
/** * Finds the SourcesSettings model based on its primary key value. * If the model is not found, a 404 HTTP exception will be thrown. * * @param integer $id * * @return SourcesSettings the loaded model * @throws NotFoundHttpException if the model cannot be found */ protected function findModel($id) { if (($model = SourcesSettings::findOne($id)) !== null) { return $model; } else { throw new NotFoundHttpException('The requested page does not exist.'); } }
/** * @return \yii\db\ActiveQuery */ public function getSourcesSettings() { return $this->hasMany(SourcesSettings::className(), ['source_id' => 'id']); }
private function processCombine($xpath) { $itemPatterns = SourcesSettings::findAll(['source_id' => $this->source->source_id, 'name' => 'rss_news_item_pattern']); if ($titlePattern = SourcesSettings::findOne(['source_id' => $this->source->source_id, 'name' => 'rss_title'])) { $titlePattern = $titlePattern->value; } if ($contentPattern = SourcesSettings::findOne(['source_id' => $this->source->source_id, 'name' => 'rss_content'])) { $contentPattern = $contentPattern->value; } if ($linkPattern = SourcesSettings::findOne(['source_id' => $this->source->source_id, 'name' => 'rss_link'])) { $linkPattern = $linkPattern->value; } if ($imagePattern = SourcesSettings::findOne(['source_id' => $this->source->source_id, 'name' => 'rss_image'])) { $imagePattern = $imagePattern->value; } if ($categoryPattern = SourcesSettings::findOne(['source_id' => $this->source->source_id, 'name' => 'rss_category'])) { $categoryPattern = $categoryPattern->value; } foreach ($itemPatterns as $pattern) { if ($newsList = $xpath->query($pattern->value)) { for ($i = 0; $i < $newsList->length; $i++) { $news = $newsList->item($i); $newsParams = array(); $newsParams['data'] = []; if ($this->source->category_id) { $newsParams['data']['category_id'] = $this->source->category_id; } $newsParams['source'] = $this->source->source; foreach ($news->childNodes as $node) { if ($titlePattern == $node->nodeName) { $newsParams['title'] = $node->nodeValue; } if ($categoryPattern) { if ($categoryPattern == $node->nodeName) { $newsParams['data']['category'] = $node->nodeValue; } } else { if ("category" == strtolower($node->nodeName)) { $newsParams['data']['category'] = $node->nodeValue; } } if ($linkPattern == $node->nodeName) { $newsParams['link'] = str_replace($this->source->source->url, '', $node->nodeValue); // $newsParams['link'] = $node->nodeValue; } if ($imagePattern == $node->nodeName) { if (preg_match_all('/(https?:\\/\\/[a-z0-9\\/_а-я\\-\\.]*\\.(?:png|jpg))/i', $node->nodeValue, $images)) { $newsParams['image_src'] = $images[1][0]; } } if (!isset($newsParams['image_src'])) { if ($node->nodeName == 'enclosure') { if (preg_match_all('/(https?:\\/\\/[a-z0-9\\/_а-я\\-\\.]*\\.(?:png|jpg))/i', $node->getAttribute('url'), $images)) { $newsParams['image_src'] = $images[1][0]; } } } } try { if (array_key_exists("link", $newsParams)) { $pqItem = new ParserQueue(); $pqItem->source_id = $this->source->source_id; $pqItem->url = $newsParams['link']; $pqItem->status = ParserQueue::STATUS_INPROCESS; $pqItem->created_at = new \yii\db\Expression('NOW()'); $pqItem->updated_at = new \yii\db\Expression('NOW()'); if ($pqItem->save()) { $pn = new PendingNews(); $pn->content = ' '; $pn->search_content = ' '; $pn->source_id = $this->source->source_id; if (isset($newsParams['title'])) { $pn->title = $newsParams['title']; } $pn->status = PendingNews::STATUS_SUSPENDED; $pn->group_hash = md5(time()); if (isset($newsParams['image_src'])) { $pn->thumb_src = $newsParams['image_src']; } if ($pqItem) { $pn->pq_id = $pqItem->id; } if (!empty($newsParams['data'])) { $pn->additonal_data = json_encode($newsParams['data']); } $pn->created_at = new \yii\db\Expression("NOW()"); $pn->update_at = new \yii\db\Expression("NOW()"); if ($pn->save()) { if ($pqItem) { $pqItem->status = ParserQueue::STATUS_DONE; $pqItem->save(); $this->mq->postMessage("parse", "parse_rss", json_encode(["pn_id" => $pn->id, "pq_id" => $pqItem->id])); } } else { if ($pqItem) { $pqItem->status = ParserQueue::STATUS_FAIL; $pqItem->save(); } } } else { // print_r( $pqItem->errors ); } } } catch (\yii\db\Exception $e) { // print_r( $e->getMessage() ); } } } } }