/**
* Creates data provider instance with search query applied
*
* @param array $params
*
* @return ActiveDataProvider
*/
public function search($params)
{
$query = SourcesSettings::find();
$dataProvider = new ActiveDataProvider(['query' => $query]);
if ($this->load($params) && !$this->validate()) {
return $dataProvider;
}
$query->andFilterWhere(['id' => $this->id, 'source_id' => $this->source_id]);
$query->andFilterWhere(['like', 'name', $this->name])->andFilterWhere(['like', 'value', $this->value]);
return $dataProvider;
}
/**
* Updates an existing RssSources model.
* If update is successful, the browser will be redirected to the 'view' page.
*
* @param integer $id
*
* @return mixed
*/
public function actionUpdate($id)
{
$model = $this->findModel($id);
$postData = Yii::$app->request->post();
if ($model->load($postData) && $model->save()) {
SourcesSettings::deleteAll(['source_id' => $model->source_id]);
if (isset($postData['RssSources']['settings'])) {
foreach ($postData['RssSources']['settings'] as $name => $value) {
$sourcesSettings = new SourcesSettings();
$sourcesSettings->source_id = $model->source_id;
$sourcesSettings->name = $name;
$sourcesSettings->value = $value;
$sourcesSettings->save();
}
}
return $this->redirect(['view', 'id' => $model->id]);
} else {
return $this->render('update', ['model' => $model]);
}
}
public function __construct(RssSources $source)
{
echo "Start parsing: {$source->source->label}\n";
$this->source = $source;
$this->itemPatterns = SourcesSettings::findAll(['source_id' => $this->source->source_id, 'name' => 'rss_news_item_pattern']);
if ($titlePattern = SourcesSettings::findOne(['source_id' => $this->source->source_id, 'name' => 'rss_title'])) {
$this->titlePattern = $titlePattern->value;
}
if ($contentPattern = SourcesSettings::findOne(['source_id' => $this->source->source_id, 'name' => 'rss_content'])) {
$this->contentPattern = $contentPattern->value;
}
if ($linkPattern = SourcesSettings::findOne(['source_id' => $this->source->source_id, 'name' => 'rss_link'])) {
$this->linkPattern = $linkPattern->value;
}
if ($imagePattern = SourcesSettings::findOne(['source_id' => $this->source->source_id, 'name' => 'rss_image'])) {
$this->imagePattern = $imagePattern->value;
}
if ($categoryPattern = SourcesSettings::findOne(['source_id' => $this->source->source_id, 'name' => 'rss_category'])) {
$this->categoryPattern = $categoryPattern->value;
}
}
private function tryContentDetect($html)
{
if ($patterns = SourcesSettings::findAll(['source_id' => $this->source->id, 'name' => 'content_pattern'])) {
$doc = new \DOMDocument("1.0", "utf-8");
$doc->preserveWhiteSpace = false;
libxml_use_internal_errors(true);
$doc->loadHTML($html);
$xpath = new \DOMXpath($doc);
$contentResult = false;
foreach ($patterns as $pattern) {
if ($content = $xpath->evaluate($pattern->value)) {
$contentResult .= $doc->saveHTML($content->item(0));
}
}
return $contentResult;
}
return false;
}
?>
<?php
echo $form->field($model, 'active')->checkbox();
?>
<?php
echo $form->field($model, 'is_full')->checkbox();
?>
<?php
$settingsFields = ["rss_news_item_pattern" => ["label" => "Item pattern", "defaultValue" => "//item"], "rss_title" => ["label" => "Title", "defaultValue" => "title"], "rss_link" => ["label" => "Link", "defaultValue" => "link"], "rss_image" => ["label" => "Image", "defaultValue" => "enclosure"], "rss_content" => ["label" => "Content", "defaultValue" => "yandex:full-text"], "rss_category" => ["label" => "RSS Category", "defaultValue" => "category"]];
foreach ($settingsFields as $key => $data) {
$placeholder = $data['defaultValue'];
$pattern = false;
if ($settings = \common\models\SourcesSettings::findOne(['source_id' => $model->source_id, "name" => $key])) {
$pattern = $settings->value;
}
?>
<div class="form-group field-<?php
echo $key;
?>
">
<label class="control-label" for="<?php
echo $key;
?>
"><?php
echo $data['label'];
?>
</label>
<input type="text" id="<?php
/**
* Finds the SourcesSettings model based on its primary key value.
* If the model is not found, a 404 HTTP exception will be thrown.
*
* @param integer $id
*
* @return SourcesSettings the loaded model
* @throws NotFoundHttpException if the model cannot be found
*/
protected function findModel($id)
{
if (($model = SourcesSettings::findOne($id)) !== null) {
return $model;
} else {
throw new NotFoundHttpException('The requested page does not exist.');
}
}
/**
* @return \yii\db\ActiveQuery
*/
public function getSourcesSettings()
{
return $this->hasMany(SourcesSettings::className(), ['source_id' => 'id']);
}
private function processCombine($xpath)
{
$itemPatterns = SourcesSettings::findAll(['source_id' => $this->source->source_id, 'name' => 'rss_news_item_pattern']);
if ($titlePattern = SourcesSettings::findOne(['source_id' => $this->source->source_id, 'name' => 'rss_title'])) {
$titlePattern = $titlePattern->value;
}
if ($contentPattern = SourcesSettings::findOne(['source_id' => $this->source->source_id, 'name' => 'rss_content'])) {
$contentPattern = $contentPattern->value;
}
if ($linkPattern = SourcesSettings::findOne(['source_id' => $this->source->source_id, 'name' => 'rss_link'])) {
$linkPattern = $linkPattern->value;
}
if ($imagePattern = SourcesSettings::findOne(['source_id' => $this->source->source_id, 'name' => 'rss_image'])) {
$imagePattern = $imagePattern->value;
}
if ($categoryPattern = SourcesSettings::findOne(['source_id' => $this->source->source_id, 'name' => 'rss_category'])) {
$categoryPattern = $categoryPattern->value;
}
foreach ($itemPatterns as $pattern) {
if ($newsList = $xpath->query($pattern->value)) {
for ($i = 0; $i < $newsList->length; $i++) {
$news = $newsList->item($i);
$newsParams = array();
$newsParams['data'] = [];
if ($this->source->category_id) {
$newsParams['data']['category_id'] = $this->source->category_id;
}
$newsParams['source'] = $this->source->source;
foreach ($news->childNodes as $node) {
if ($titlePattern == $node->nodeName) {
$newsParams['title'] = $node->nodeValue;
}
if ($categoryPattern) {
if ($categoryPattern == $node->nodeName) {
$newsParams['data']['category'] = $node->nodeValue;
}
} else {
if ("category" == strtolower($node->nodeName)) {
$newsParams['data']['category'] = $node->nodeValue;
}
}
if ($linkPattern == $node->nodeName) {
$newsParams['link'] = str_replace($this->source->source->url, '', $node->nodeValue);
// $newsParams['link'] = $node->nodeValue;
}
if ($imagePattern == $node->nodeName) {
if (preg_match_all('/(https?:\\/\\/[a-z0-9\\/_а-я\\-\\.]*\\.(?:png|jpg))/i', $node->nodeValue, $images)) {
$newsParams['image_src'] = $images[1][0];
}
}
if (!isset($newsParams['image_src'])) {
if ($node->nodeName == 'enclosure') {
if (preg_match_all('/(https?:\\/\\/[a-z0-9\\/_а-я\\-\\.]*\\.(?:png|jpg))/i', $node->getAttribute('url'), $images)) {
$newsParams['image_src'] = $images[1][0];
}
}
}
}
try {
if (array_key_exists("link", $newsParams)) {
$pqItem = new ParserQueue();
$pqItem->source_id = $this->source->source_id;
$pqItem->url = $newsParams['link'];
$pqItem->status = ParserQueue::STATUS_INPROCESS;
$pqItem->created_at = new \yii\db\Expression('NOW()');
$pqItem->updated_at = new \yii\db\Expression('NOW()');
if ($pqItem->save()) {
$pn = new PendingNews();
$pn->content = ' ';
$pn->search_content = ' ';
$pn->source_id = $this->source->source_id;
if (isset($newsParams['title'])) {
$pn->title = $newsParams['title'];
}
$pn->status = PendingNews::STATUS_SUSPENDED;
$pn->group_hash = md5(time());
if (isset($newsParams['image_src'])) {
$pn->thumb_src = $newsParams['image_src'];
}
if ($pqItem) {
$pn->pq_id = $pqItem->id;
}
if (!empty($newsParams['data'])) {
$pn->additonal_data = json_encode($newsParams['data']);
}
$pn->created_at = new \yii\db\Expression("NOW()");
$pn->update_at = new \yii\db\Expression("NOW()");
if ($pn->save()) {
if ($pqItem) {
$pqItem->status = ParserQueue::STATUS_DONE;
$pqItem->save();
$this->mq->postMessage("parse", "parse_rss", json_encode(["pn_id" => $pn->id, "pq_id" => $pqItem->id]));
}
} else {
if ($pqItem) {
$pqItem->status = ParserQueue::STATUS_FAIL;
$pqItem->save();
}
}
} else {
// print_r( $pqItem->errors );
}
}
} catch (\yii\db\Exception $e) {
// print_r( $e->getMessage() );
}
}
}
}
}
