/**
* Creates a new CrawlerUrls model.
* If creation is successful, the browser will be redirected to the 'view' page.
* @return mixed
*/
public function actionCreate()
{
$model = new CrawlerUrls();
if ($model->load(Yii::$app->request->post()) && $model->save()) {
return $this->redirect(['view', 'id' => $model->id]);
} else {
return $this->render('create', ['model' => $model]);
}
}
/**
* 增加 待采集的 URL
* @param $urls
* @param string $urlPath 如果url是相对路径,需要对路径进行拼接
* @return string
*/
public function addCrawlerUrls($currentUrl, $urls)
{
if (empty($urls)) {
return '';
}
$commonUrl = new CommonUrl();
if (is_string($urls)) {
$url = $commonUrl->getFullUrl($currentUrl, $urls);
if (!$this->getCrawlerUrlByUrl($url)) {
$crawlerUrls = new CrawlerUrls();
$crawlerUrls->url = $url;
$crawlerUrls->save();
}
} elseif (is_array($urls)) {
foreach ($urls as $url) {
$url = $commonUrl->getFullUrl($currentUrl, $url);
if (!$this->getCrawlerUrlByUrl($url)) {
$crawlerUrls = new CrawlerUrls();
$crawlerUrls->url = $url;
$crawlerUrls->save();
}
}
}
}
