public function crawl($url)
{
$this->url = $url;
Log::info('Start to fetch : ' . $this->url);
if ($this->referer) {
$this->addHeader('Referer', $this->referer);
} else {
$this->referer = $url;
}
try {
$startTime = microtime(true);
$res = $this->client->get($this->url, ['headers' => $this->headers]);
$useTime = microtime(true) - $startTime;
if ($res->getStatusCode() == 200) {
Log::success('Crawl Success.');
$resp = new Response($res, $this->url);
} else {
Log::error('Crawl Got Status ' . $res->getStatusCode());
$resp = false;
}
Log::info('use time: ' . $useTime . ' seconds.');
} catch (RequestException $e) {
Log::error('request error: ' . $e->getMessage());
$resp = false;
}
if ($this->interval > 0) {
sleep($this->interval);
}
return $resp;
}
public function result(array $item = [])
{
Log::info('Method `result` is not override');
}
