public static function grabLinks()
{
$client = new Client();
$baseUrl = 'http://olx.ua/nedvizhimost/cri/?page=%d';
$page = 1;
$retryCount = 5;
do {
$time = time();
$response = null;
try {
$response = $client->get(sprintf($baseUrl, $page++));
} catch (ServerException $e) {
Log::error('OLX server error', ['code' => $e->getCode(), 'message' => $e->getMessage()]);
return;
}
$urls = [];
if ($response) {
$crawler = new Crawler($response->getBody()->getContents());
$urls = array_map(function ($link) {
return preg_replace('/#.*$/', '', $link);
}, $crawler->filter('td.offer a.link')->extract('href'));
}
if (0 === count($urls)) {
Log::info('Grabbed 0 OLX links');
}
$grabbedCount = 0;
$droppedCount = 0;
/** @var Dispatcher $dispatcher */
$dispatcher = app('Illuminate\\Bus\\Dispatcher');
foreach ($urls as $url) {
/** @var GrabbedUrl $grabbedUrl */
$grabbedUrl = GrabbedUrl::where('url', $url)->first();
if ($grabbedUrl) {
/** @var GrabbedUrl $grabbedUrl */
$grabbedUrl->count = $grabbedUrl->count + 1;
$grabbedUrl->save();
$droppedCount++;
} else {
$grabbedUrl = GrabbedUrl::create(['url' => $url]);
$dispatcher->dispatch(new Parse($grabbedUrl));
$grabbedCount++;
}
}
printf('[%d] grabbed: %d, dropped: %d' . PHP_EOL, time() - $time, $grabbedCount, $droppedCount);
} while ($grabbedCount > 0 or $retryCount-- > 0);
}
