public function run()
{
try {
$this->csv = new Writer();
$totalPages = $this->crawler->totalPages();
$this->progressBar = new \ProgressBar\Manager(0, $totalPages);
$pool = new \GuzzleHttp\Pool($this->client, $this->crawler->doRequests($totalPages), ['concurrency' => 40, 'fulfilled' => function ($response, $index) {
$html = $response->getBody()->getContents();
if (!empty($html)) {
$rows = $this->crawler->filter($html);
$this->csv->write($rows);
$this->progressBar->update($index);
}
unset($response);
unset($html);
unset($rows);
}, 'rejected' => function ($reason, $index) {
echo $reason->getMessage();
}]);
$promise = $pool->promise();
$promise->wait();
} catch (\Exception $e) {
$response = $e->getMessage();
echo $response;
}
}
/**
* @param string[] $imageSrcs
* @param bool $returnAll
* @param Configuration $config
*
* @return object[]|null
*/
private static function handleEntity($imageSrcs, $returnAll, $config)
{
$guzzle = new \GuzzleHttp\Client();
$results = [];
$requests = function ($urls) use($guzzle, $config, &$results) {
foreach ($urls as $key => $url) {
$file = tempnam(sys_get_temp_dir(), 'goose');
$results[] = (object) ['url' => $url, 'file' => $file];
(yield $key => function ($options) use($guzzle, $url, $file) {
$options['sink'] = $file;
return $guzzle->sendAsync(new \GuzzleHttp\Psr7\Request('GET', $url), $options);
});
}
};
$pool = new \GuzzleHttp\Pool($guzzle, $requests($imageSrcs), ['concurrency' => 25, 'fulfilled' => function ($response, $index) use(&$results, $returnAll) {
if (!$returnAll && $response->getStatusCode() != 200) {
unset($results[$index]);
}
}, 'rejected' => function ($reason, $index) use(&$results, $returnAll) {
if ($returnAll) {
$results[$index]->file = null;
} else {
unset($results[$index]);
}
}, 'options' => $config->get('browser')]);
$pool->promise()->wait();
if (empty($results)) {
return null;
}
return $results;
}