public function next()
{
if (count($this->responseCache) == 0) {
$urls = $this->pageContainer->pop($this->parallelReqeusts);
if (empty($urls)) {
return false;
}
$requests = array();
foreach ($urls as $url) {
if (!$this->isFiltered($url)) {
$request = RequestFactory::getRequest($url, 'GET', 'php://memory', [], []);
$requests[] = $request;
}
}
if (empty($requests)) {
return $this->next();
}
try {
$this->responseCache = $this->httpClient->sendRequests($requests);
} catch (MultiHttpAdapterException $e) {
$exceptions = $e->getExceptions();
$errorMessages = "";
foreach ($exceptions as $exception) {
// @fixme this must be part of the http client
$message = $exception->getMessage();
if (strpos($message, "An error occurred when fetching the URI") === 0) {
$url = substr($message, "41", strpos($message, '"', 41) - 41);
if (strpos($url, '/') === 0) {
$this->pageContainer->push(new Uri($this->startUri->getScheme() . '://' . $this->startUri->getHost() . $url));
}
} else {
$errorMessages .= $exception->getMessage() . "\n";
}
}
if ($errorMessages != "") {
throw new \RuntimeException($errorMessages);
}
}
}
if (empty($this->responseCache)) {
return $this->next();
}
$response = array_pop($this->responseCache);
if ($response->hasHeader('Content-Type')) {
$contentTypeElements = explode(';', $response->getHeader('Content-Type')[0]);
$contentType = array_shift($contentTypeElements);
if ($contentType === "text/html") {
$document = new Document((string) $response->getBody(), true);
$elements = $document->getUnorderedDependencies($response->getUri());
foreach ($elements as $element) {
$urlString = $this->createCleanUriString($element);
if (!array_key_exists($urlString, $this->comingFrom)) {
$this->comingFrom[$urlString] = $response->getUri();
}
$this->pageContainer->push($element);
}
}
}
return $response;
}
