private function showSimple($total_req, $total_size, $stats)
{
foreach (ContentCategorizer::getAllTypes() as $type => $display_name) {
if (isset($stats[$type])) {
printf("This %s document is %s bytes\n", $display_name, number_format($total_size));
break;
}
}
}
private function processQueue()
{
for ($i = 0; $i < $this->promises_total; $i++) {
list($url, $promise) = $this->promises[$i];
try {
$response = $promise->wait();
} catch (\GuzzleHttp\Exception\ClientException $e) {
// 4xx (probably 404) errors
// if the resource is missing, there is nothing to report about it
// but if this is the main URL (rather than a dependency, should expose that issue)
if ($i == 0) {
throw new \Exception('This URL is returning a 4xx error, cannot continue');
}
continue;
} catch (\Exception $e) {
// other errors such as garbage URLs
if ($i == 0) {
throw new \Exception('This URL is not reachable, cannot continue');
}
continue;
}
if ($response->hasHeader('Content-Type')) {
$content_type = $response->getHeader('Content-Type')[0];
} else {
$content_type = null;
}
$category = ContentCategorizer::getCategory($content_type);
// have we requested the entire resource or just the headers?
$is_complete_request = isset($this->all_visited_complete[$url]);
// for web pages, find references to other assets
if ($category == ContentCategorizer::HTML) {
if ($is_complete_request) {
// right here we need to load the whole html / response body into a variable
// this may use a lot of memory; there is no easy way around that
$this->findUrlsInHtml($url, (string) $response->getBody());
} else {
$this->queueCompleteUrl($url);
}
}
// example url, record stats
if (!$is_complete_request && $response->hasHeader('Content-Length')) {
// record size based on header
$this->recordStats($url, $category, $response->getHeader('Content-Length')[0]);
} else {
if (!$is_complete_request) {
// request body to see how big it is (if no content length -- last resort)
$this->queueCompleteUrl($url);
} else {
// examine body to see how big it is
$this->recordStats($url, $category, $response->getBody()->getSize());
}
}
}
$this->promises = [];
}
