private function showSimple($total_req, $total_size, $stats) { foreach (ContentCategorizer::getAllTypes() as $type => $display_name) { if (isset($stats[$type])) { printf("This %s document is %s bytes\n", $display_name, number_format($total_size)); break; } } }
private function processQueue() { for ($i = 0; $i < $this->promises_total; $i++) { list($url, $promise) = $this->promises[$i]; try { $response = $promise->wait(); } catch (\GuzzleHttp\Exception\ClientException $e) { // 4xx (probably 404) errors // if the resource is missing, there is nothing to report about it // but if this is the main URL (rather than a dependency, should expose that issue) if ($i == 0) { throw new \Exception('This URL is returning a 4xx error, cannot continue'); } continue; } catch (\Exception $e) { // other errors such as garbage URLs if ($i == 0) { throw new \Exception('This URL is not reachable, cannot continue'); } continue; } if ($response->hasHeader('Content-Type')) { $content_type = $response->getHeader('Content-Type')[0]; } else { $content_type = null; } $category = ContentCategorizer::getCategory($content_type); // have we requested the entire resource or just the headers? $is_complete_request = isset($this->all_visited_complete[$url]); // for web pages, find references to other assets if ($category == ContentCategorizer::HTML) { if ($is_complete_request) { // right here we need to load the whole html / response body into a variable // this may use a lot of memory; there is no easy way around that $this->findUrlsInHtml($url, (string) $response->getBody()); } else { $this->queueCompleteUrl($url); } } // example url, record stats if (!$is_complete_request && $response->hasHeader('Content-Length')) { // record size based on header $this->recordStats($url, $category, $response->getHeader('Content-Length')[0]); } else { if (!$is_complete_request) { // request body to see how big it is (if no content length -- last resort) $this->queueCompleteUrl($url); } else { // examine body to see how big it is $this->recordStats($url, $category, $response->getBody()->getSize()); } } } $this->promises = []; }