/**
* Reads the response-header.
*
* @param int &$error_code Error-code by reference if an error occured.
* @param string &$error_string Error-string by reference
*
* @return string The response-header or NULL if an error occured
*/
protected function readResponseHeader(&$error_code, &$error_string)
{
PHPCrawlerBenchmark::reset("server_response_time");
PHPCrawlerBenchmark::start("server_response_time");
$status = socket_get_status($this->socket);
$source_read = "";
$header = "";
$server_responded = false;
while ($status["eof"] == false) {
socket_set_timeout($this->socket, $this->socketReadTimeout);
// Read line from socket
$line_read = fgets($this->socket, 1024);
// Server responded
if ($server_responded == false) {
$server_responded = true;
$this->server_response_time = PHPCrawlerBenchmark::stop("server_response_time");
// Determinate socket prefill size
$status = socket_get_status($this->socket);
$this->socket_prefill_size = $status["unread_bytes"];
// Start data-transfer-time bechmark
PHPCrawlerBenchmark::reset("data_transfer_time");
PHPCrawlerBenchmark::start("data_transfer_time");
}
$source_read .= $line_read;
$this->global_traffic_count += strlen($line_read);
$status = socket_get_status($this->socket);
// Socket timed out
if ($status["timed_out"] == true) {
$error_code = PHPCrawlerRequestErrors::ERROR_SOCKET_TIMEOUT;
$error_string = "Socket-stream timed out (timeout set to " . $this->socketReadTimeout . " sec).";
return $header;
}
// No "HTTP" at beginnig of response
if (strtolower(substr($source_read, 0, 4)) != "http") {
$error_code = PHPCrawlerRequestErrors::ERROR_NO_HTTP_HEADER;
$error_string = "HTTP-protocol error.";
return $header;
}
// Header found and read (2 newlines) -> stop
if (substr($source_read, -4, 4) == "\r\n\r\n" || substr($source_read, -2, 2) == "\n\n") {
$header = substr($source_read, 0, strlen($source_read) - 2);
break;
}
}
// Stop data-transfer-time bechmark
PHPCrawlerBenchmark::stop("data_transfer_time");
// Header was found
if ($header != "") {
// Search for links (redirects) in the header
$this->LinkFinder->processHTTPHeader($header);
$this->header_bytes_received = strlen($header);
return $header;
}
// No header found
if ($header == "") {
$this->server_response_time = null;
$error_code = PHPCrawlerRequestErrors::ERROR_NO_HTTP_HEADER;
$error_string = "Host doesn't respond with a HTTP-header.";
return null;
}
}
/**
* Reads the response-content.
*
* @param bool $stream_to_file If TRUE, the content will be streamed diretly to the temporary file and
* this method will not return the content as a string.
* @param int &$error_code Error-code by reference if an error occured.
* @param &string &$error_string Error-string by reference
* @param &string &$document_received_completely Flag indicatign whether the content was received completely passed by reference
* @param &string &$bytes_received Number of bytes received, passed by reference
* @return string The response-content/source. May be emtpy if an error ocdured or data was streamed to the tmp-file.
*/
protected function readResponseContent($stream_to_file = false, &$error_code, &$error_string, &$document_received_completely, &$bytes_received)
{
PHPCrawlerBenchmark::start("retreiving_content");
PHPCrawlerBenchmark::start("data_transfer_time", true);
// If content should be streamed to file
if ($stream_to_file == true) {
$fp = @fopen($this->tmpFile, "w");
if ($fp == false) {
$error_code = PHPCrawlerRequestErrors::ERROR_TMP_FILE_NOT_WRITEABLE;
$error_string = "Couldn't open the temporary file " . $this->tmpFile . " for writing.";
return "";
}
}
// Init
$status = socket_get_status($this->socket);
$source_portion = "";
$source_complete = "";
$bytes_received = 0;
$document_received_completely = true;
$stop_receving = false;
while ($stop_receving == false) {
socket_set_timeout($this->socket, $this->socketReadTimeout);
// Read from socket
$line_read = @fread($this->socket, 1024);
// Das @ ist da um die blöde "SSL fatal protocol error"-Warnung zu unterdrücken,
// die keinen Sinn macht
// Check socket-status
$status = socket_get_status($this->socket);
// Check for EOF
if ($status["eof"] == true) {
$stop_receving = true;
}
// Socket timed out
if ($status["timed_out"] == true) {
$stop_receving = true;
$error_code = PHPCrawlerRequestErrors::ERROR_SOCKET_TIMEOUT;
$error_string = "Socket-stream timed out (timeout set to " . $this->socketReadTimeout . " sec).";
$document_received_completely = false;
} else {
$source_portion .= $line_read;
$bytes_received += strlen($line_read);
$this->global_traffic_count += strlen($line_read);
// Stream to file or store source in memory
if ($stream_to_file == true) {
@fwrite($fp, $line_read);
} else {
$source_complete .= $line_read;
}
}
// Check if content-length stated in the header is reached
if ($this->lastResponseHeader->content_length == $bytes_received) {
$stop_receving = true;
}
// Check if contentsize-limit is reached
if ($this->content_size_limit > 0 && $this->content_size_limit <= $bytes_received) {
$stop_receving = true;
}
// Find links in portion of the source
if (strlen($source_portion) >= 100000 || $stop_receving == true) {
if (PHPCrawlerUtils::checkStringAgainstRegexArray($this->lastResponseHeader->content_type, $this->linksearch_content_types)) {
PHPCrawlerBenchmark::stop("retreiving_content");
PHPCrawlerBenchmark::stop("data_transfer_time");
$this->LinkFinder->findLinksInHTMLChunk($source_portion);
$source_portion = substr($source_portion, -1500);
PHPCrawlerBenchmark::start("retreiving_content");
PHPCrawlerBenchmark::start("data_transfer_time", true);
}
}
}
if ($stream_to_file == true) {
@fclose($fp);
}
PHPCrawlerBenchmark::stop("retreiving_content");
PHPCrawlerBenchmark::stop("data_transfer_time");
$this->data_transfer_time = PHPCrawlerBenchmark::getElapsedTime("data_transfer_time");
PHPCrawlerBenchmark::reset("data_transfer_time");
return $source_complete;
}
