コード例 #1
0
 /**
  * Reads the response-header.
  *
  * @param  int    &$error_code           Error-code by reference if an error occured.
  * @param  string &$error_string         Error-string by reference
  *
  * @return string The response-header or NULL if an error occured
  */
 protected function readResponseHeader(&$error_code, &$error_string)
 {
     PHPCrawlerBenchmark::reset("server_response_time");
     PHPCrawlerBenchmark::start("server_response_time");
     $status = socket_get_status($this->socket);
     $source_read = "";
     $header = "";
     $server_responded = false;
     while ($status["eof"] == false) {
         socket_set_timeout($this->socket, $this->socketReadTimeout);
         // Read line from socket
         $line_read = fgets($this->socket, 1024);
         // Server responded
         if ($server_responded == false) {
             $server_responded = true;
             $this->server_response_time = PHPCrawlerBenchmark::stop("server_response_time");
             // Determinate socket prefill size
             $status = socket_get_status($this->socket);
             $this->socket_prefill_size = $status["unread_bytes"];
             // Start data-transfer-time bechmark
             PHPCrawlerBenchmark::reset("data_transfer_time");
             PHPCrawlerBenchmark::start("data_transfer_time");
         }
         $source_read .= $line_read;
         $this->global_traffic_count += strlen($line_read);
         $status = socket_get_status($this->socket);
         // Socket timed out
         if ($status["timed_out"] == true) {
             $error_code = PHPCrawlerRequestErrors::ERROR_SOCKET_TIMEOUT;
             $error_string = "Socket-stream timed out (timeout set to " . $this->socketReadTimeout . " sec).";
             return $header;
         }
         // No "HTTP" at beginnig of response
         if (strtolower(substr($source_read, 0, 4)) != "http") {
             $error_code = PHPCrawlerRequestErrors::ERROR_NO_HTTP_HEADER;
             $error_string = "HTTP-protocol error.";
             return $header;
         }
         // Header found and read (2 newlines) -> stop
         if (substr($source_read, -4, 4) == "\r\n\r\n" || substr($source_read, -2, 2) == "\n\n") {
             $header = substr($source_read, 0, strlen($source_read) - 2);
             break;
         }
     }
     // Stop data-transfer-time bechmark
     PHPCrawlerBenchmark::stop("data_transfer_time");
     // Header was found
     if ($header != "") {
         // Search for links (redirects) in the header
         $this->LinkFinder->processHTTPHeader($header);
         $this->header_bytes_received = strlen($header);
         return $header;
     }
     // No header found
     if ($header == "") {
         $this->server_response_time = null;
         $error_code = PHPCrawlerRequestErrors::ERROR_NO_HTTP_HEADER;
         $error_string = "Host doesn't respond with a HTTP-header.";
         return null;
     }
 }
コード例 #2
0
 /**
  * Reads the response-content.
  * 
  * @param bool    $stream_to_file If TRUE, the content will be streamed diretly to the temporary file and
  *                                this method will not return the content as a string.                            
  * @param int     &$error_code    Error-code by reference if an error occured.
  * @param &string &$error_string  Error-string by reference
  * @param &string &$document_received_completely Flag indicatign whether the content was received completely passed by reference
  * @param &string &$bytes_received Number of bytes received, passed by reference
  * @return string  The response-content/source. May be emtpy if an error ocdured or data was streamed to the tmp-file.
  */
 protected function readResponseContent($stream_to_file = false, &$error_code, &$error_string, &$document_received_completely, &$bytes_received)
 {
     PHPCrawlerBenchmark::start("retreiving_content");
     PHPCrawlerBenchmark::start("data_transfer_time", true);
     // If content should be streamed to file
     if ($stream_to_file == true) {
         $fp = @fopen($this->tmpFile, "w");
         if ($fp == false) {
             $error_code = PHPCrawlerRequestErrors::ERROR_TMP_FILE_NOT_WRITEABLE;
             $error_string = "Couldn't open the temporary file " . $this->tmpFile . " for writing.";
             return "";
         }
     }
     // Init
     $status = socket_get_status($this->socket);
     $source_portion = "";
     $source_complete = "";
     $bytes_received = 0;
     $document_received_completely = true;
     $stop_receving = false;
     while ($stop_receving == false) {
         socket_set_timeout($this->socket, $this->socketReadTimeout);
         // Read from socket
         $line_read = @fread($this->socket, 1024);
         // Das @ ist da um die blöde "SSL fatal protocol error"-Warnung zu unterdrücken,
         // die keinen Sinn macht
         // Check socket-status
         $status = socket_get_status($this->socket);
         // Check for EOF
         if ($status["eof"] == true) {
             $stop_receving = true;
         }
         // Socket timed out
         if ($status["timed_out"] == true) {
             $stop_receving = true;
             $error_code = PHPCrawlerRequestErrors::ERROR_SOCKET_TIMEOUT;
             $error_string = "Socket-stream timed out (timeout set to " . $this->socketReadTimeout . " sec).";
             $document_received_completely = false;
         } else {
             $source_portion .= $line_read;
             $bytes_received += strlen($line_read);
             $this->global_traffic_count += strlen($line_read);
             // Stream to file or store source in memory
             if ($stream_to_file == true) {
                 @fwrite($fp, $line_read);
             } else {
                 $source_complete .= $line_read;
             }
         }
         // Check if content-length stated in the header is reached
         if ($this->lastResponseHeader->content_length == $bytes_received) {
             $stop_receving = true;
         }
         // Check if contentsize-limit is reached
         if ($this->content_size_limit > 0 && $this->content_size_limit <= $bytes_received) {
             $stop_receving = true;
         }
         // Find links in portion of the source
         if (strlen($source_portion) >= 100000 || $stop_receving == true) {
             if (PHPCrawlerUtils::checkStringAgainstRegexArray($this->lastResponseHeader->content_type, $this->linksearch_content_types)) {
                 PHPCrawlerBenchmark::stop("retreiving_content");
                 PHPCrawlerBenchmark::stop("data_transfer_time");
                 $this->LinkFinder->findLinksInHTMLChunk($source_portion);
                 $source_portion = substr($source_portion, -1500);
                 PHPCrawlerBenchmark::start("retreiving_content");
                 PHPCrawlerBenchmark::start("data_transfer_time", true);
             }
         }
     }
     if ($stream_to_file == true) {
         @fclose($fp);
     }
     PHPCrawlerBenchmark::stop("retreiving_content");
     PHPCrawlerBenchmark::stop("data_transfer_time");
     $this->data_transfer_time = PHPCrawlerBenchmark::getElapsedTime("data_transfer_time");
     PHPCrawlerBenchmark::reset("data_transfer_time");
     return $source_complete;
 }