Esempio n. 1
0
 public function Process($url, $profile = "auto", $tempoptions = array())
 {
     $startts = microtime(true);
     $redirectts = $startts;
     if (isset($tempoptions["timeout"])) {
         $timeout = $tempoptions["timeout"];
     } else {
         if (isset($this->data["httpopts"]["timeout"])) {
             $timeout = $this->data["httpopts"]["timeout"];
         } else {
             $timeout = false;
         }
     }
     if (!isset($this->data["httpopts"]["headers"])) {
         $this->data["httpopts"]["headers"] = array();
     }
     $this->data["httpopts"]["headers"] = HTTPNormalizeHeaders($this->data["httpopts"]["headers"]);
     unset($this->data["httpopts"]["method"]);
     unset($this->data["httpopts"]["write_body_callback"]);
     unset($this->data["httpopts"]["body"]);
     unset($this->data["httpopts"]["postvars"]);
     unset($this->data["httpopts"]["files"]);
     $httpopts = $this->data["httpopts"];
     $numfollow = $this->data["maxfollow"];
     $numredirects = 0;
     $totalrawsendsize = 0;
     if (!isset($tempoptions["headers"])) {
         $tempoptions["headers"] = array();
     }
     $tempoptions["headers"] = HTTPNormalizeHeaders($tempoptions["headers"]);
     if (isset($tempoptions["headers"]["Referer"])) {
         $this->data["referer"] = $tempoptions["headers"]["Referer"];
     }
     // If a referrer is specified, use it to generate an absolute URL.
     if ($this->data["referer"] != "") {
         $url = ConvertRelativeToAbsoluteURL($this->data["referer"], $url);
     }
     $urlinfo = ExtractURL($url);
     do {
         if (!isset($this->data["allowedprotocols"][$urlinfo["scheme"]]) || !$this->data["allowedprotocols"][$urlinfo["scheme"]]) {
             return array("success" => false, "error" => HTTPTranslate("Protocol '%s' is not allowed in '%s'.", $urlinfo["scheme"], $url), "errorcode" => "allowed_protocols");
         }
         $filename = HTTPExtractFilename($urlinfo["path"]);
         $pos = strrpos($filename, ".");
         $fileext = $pos !== false ? strtolower(substr($filename, $pos + 1)) : "";
         // Set up some standard headers.
         $headers = array();
         $profile = strtolower($profile);
         $tempprofile = explode("-", $profile);
         if (count($tempprofile) == 2) {
             $profile = $tempprofile[0];
             $fileext = $tempprofile[1];
         }
         if (substr($profile, 0, 2) == "ie" || $profile == "auto" && substr($this->data["useragent"], 0, 2) == "ie") {
             if ($fileext == "css") {
                 $headers["Accept"] = "text/css";
             } else {
                 if ($fileext == "png" || $fileext == "jpg" || $fileext == "jpeg" || $fileext == "gif" || $fileext == "svg") {
                     $headers["Accept"] = "image/png, image/svg+xml, image/*;q=0.8, */*;q=0.5";
                 } else {
                     if ($fileext == "js") {
                         $headers["Accept"] = "application/javascript, */*;q=0.8";
                     } else {
                         if ($this->data["referer"] != "" || $fileext == "" || $fileext == "html" || $fileext == "xhtml" || $fileext == "xml") {
                             $headers["Accept"] = "text/html, application/xhtml+xml, */*";
                         } else {
                             $headers["Accept"] = "*/*";
                         }
                     }
                 }
             }
             $headers["Accept-Language"] = "en-US";
             $headers["User-Agent"] = GetWebUserAgent(substr($profile, 0, 2) == "ie" ? $profile : $this->data["useragent"]);
         } else {
             if ($profile == "firefox" || $profile == "auto" && $this->data["useragent"] == "firefox") {
                 if ($fileext == "css") {
                     $headers["Accept"] = "text/css,*/*;q=0.1";
                 } else {
                     if ($fileext == "png" || $fileext == "jpg" || $fileext == "jpeg" || $fileext == "gif" || $fileext == "svg") {
                         $headers["Accept"] = "image/png,image/*;q=0.8,*/*;q=0.5";
                     } else {
                         if ($fileext == "js") {
                             $headers["Accept"] = "*/*";
                         } else {
                             $headers["Accept"] = "text/html, application/xhtml+xml, */*";
                         }
                     }
                 }
                 $headers["Accept-Language"] = "en-us,en;q=0.5";
                 $headers["Cache-Control"] = "max-age=0";
                 $headers["User-Agent"] = GetWebUserAgent("firefox");
             } else {
                 if ($profile == "opera" || $profile == "auto" && $this->data["useragent"] == "opera") {
                     // Opera has the right idea:  Just send the same thing regardless of the request type.
                     $headers["Accept"] = "text/html, application/xml;q=0.9, application/xhtml+xml, image/png, image/webp, image/jpeg, image/gif, image/x-xbitmap, */*;q=0.1";
                     $headers["Accept-Language"] = "en-US,en;q=0.9";
                     $headers["Cache-Control"] = "no-cache";
                     $headers["User-Agent"] = GetWebUserAgent("opera");
                 } else {
                     if ($profile == "safari" || $profile == "chrome" || $profile == "auto" && ($this->data["useragent"] == "safari" || $this->data["useragent"] == "chrome")) {
                         if ($fileext == "css") {
                             $headers["Accept"] = "text/css,*/*;q=0.1";
                         } else {
                             if ($fileext == "png" || $fileext == "jpg" || $fileext == "jpeg" || $fileext == "gif" || $fileext == "svg" || $fileext == "js") {
                                 $headers["Accept"] = "*/*";
                             } else {
                                 $headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
                             }
                         }
                         $headers["Accept-Charset"] = "ISO-8859-1,utf-8;q=0.7,*;q=0.3";
                         $headers["Accept-Language"] = "en-US,en;q=0.8";
                         $headers["User-Agent"] = GetWebUserAgent($profile == "safari" || $profile == "chrome" ? $profile : $this->data["useragent"]);
                     }
                 }
             }
         }
         if ($this->data["referer"] != "") {
             $headers["Referer"] = $this->data["referer"];
         }
         // Generate the final headers array.
         $headers = array_merge($headers, $httpopts["headers"], $tempoptions["headers"]);
         // Calculate the host and reverse host and remove port information.
         $host = isset($headers["Host"]) ? $headers["Host"] : $urlinfo["host"];
         $pos = strpos($host, "]");
         if (substr($host, 0, 1) == "[" && $pos !== false) {
             $host = substr($host, 0, $pos + 1);
         } else {
             $pos = strpos($host, ":");
             if ($pos !== false) {
                 $host = substr($host, 0, $pos);
             }
         }
         $dothost = $host;
         if (substr($dothost, 0, 1) != ".") {
             $dothost = "." . $dothost;
         }
         // Append cookies and delete old, invalid cookies.
         $secure = $urlinfo["scheme"] == "https";
         $cookiepath = $urlinfo["path"];
         if ($cookiepath == "") {
             $cookiepath = "/";
         }
         $pos = strrpos($cookiepath, "/");
         if ($pos !== false) {
             $cookiepath = substr($cookiepath, 0, $pos + 1);
         }
         $cookies = array();
         foreach ($this->data["cookies"] as $domain => $paths) {
             if (substr($domain, -strlen($dothost)) == $dothost) {
                 foreach ($paths as $path => $cookies2) {
                     if (substr($cookiepath, 0, strlen($path)) == $path) {
                         foreach ($cookies2 as $num => $info) {
                             if (isset($info["expires_ts"]) && $this->GetExpiresTimestamp($info["expires_ts"]) < time()) {
                                 unset($this->data["cookies"][$domain][$path][$num]);
                             } else {
                                 if ($secure || !isset($info["secure"])) {
                                     $cookies[$info["name"]] = $info["value"];
                                 }
                             }
                         }
                         if (!count($this->data["cookies"][$domain][$path])) {
                             unset($this->data["cookies"][$domain][$path]);
                         }
                     }
                 }
                 if (!count($this->data["cookies"][$domain])) {
                     unset($this->data["cookies"][$domain]);
                 }
             }
         }
         $cookies2 = array();
         foreach ($cookies as $name => $value) {
             $cookies2[] = rawurlencode($name) . "=" . rawurlencode($value);
         }
         $headers["Cookie"] = implode("; ", $cookies2);
         if ($headers["Cookie"] == "") {
             unset($headers["Cookie"]);
         }
         // Generate the final options array.
         $options = array_merge($httpopts, $tempoptions);
         $options["headers"] = $headers;
         if ($timeout !== false) {
             $options["timeout"] = HTTPGetTimeLeft($startts, $timeout);
         }
         // Process the request.
         $result = RetrieveWebpage($url, $options);
         $result["url"] = $url;
         $result["options"] = $options;
         $result["firstreqts"] = $startts;
         $result["numredirects"] = $numredirects;
         $result["redirectts"] = $redirectts;
         if (isset($result["rawsendsize"])) {
             $totalrawsendsize += $result["rawsendsize"];
         }
         $result["totalrawsendsize"] = $totalrawsendsize;
         unset($result["options"]["files"]);
         unset($result["options"]["body"]);
         if (!$result["success"]) {
             return array("success" => false, "error" => HTTPTranslate("Unable to retrieve content.  %s", $result["error"]), "info" => $result, "errorcode" => "retrievewebpage");
         }
         // Set up structures for another round.
         if ($this->data["autoreferer"]) {
             $this->data["referer"] = $url;
         }
         if (isset($result["headers"]["Location"]) && $this->data["followlocation"]) {
             $redirectts = microtime(true);
             unset($tempoptions["method"]);
             unset($tempoptions["write_body_callback"]);
             unset($tempoptions["body"]);
             unset($tempoptions["postvars"]);
             unset($tempoptions["files"]);
             $tempoptions["headers"]["Referer"] = $url;
             $url = $result["headers"]["Location"][0];
             // Generate an absolute URL.
             if ($this->data["referer"] != "") {
                 $url = ConvertRelativeToAbsoluteURL($this->data["referer"], $url);
             }
             $urlinfo2 = ExtractURL($url);
             if (!isset($this->data["allowedredirprotocols"][$urlinfo2["scheme"]]) || !$this->data["allowedredirprotocols"][$urlinfo2["scheme"]]) {
                 return array("success" => false, "error" => HTTPTranslate("Protocol '%s' is not allowed.  Server attempted to redirect to '%s'.", $urlinfo2["scheme"], $url), "info" => $result, "errorcode" => "allowed_redir_protocols");
             }
             if ($urlinfo2["host"] != $urlinfo["host"]) {
                 unset($tempoptions["headers"]["Host"]);
                 unset($httpopts["headers"]["Host"]);
             }
             $urlinfo = $urlinfo2;
             $numredirects++;
         }
         // Handle any 'Set-Cookie' headers.
         if (isset($result["headers"]["Set-Cookie"])) {
             foreach ($result["headers"]["Set-Cookie"] as $cookie) {
                 $items = explode("; ", $cookie);
                 $item = trim(array_shift($items));
                 if ($item != "") {
                     $cookie2 = array();
                     $pos = strpos($item, "=");
                     if ($pos === false) {
                         $cookie2["name"] = urldecode($item);
                         $cookie2["value"] = "";
                     } else {
                         $cookie2["name"] = urldecode(substr($item, 0, $pos));
                         $cookie2["value"] = urldecode(substr($item, $pos + 1));
                     }
                     $cookie = array();
                     foreach ($items as $item) {
                         $item = trim($item);
                         if ($item != "") {
                             $pos = strpos($item, "=");
                             if ($pos === false) {
                                 $cookie[strtolower(trim(urldecode($item)))] = "";
                             } else {
                                 $cookie[strtolower(trim(urldecode(substr($item, 0, $pos))))] = urldecode(substr($item, $pos + 1));
                             }
                         }
                     }
                     $cookie = array_merge($cookie, $cookie2);
                     if (isset($cookie["expires"])) {
                         $ts = GetHTTPDateTimestamp($cookie["expires"]);
                         $cookie["expires_ts"] = gmdate("Y-m-d H:i:s", $ts === false ? time() - 24 * 60 * 60 : $ts);
                     } else {
                         if (isset($cookie["max-age"])) {
                             $cookie["expires_ts"] = gmdate("Y-m-d H:i:s", time() + (int) $cookie["max-age"]);
                         } else {
                             unset($cookie["expires_ts"]);
                         }
                     }
                     if (!isset($cookie["domain"])) {
                         $cookie["domain"] = $dothost;
                     }
                     if (substr($cookie["domain"], 0, 1) != ".") {
                         $cookie["domain"] = "." . $cookie["domain"];
                     }
                     if (!isset($cookie["path"])) {
                         $cookie["path"] = $cookiepath;
                     }
                     $cookie["path"] = str_replace("\\", "/", $cookie["path"]);
                     if (substr($cookie["path"], -1) != "/") {
                         $cookie["path"] = "/";
                     }
                     if (!isset($this->data["cookies"][$cookie["domain"]])) {
                         $this->data["cookies"][$cookie["domain"]] = array();
                     }
                     if (!isset($this->data["cookies"][$cookie["domain"]][$cookie["path"]])) {
                         $this->data["cookies"][$cookie["domain"]][$cookie["path"]] = array();
                     }
                     $this->data["cookies"][$cookie["domain"]][$cookie["path"]][] = $cookie;
                 }
             }
         }
         if ($numfollow > 0) {
             $numfollow--;
         }
     } while (isset($result["headers"]["Location"]) && $this->data["followlocation"] && $numfollow);
     $result["numredirects"] = $numredirects;
     $result["redirectts"] = $redirectts;
     // Extract the forms from the page in a parsed format.
     // Call WebBrowser::GenerateFormRequest() to prepare an actual request for Process().
     if ($this->data["extractforms"]) {
         $result["forms"] = $this->ExtractForms($result["url"], $result["body"]);
     }
     return $result;
 }
 function curl_getinfo($ch, $opt = 0)
 {
     global $curl_init__map;
     $key = get_check_curl_init_key($ch);
     if (!isset($curl_init__map[$key]["lastresult"])) {
         return false;
     }
     $result = array("url" => $curl_init__map[$key]["lastresult"]["url"], "content_type" => isset($curl_init__map[$key]["lastresult"]["headers"]) && isset($curl_init__map[$key]["lastresult"]["headers"]["Content-Type"]) ? $curl_init__map[$key]["lastresult"]["headers"]["Content-Type"][0] : null, "http_code" => isset($curl_init__map[$key]["lastresult"]["response"]) && isset($curl_init__map[$key]["lastresult"]["response"]["code"]) ? (int) $curl_init__map[$key]["lastresult"]["response"]["code"] : null, "header_size" => isset($curl_init__map[$key]["lastresult"]["rawrecvheadersize"]) ? $curl_init__map[$key]["lastresult"]["rawrecvheadersize"] : 0, "request_size" => isset($curl_init__map[$key]["lastresult"]["totalrawsendsize"]) ? $curl_init__map[$key]["lastresult"]["totalrawsendsize"] : 0, "filetime" => isset($curl_init__map[$key]["options"][CURLOPT_FILETIME]) && $curl_init__map[$key]["options"][CURLOPT_FILETIME] && isset($curl_init__map[$key]["lastresult"]["headers"]) && isset($curl_init__map[$key]["lastresult"]["headers"]["Last-Modified"]) ? GetHTTPDateTimestamp($curl_init__map[$key]["lastresult"]["headers"]["Last-Modified"][0]) : -1, "ssl_verify_result" => 0, "redirect_count" => isset($curl_init__map[$key]["lastresult"]["numredirects"]) ? $curl_init__map[$key]["lastresult"]["numredirects"] : 0, "total_time" => isset($curl_init__map[$key]["lastresult"]["startts"]) && isset($curl_init__map[$key]["lastresult"]["endts"]) ? $curl_init__map[$key]["lastresult"]["endts"] - $curl_init__map[$key]["lastresult"]["startts"] : 0, "namelookup_time" => isset($curl_init__map[$key]["lastresult"]["startts"]) && isset($curl_init__map[$key]["lastresult"]["connected"]) ? ($curl_init__map[$key]["lastresult"]["connected"] - $curl_init__map[$key]["lastresult"]["startts"]) / 2 : 0, "connect_time" => isset($curl_init__map[$key]["lastresult"]["startts"]) && isset($curl_init__map[$key]["lastresult"]["connected"]) ? ($curl_init__map[$key]["lastresult"]["connected"] - $curl_init__map[$key]["lastresult"]["startts"]) / 2 : 0, "pretransfer_time" => isset($curl_init__map[$key]["lastresult"]["connected"]) && isset($curl_init__map[$key]["lastresult"]["sendstart"]) ? $curl_init__map[$key]["lastresult"]["sendstart"] - $curl_init__map[$key]["lastresult"]["connected"] : 0, "size_upload" => isset($curl_init__map[$key]["lastresult"]["rawsendsize"]) && isset($curl_init__map[$key]["lastresult"]["rawsendheadersize"]) ? $curl_init__map[$key]["lastresult"]["rawsendsize"] - $curl_init__map[$key]["lastresult"]["rawsendheadersize"] : 0, "size_download" => isset($curl_init__map[$key]["lastresult"]["rawrecvsize"]) && isset($curl_init__map[$key]["lastresult"]["rawrecvheadersize"]) ? $curl_init__map[$key]["lastresult"]["rawrecvsize"] - $curl_init__map[$key]["lastresult"]["rawrecvheadersize"] : 0);
     $result["speed_download"] = isset($curl_init__map[$key]["lastresult"]["recvstart"]) && isset($curl_init__map[$key]["lastresult"]["endts"]) && $curl_init__map[$key]["lastresult"]["endts"] - $curl_init__map[$key]["lastresult"]["recvstart"] > 0 ? $result["size_download"] / ($curl_init__map[$key]["lastresult"]["endts"] - $curl_init__map[$key]["lastresult"]["recvstart"]) : 0;
     $result["speed_upload"] = isset($curl_init__map[$key]["lastresult"]["sendstart"]) && isset($curl_init__map[$key]["lastresult"]["recvstart"]) && $curl_init__map[$key]["lastresult"]["recvstart"] - $curl_init__map[$key]["lastresult"]["sendstart"] > 0 ? $result["size_upload"] / ($curl_init__map[$key]["lastresult"]["recvstart"] - $curl_init__map[$key]["lastresult"]["sendstart"]) : 0;
     $result["download_content_length"] = isset($curl_init__map[$key]["lastresult"]["headers"]) && isset($curl_init__map[$key]["lastresult"]["headers"]["Content-Length"]) ? $curl_init__map[$key]["lastresult"]["headers"]["Content-Length"][0] : -1;
     $result["upload_content_length"] = $result["size_upload"];
     $result["starttransfer_time"] = isset($curl_init__map[$key]["lastresult"]["startts"]) && isset($curl_init__map[$key]["lastresult"]["sendstart"]) ? $curl_init__map[$key]["lastresult"]["sendstart"] - $curl_init__map[$key]["lastresult"]["startts"] : 0;
     $result["redirect_time"] = isset($curl_init__map[$key]["lastresult"]["firstreqts"]) && isset($curl_init__map[$key]["lastresult"]["redirectts"]) ? $curl_init__map[$key]["lastresult"]["redirectts"] - $curl_init__map[$key]["lastresult"]["firstreqts"] : 0;
     if (isset($curl_init__map[$key]["rawheaders"])) {
         $result["request_header"] = $curl_init__map[$key]["rawheaders"];
     }
     if ($opt == 0) {
         return $result;
     }
     $tempmap = array(CURLINFO_EFFECTIVE_URL => "url", CURLINFO_HTTP_CODE => "http_code", CURLINFO_FILETIME => "filetime", CURLINFO_TOTAL_TIME => "total_time", CURLINFO_NAMELOOKUP_TIME => "namelookup_time", CURLINFO_CONNECT_TIME => "connect_time", CURLINFO_PRETRANSFER_TIME => "pretransfer_time", CURLINFO_STARTTRANSFER_TIME => "starttransfer_time", CURLINFO_REDIRECT_TIME => "redirect_time", CURLINFO_SIZE_UPLOAD => "size_upload", CURLINFO_SIZE_DOWNLOAD => "size_download", CURLINFO_SPEED_DOWNLOAD => "speed_download", CURLINFO_SPEED_UPLOAD => "speed_upload", CURLINFO_HEADER_SIZE => "header_size", CURLINFO_HEADER_OUT => "request_header", CURLINFO_REQUEST_SIZE => "request_size", CURLINFO_SSL_VERIFYRESULT => "ssl_verify_result", CURLINFO_CONTENT_LENGTH_DOWNLOAD => "download_content_length", CURLINFO_CONTENT_LENGTH_UPLOAD => "upload_content_length", CURLINFO_CONTENT_TYPE => "content_type");
     if (!isset($tempmap[$opt]) || !isset($result[$tempmap[$opt]])) {
         return false;
     }
     return $result[$tempmap[$opt]];
 }