/** * Fetches content of file from Internet (using proxy if defined). Uses cURL extension if present. * Due to security concerns only downloads from http(s) sources are supported. * * @todo MDL-31073 add version test for '7.10.5' * @category files * @param string $url file url starting with http(s):// * @param array $headers http headers, null if none. If set, should be an * associative array of header name => value pairs. * @param array $postdata array means use POST request with given parameters * @param bool $fullresponse return headers, responses, etc in a similar way snoopy does * (if false, just returns content) * @param int $timeout timeout for complete download process including all file transfer * (default 5 minutes) * @param int $connecttimeout timeout for connection to server; this is the timeout that * usually happens if the remote server is completely down (default 20 seconds); * may not work when using proxy * @param bool $skipcertverify If true, the peer's SSL certificate will not be checked. * Only use this when already in a trusted location. * @param string $tofile store the downloaded content to file instead of returning it. * @param bool $calctimeout false by default, true enables an extra head request to try and determine * filesize and appropriately larger timeout based on $CFG->curltimeoutkbitrate * @return mixed false if request failed or content of the file as string if ok. True if file downloaded into $tofile successfully. */ function download_file_content($url, $headers = null, $postdata = null, $fullresponse = false, $timeout = 300, $connecttimeout = 20, $skipcertverify = false, $tofile = NULL, $calctimeout = false) { global $CFG; // some extra security $newlines = array("\r", "\n"); if (is_array($headers)) { foreach ($headers as $key => $value) { $headers[$key] = str_replace($newlines, '', $value); } } $url = str_replace($newlines, '', $url); if (!preg_match('|^https?://|i', $url)) { if ($fullresponse) { $response = new stdClass(); $response->status = 0; $response->headers = array(); $response->response_code = 'Invalid protocol specified in url'; $response->results = ''; $response->error = 'Invalid protocol specified in url'; return $response; } else { return false; } } // check if proxy (if used) should be bypassed for this url $proxybypass = is_proxybypass($url); if (!($ch = curl_init($url))) { debugging('Can not init curl.'); return false; } // set extra headers if (is_array($headers)) { $headers2 = array(); foreach ($headers as $key => $value) { $headers2[] = "{$key}: {$value}"; } curl_setopt($ch, CURLOPT_HTTPHEADER, $headers2); } if ($skipcertverify) { curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); } // use POST if requested if (is_array($postdata)) { $postdata = format_postdata_for_curlcall($postdata); curl_setopt($ch, CURLOPT_POST, true); curl_setopt($ch, CURLOPT_POSTFIELDS, $postdata); } curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); curl_setopt($ch, CURLOPT_HEADER, false); curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, $connecttimeout); if ($cacert = curl::get_cacert()) { curl_setopt($ch, CURLOPT_CAINFO, $cacert); } if (!ini_get('open_basedir') and !ini_get('safe_mode')) { // TODO: add version test for '7.10.5' curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); curl_setopt($ch, CURLOPT_MAXREDIRS, 5); } if (!empty($CFG->proxyhost) and !$proxybypass) { // SOCKS supported in PHP5 only if (!empty($CFG->proxytype) and $CFG->proxytype == 'SOCKS5') { if (defined('CURLPROXY_SOCKS5')) { curl_setopt($ch, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5); } else { curl_close($ch); if ($fullresponse) { $response = new stdClass(); $response->status = '0'; $response->headers = array(); $response->response_code = 'SOCKS5 proxy is not supported in PHP4'; $response->results = ''; $response->error = 'SOCKS5 proxy is not supported in PHP4'; return $response; } else { debugging("SOCKS5 proxy is not supported in PHP4.", DEBUG_ALL); return false; } } } curl_setopt($ch, CURLOPT_HTTPPROXYTUNNEL, false); if (empty($CFG->proxyport)) { curl_setopt($ch, CURLOPT_PROXY, $CFG->proxyhost); } else { curl_setopt($ch, CURLOPT_PROXY, $CFG->proxyhost . ':' . $CFG->proxyport); } if (!empty($CFG->proxyuser) and !empty($CFG->proxypassword)) { curl_setopt($ch, CURLOPT_PROXYUSERPWD, $CFG->proxyuser . ':' . $CFG->proxypassword); if (defined('CURLOPT_PROXYAUTH')) { // any proxy authentication if PHP 5.1 curl_setopt($ch, CURLOPT_PROXYAUTH, CURLAUTH_BASIC | CURLAUTH_NTLM); } } } // set up header and content handlers $received = new stdClass(); $received->headers = array(); // received headers array $received->tofile = $tofile; $received->fh = null; curl_setopt($ch, CURLOPT_HEADERFUNCTION, partial('download_file_content_header_handler', $received)); if ($tofile) { curl_setopt($ch, CURLOPT_WRITEFUNCTION, partial('download_file_content_write_handler', $received)); } if (!isset($CFG->curltimeoutkbitrate)) { //use very slow rate of 56kbps as a timeout speed when not set $bitrate = 56; } else { $bitrate = $CFG->curltimeoutkbitrate; } // try to calculate the proper amount for timeout from remote file size. // if disabled or zero, we won't do any checks nor head requests. if ($calctimeout && $bitrate > 0) { //setup header request only options curl_setopt_array($ch, array(CURLOPT_RETURNTRANSFER => false, CURLOPT_NOBODY => true)); curl_exec($ch); $info = curl_getinfo($ch); $err = curl_error($ch); if ($err === '' && $info['download_content_length'] > 0) { //no curl errors $timeout = max($timeout, ceil($info['download_content_length'] * 8 / ($bitrate * 1024))); //adjust for large files only - take max timeout. } //reinstate affected curl options curl_setopt_array($ch, array(CURLOPT_RETURNTRANSFER => true, CURLOPT_NOBODY => false, CURLOPT_HTTPGET => true)); } curl_setopt($ch, CURLOPT_TIMEOUT, $timeout); $result = curl_exec($ch); // try to detect encoding problems if ((curl_errno($ch) == 23 or curl_errno($ch) == 61) and defined('CURLOPT_ENCODING')) { curl_setopt($ch, CURLOPT_ENCODING, 'none'); $result = curl_exec($ch); } if ($received->fh) { fclose($received->fh); } if (curl_errno($ch)) { $error = curl_error($ch); $error_no = curl_errno($ch); curl_close($ch); if ($fullresponse) { $response = new stdClass(); if ($error_no == 28) { $response->status = '-100'; // mimic snoopy } else { $response->status = '0'; } $response->headers = array(); $response->response_code = $error; $response->results = false; $response->error = $error; return $response; } else { debugging("cURL request for \"{$url}\" failed with: {$error} ({$error_no})", DEBUG_ALL); return false; } } else { $info = curl_getinfo($ch); curl_close($ch); if (empty($info['http_code'])) { // for security reasons we support only true http connections (Location: file:// exploit prevention) $response = new stdClass(); $response->status = '0'; $response->headers = array(); $response->response_code = 'Unknown cURL error'; $response->results = false; // do NOT change this, we really want to ignore the result! $response->error = 'Unknown cURL error'; } else { $response = new stdClass(); $response->status = (string) $info['http_code']; $response->headers = $received->headers; $response->response_code = $received->headers[0]; $response->results = $result; $response->error = ''; } if ($fullresponse) { return $response; } else { if ($info['http_code'] != 200) { debugging("cURL request for \"{$url}\" failed, HTTP response code: " . $response->response_code, DEBUG_ALL); return false; } else { return $response->results; } } } }