/**
* Fetches content of file from Internet (using proxy if defined). Uses cURL extension if present.
* Due to security concerns only downloads from http(s) sources are supported.
*
* @todo MDL-31073 add version test for '7.10.5'
* @category files
* @param string $url file url starting with http(s)://
* @param array $headers http headers, null if none. If set, should be an
* associative array of header name => value pairs.
* @param array $postdata array means use POST request with given parameters
* @param bool $fullresponse return headers, responses, etc in a similar way snoopy does
* (if false, just returns content)
* @param int $timeout timeout for complete download process including all file transfer
* (default 5 minutes)
* @param int $connecttimeout timeout for connection to server; this is the timeout that
* usually happens if the remote server is completely down (default 20 seconds);
* may not work when using proxy
* @param bool $skipcertverify If true, the peer's SSL certificate will not be checked.
* Only use this when already in a trusted location.
* @param string $tofile store the downloaded content to file instead of returning it.
* @param bool $calctimeout false by default, true enables an extra head request to try and determine
* filesize and appropriately larger timeout based on $CFG->curltimeoutkbitrate
* @return mixed false if request failed or content of the file as string if ok. True if file downloaded into $tofile successfully.
*/
function download_file_content($url, $headers = null, $postdata = null, $fullresponse = false, $timeout = 300, $connecttimeout = 20, $skipcertverify = false, $tofile = NULL, $calctimeout = false)
{
global $CFG;
// some extra security
$newlines = array("\r", "\n");
if (is_array($headers)) {
foreach ($headers as $key => $value) {
$headers[$key] = str_replace($newlines, '', $value);
}
}
$url = str_replace($newlines, '', $url);
if (!preg_match('|^https?://|i', $url)) {
if ($fullresponse) {
$response = new stdClass();
$response->status = 0;
$response->headers = array();
$response->response_code = 'Invalid protocol specified in url';
$response->results = '';
$response->error = 'Invalid protocol specified in url';
return $response;
} else {
return false;
}
}
// check if proxy (if used) should be bypassed for this url
$proxybypass = is_proxybypass($url);
if (!($ch = curl_init($url))) {
debugging('Can not init curl.');
return false;
}
// set extra headers
if (is_array($headers)) {
$headers2 = array();
foreach ($headers as $key => $value) {
$headers2[] = "{$key}: {$value}";
}
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers2);
}
if ($skipcertverify) {
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
}
// use POST if requested
if (is_array($postdata)) {
$postdata = format_postdata_for_curlcall($postdata);
curl_setopt($ch, CURLOPT_POST, true);
curl_setopt($ch, CURLOPT_POSTFIELDS, $postdata);
}
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_HEADER, false);
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, $connecttimeout);
if ($cacert = curl::get_cacert()) {
curl_setopt($ch, CURLOPT_CAINFO, $cacert);
}
if (!ini_get('open_basedir') and !ini_get('safe_mode')) {
// TODO: add version test for '7.10.5'
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($ch, CURLOPT_MAXREDIRS, 5);
}
if (!empty($CFG->proxyhost) and !$proxybypass) {
// SOCKS supported in PHP5 only
if (!empty($CFG->proxytype) and $CFG->proxytype == 'SOCKS5') {
if (defined('CURLPROXY_SOCKS5')) {
curl_setopt($ch, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5);
} else {
curl_close($ch);
if ($fullresponse) {
$response = new stdClass();
$response->status = '0';
$response->headers = array();
$response->response_code = 'SOCKS5 proxy is not supported in PHP4';
$response->results = '';
$response->error = 'SOCKS5 proxy is not supported in PHP4';
return $response;
} else {
debugging("SOCKS5 proxy is not supported in PHP4.", DEBUG_ALL);
return false;
}
}
}
curl_setopt($ch, CURLOPT_HTTPPROXYTUNNEL, false);
if (empty($CFG->proxyport)) {
curl_setopt($ch, CURLOPT_PROXY, $CFG->proxyhost);
} else {
curl_setopt($ch, CURLOPT_PROXY, $CFG->proxyhost . ':' . $CFG->proxyport);
}
if (!empty($CFG->proxyuser) and !empty($CFG->proxypassword)) {
curl_setopt($ch, CURLOPT_PROXYUSERPWD, $CFG->proxyuser . ':' . $CFG->proxypassword);
if (defined('CURLOPT_PROXYAUTH')) {
// any proxy authentication if PHP 5.1
curl_setopt($ch, CURLOPT_PROXYAUTH, CURLAUTH_BASIC | CURLAUTH_NTLM);
}
}
}
// set up header and content handlers
$received = new stdClass();
$received->headers = array();
// received headers array
$received->tofile = $tofile;
$received->fh = null;
curl_setopt($ch, CURLOPT_HEADERFUNCTION, partial('download_file_content_header_handler', $received));
if ($tofile) {
curl_setopt($ch, CURLOPT_WRITEFUNCTION, partial('download_file_content_write_handler', $received));
}
if (!isset($CFG->curltimeoutkbitrate)) {
//use very slow rate of 56kbps as a timeout speed when not set
$bitrate = 56;
} else {
$bitrate = $CFG->curltimeoutkbitrate;
}
// try to calculate the proper amount for timeout from remote file size.
// if disabled or zero, we won't do any checks nor head requests.
if ($calctimeout && $bitrate > 0) {
//setup header request only options
curl_setopt_array($ch, array(CURLOPT_RETURNTRANSFER => false, CURLOPT_NOBODY => true));
curl_exec($ch);
$info = curl_getinfo($ch);
$err = curl_error($ch);
if ($err === '' && $info['download_content_length'] > 0) {
//no curl errors
$timeout = max($timeout, ceil($info['download_content_length'] * 8 / ($bitrate * 1024)));
//adjust for large files only - take max timeout.
}
//reinstate affected curl options
curl_setopt_array($ch, array(CURLOPT_RETURNTRANSFER => true, CURLOPT_NOBODY => false, CURLOPT_HTTPGET => true));
}
curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);
$result = curl_exec($ch);
// try to detect encoding problems
if ((curl_errno($ch) == 23 or curl_errno($ch) == 61) and defined('CURLOPT_ENCODING')) {
curl_setopt($ch, CURLOPT_ENCODING, 'none');
$result = curl_exec($ch);
}
if ($received->fh) {
fclose($received->fh);
}
if (curl_errno($ch)) {
$error = curl_error($ch);
$error_no = curl_errno($ch);
curl_close($ch);
if ($fullresponse) {
$response = new stdClass();
if ($error_no == 28) {
$response->status = '-100';
// mimic snoopy
} else {
$response->status = '0';
}
$response->headers = array();
$response->response_code = $error;
$response->results = false;
$response->error = $error;
return $response;
} else {
debugging("cURL request for \"{$url}\" failed with: {$error} ({$error_no})", DEBUG_ALL);
return false;
}
} else {
$info = curl_getinfo($ch);
curl_close($ch);
if (empty($info['http_code'])) {
// for security reasons we support only true http connections (Location: file:// exploit prevention)
$response = new stdClass();
$response->status = '0';
$response->headers = array();
$response->response_code = 'Unknown cURL error';
$response->results = false;
// do NOT change this, we really want to ignore the result!
$response->error = 'Unknown cURL error';
} else {
$response = new stdClass();
$response->status = (string) $info['http_code'];
$response->headers = $received->headers;
$response->response_code = $received->headers[0];
$response->results = $result;
$response->error = '';
}
if ($fullresponse) {
return $response;
} else {
if ($info['http_code'] != 200) {
debugging("cURL request for \"{$url}\" failed, HTTP response code: " . $response->response_code, DEBUG_ALL);
return false;
} else {
return $response->results;
}
}
}
}
