/**
* Open one or more URL, and return an array of arrays with dictionary of header information and a stream to the contents of the URL
* @param $urls array of strings of resource to download
* @return array of dictionaries of header information and the contents of the URL
*/
public static function open_multi_url($urls, $additional_options = array())
{
if (AmberNetworkUtils::curl_installed()) {
$result = array();
try {
$options = array(CURLOPT_FAILONERROR => TRUE, CURLOPT_FOLLOWLOCATION => AmberNetworkUtils::curl_redirects_allowed(), CURLOPT_MAXREDIRS => 10, CURLOPT_CONNECTTIMEOUT => 5, CURLOPT_TIMEOUT => 10, CURLOPT_RETURNTRANSFER => 1, CURLOPT_HEADER => TRUE, CURLOPT_USERAGENT => AmberNetworkUtils::get_user_agent_string(), CURLOPT_ENCODING => '', CURLINFO_HEADER_OUT => 1);
$multi = curl_multi_init();
$channels = array();
foreach ($urls as $url) {
if (($ch = curl_init($url)) === FALSE) {
error_log(join(":", array(__FILE__, __METHOD__, $url, "CURL init error")));
return FALSE;
}
if (curl_setopt_array($ch, $additional_options + $options) === FALSE) {
throw new RuntimeException(join(":", array(__FILE__, __METHOD__, "Error setting CURL options", $url, curl_error($ch))));
}
curl_multi_add_handle($multi, $ch);
$channels[$url] = $ch;
}
/* While we're still active, execute curl over all the channels */
$active = null;
do {
$mrc = curl_multi_exec($multi, $active);
} while ($mrc == CURLM_CALL_MULTI_PERFORM);
while ($active && $mrc == CURLM_OK) {
curl_multi_select($multi);
do {
$mrc = curl_multi_exec($multi, $active);
} while ($mrc == CURLM_CALL_MULTI_PERFORM);
}
/* Now we should have all of the data */
foreach ($channels as $url => $channel) {
/* Get the CURL result */
$data = curl_multi_getcontent($channel);
$response_info = curl_getinfo($channel);
/* Split into header and body */
$header_size = $response_info['header_size'];
$header = substr($data, 0, $header_size - 1);
$body = substr($data, $header_size);
$headers = AmberNetworkUtils::extract_headers($header);
$result[$url] = array("headers" => $headers, "body" => $body, "info" => $response_info);
curl_multi_remove_handle($multi, $channel);
}
curl_multi_close($multi);
/* It's possible that one or more of these responses may require a redirect
that hasn't yet been followed. Some cases where this could happen:
- The webserver has safe_mode or open_basedir set, so we couldn't set CURLOPT_FOLLOWLOCATION
- The redirect is triggered by a META tag in the HTML
- The redirect is triggered by Javascript (We do NOT handle this case)
For the first two cases, which we can handle, we find URLs that still need redirection,
and fetch them. */
$redirects_required = AmberNetworkUtils::find_urls_requiring_redirects($result);
foreach ($redirects_required as $url => $data) {
$a = AmberNetworkUtils::open_single_url($url, $additional_options);
if ($a) {
$result[$url] = $a;
}
}
return $result;
} catch (RuntimeException $e) {
error_log($e->getMessage());
curl_multi_close($multi);
return FALSE;
}
} else {
// TODO: If curl is not installed, see if remote file opening is enabled, and fall back to that method
error_log(join(":", array(__FILE__, __METHOD__, "CURL not installed")));
return FALSE;
}
}
