public static function fetch_proxyable_url_set($domain, $etag, $ps_domain) { if (!$domain) { $failure_message = 'Domain not provided, not fetching sitemap.xml'; UBLogger::warning($failure_message); return UBConfig::create_failure_response($failure_message); } try { $url = 'https://' . $ps_domain . '/sitemap.xml'; $curl = curl_init(); $curl_options = array(CURLOPT_URL => $url, CURLOPT_CUSTOMREQUEST => "GET", CURLOPT_HEADER => true, CURLOPT_USERAGENT => UBConfig::UB_USER_AGENT, CURLOPT_HTTPHEADER => array('Host: ' . $domain, 'If-None-Match: ' . $etag), CURLOPT_RETURNTRANSFER => true, CURLOPT_FOLLOWLOCATION => false, CURLOPT_TIMEOUT => 5); UBLogger::debug("Retrieving routes from '{$url}', etag: '{$etag}', host: '{$domain}'"); curl_setopt_array($curl, $curl_options); $data = curl_exec($curl); $http_code = curl_getinfo($curl, CURLINFO_HTTP_CODE); $header_size = strlen($data) - curl_getinfo($curl, CURLINFO_SIZE_DOWNLOAD); $curl_error = null; $etag = null; $max_age = null; // when having an CURL error, http_code is 0 if ($http_code == 0) { $curl_error = curl_error($curl); } curl_close($curl); $headers = substr($data, 0, $header_size); $matches = array(); $does_match = preg_match('/ETag: (\\S+)/is', $headers, $matches); if ($does_match) { $etag = $matches[1]; } $matches = array(); $does_match = preg_match('/Cache-Control: max-age=(\\S+)/is', $headers, $matches); if ($does_match) { $max_age = $matches[1]; } if ($http_code == 200) { $body = substr($data, $header_size); list($success, $result) = UBConfig::url_list_from_sitemap($body); if ($success) { UBLogger::debug("Retrieved new routes, HTTP code: '{$http_code}'"); return UBConfig::create_new_response($etag, $max_age, $result); } else { $errors = join(', ', $result); $failure_message = "An error occurred while processing pages, XML errors: '{$errors}'"; UBLogger::warning($failure_message); return UBConfig::create_failure_response($failure_message); } } if ($http_code == 304) { UBLogger::debug("Routes have not changed, HTTP code: '{$http_code}'"); return UBConfig::create_same_response($etag, $max_age); } if ($http_code == 404) { UBLogger::debug("No routes to retrieve, HTTP code: '{$http_code}'"); return UBConfig::create_none_response(); } else { $failure_message = "An error occurred while retrieving routes; HTTP code: '{$http_code}'; Error: " . $curl_error; UBLogger::warning($failure_message); return UBConfig::create_failure_response($failure_message); } } catch (Exception $e) { $failure_message = "An error occurred while retrieving routes; Error: " . $e; UBLogger::warning($failure_message); return UBConfig::create_failure_response($failure_message); } }