/**
* Performs parallel asynchronous GET requests.
*
* @param array $hrefs the hrefs to request
* @param array $header the header to be passed in all requests
* @param callable $getAuthHeader that returns an appropriate bearer authentication header line, for instance
* Client::getBearerAuthHeader(). We do this on-the-fly as during large multi GET batches the access token might
* expire.
* @param callable $callback a function expecting one \iveeCrest\Response object as argument, called for every
* successful response
* @param callable $errCallback a function expecting one \iveeCrest\Response object as argument, called for every
* non-successful response
* @param bool $cache whether the Responses should be cached
*
* @return void
* @throws \iveeCrest\Exceptions\IveeCrestException on general CURL error
*/
public function asyncMultiGet(array $hrefs, array $header, callable $getAuthHeader, callable $callback, callable $errCallback = null, $cache = true)
{
//echo time2s()."curl.asyncMultiGet()\n"; //var_dump($hrefs);
//separate hrefs that are already cached from those that need to be requested
$hrefsToQuery = array();
foreach ($hrefs as $href) {
$responseKey = 'get:' . $href;
try {
$callback($this->cache->getItem($responseKey));
} catch (Exceptions\KeyNotFoundInCacheException $e) {
$hrefsToQuery[] = $href;
}
if (!in_array($href, $hrefsToQuery)) {
// $hrefsToQuery: not in cache
$url_short = str_replace(Config::getCrestBaseUrl(), '', $href);
echo time2s() . "cache " . $url_short . "\n";
}
}
// make sure the rolling window isn't greater than the number of hrefs
$rollingWindow = count($hrefsToQuery) > 10 ? 10 : count($hrefsToQuery);
//CURL options for all requests
$stdOptions = array(CURLOPT_RETURNTRANSFER => true, CURLOPT_USERAGENT => $this->userAgent, CURLOPT_SSL_VERIFYPEER => true, CURLOPT_SSL_CIPHER_LIST => 'TLSv1', CURLOPT_CAINFO => __DIR__ . '/cacert.pem', CURLOPT_HTTPHEADER => $header);
$responses = array();
$master = curl_multi_init();
//setup the first batch of requests
for ($i = 0; $i < $rollingWindow; $i++) {
$href = $hrefsToQuery[$i];
//echo time2s()."curl.multi $href\n";
$responses[$href] = $this->addHandleToMulti($master, $href, $stdOptions, $getAuthHeader, $header);
}
$running = false;
do {
//execute whichever handles need to be started
do {
$execrun = curl_multi_exec($master, $running);
} while ($execrun == CURLM_CALL_MULTI_PERFORM);
if ($execrun != CURLM_OK) {
$crestExceptionClass = Config::getIveeClassName('IveeCrestException');
throw new $crestExceptionClass("CURL Multi-GET error", $execrun);
}
//block until we have anything on at least one of the handles
curl_multi_select($master);
//a request returned, process it
while ($done = curl_multi_info_read($master)) {
//echo "curl_multi_info_read()...\n"; var_dump($done);
$info = curl_getinfo($done['handle']);
//find the Response object matching the URL
$res = $responses[$info['url']];
$url_short = str_replace(Config::getCrestBaseUrl(), '', $info['url']);
//set info and content to Response object
$res->setInfo($info);
$res->setContent(curl_multi_getcontent($done['handle']));
//execute the callbacks passing the response as argument
if ($info['http_code'] == 200) {
//cache it if configured
if ($cache) {
$this->cache->setItem($res);
}
$callback($res);
if (isset($this->requeued[$info['url']])) {
$this->requeued[$info['url']] = NULL;
time2s() . ">>> recaptured " . $url_short . "\n";
}
echo time2s() . "got " . $url_short . "\n";
} elseif (isset($errCallback)) {
echo time2s() . "cw.asyncMultiGet(): curl_multi, http " . $info['http_code'] . "\n";
echo time2s() . "requeueing " . $url_short . "\n";
$errCallback($res);
$hrefsToQuery[] = $info['url'];
//put back on queue
$this->requeued[$info['url']] = true;
}
//remove the reference to response to conserve memory on large batches
$responses[$info['url']] = null;
//start a new request (it's important to do this before removing the old one)
if ($i < count($hrefsToQuery)) {
$href = $hrefsToQuery[$i++];
//echo time2s()."curl.multi $href\n";
$responses[$href] = $this->addHandleToMulti($master, $href, $stdOptions, $getAuthHeader, $header);
}
//remove the curl handle that just completed
curl_multi_remove_handle($master, $done['handle']);
}
//don't waste too many CPU cycles on looping
usleep(1000);
} while ($running > 0);
curl_multi_close($master);
}
/**
* Gathers multipage endpoint responses and joins them into one array, using the passed callback functions to
* traverse and index the data. Since this operation is potentially expensive, it is recommended to use
* gatherCached() instead, which introduces another layer of caching.
*
* @param string $endpointHref the URL to the first page of the endpoint
* @param callable $indexFunc function to be used to extract the ID from/for and individual response item
* @param callable $elementFunc function to be used to extract the desired data from and individual response item
* @param string $accept the representation to request from CREST
*
* @return array
*/
public function gather($endpointHref, callable $indexFunc = null, callable $elementFunc = null, $accept = null)
{
//if (strpos($endpointHref, 'https://crest-tq.eveonline.com/market/types') !== false)
if (strpos($endpointHref, 'https://public-crest.eveonline.com/market/types') !== false) {
//echo time2s()."cl.gather(mktTypes)\n";
//echo " indexFunc="; echo ($indexFunc == null) ? '(null)' : 'defined';
//echo " elementFunc="; echo ($elementFunc == null) ? '(null)' : 'defined';
//echo " accept="; echo ($elementFunc == null) ? '(null)' : 'defined';
//echo "\n";
} else {
//echo time2s()."cl.gather()\n";
}
$ret = array();
$href = $endpointHref;
while (true) {
//get the response for the current href
$response = $this->getEndpointResponse($href, true, $accept);
foreach ($response->content->items as $item) {
//if an element function has been given, call it, otherwise use the full item in the result array
if (is_null($elementFunc)) {
$element = $item;
} else {
$element = $elementFunc($item);
}
//if an index function has been given, call it to get a result key, otherwise just push the element
//onto result array
if (is_null($indexFunc)) {
$ret[] = $element;
} else {
$ret[$indexFunc($item)] = $element;
}
}
//if there are more pages, do another iteration with updated href
if ($response->hasNextPage()) {
$href = $response->getNextPageHref();
}
// HACK: for /market/types/?page=[2-N], do async multiget
// multiget faster (5-18s) vs. 11 sequential gets (21-29s)
//$prefix = 'https://crest-tq.eveonline.com/market/types/?page=';
//$prefix = 'https://public-crest.eveonline.com/market/types/?page='; // no-auth CREST
$base = Config::getCrestBaseUrl();
$prefix = $base . 'market/types/?page=';
if (strpos($href, $prefix . '2') !== false) {
$hrefs = array();
$pageCount = $response->content->pageCount;
for ($i = 2; $i <= $pageCount; $i++) {
$hrefs[] = $prefix . $i;
}
$this->asyncGetMultiEndpointResponses($hrefs, function (Response $res2) use(&$ret, &$elementFunc, &$indexFunc) {
foreach ($res2->content->items as $item2) {
$element2 = is_null($elementFunc) ? $item2 : $elementFunc($item2);
if (is_null($indexFunc)) {
$ret[] = $element2;
} else {
$ret[$indexFunc($item2)] = $element2;
}
}
}, function (Response $res) {
//echo time2s().">>> cl.gather().callbackErrMarketType() ".str_replace(Config::getCrestBaseUrl(), '', $res->info['url'])."\n";
var_dump($res);
}, $accept);
break;
} else {
break;
}
}
return $ret;
}
