Exemplo n.º 1
0
 /**
  * Performs parallel asynchronous GET requests.
  * 
  * @param array $hrefs the hrefs to request
  * @param array $header the header to be passed in all requests
  * @param callable $getAuthHeader that returns an appropriate bearer authentication header line, for instance 
  * Client::getBearerAuthHeader(). We do this on-the-fly as during large multi GET batches the access token might
  * expire.
  * @param callable $callback a function expecting one \iveeCrest\Response object as argument, called for every
  * successful response
  * @param callable $errCallback a function expecting one \iveeCrest\Response object as argument, called for every
  * non-successful response
  * @param bool $cache whether the Responses should be cached
  * 
  * @return void
  * @throws \iveeCrest\Exceptions\IveeCrestException on general CURL error
  */
 public function asyncMultiGet(array $hrefs, array $header, callable $getAuthHeader, callable $callback, callable $errCallback = null, $cache = true)
 {
     //echo time2s()."curl.asyncMultiGet()\n"; //var_dump($hrefs);
     //separate hrefs that are already cached from those that need to be requested
     $hrefsToQuery = array();
     foreach ($hrefs as $href) {
         $responseKey = 'get:' . $href;
         try {
             $callback($this->cache->getItem($responseKey));
         } catch (Exceptions\KeyNotFoundInCacheException $e) {
             $hrefsToQuery[] = $href;
         }
         if (!in_array($href, $hrefsToQuery)) {
             // $hrefsToQuery: not in cache
             $url_short = str_replace(Config::getCrestBaseUrl(), '', $href);
             echo time2s() . "cache " . $url_short . "\n";
         }
     }
     // make sure the rolling window isn't greater than the number of hrefs
     $rollingWindow = count($hrefsToQuery) > 10 ? 10 : count($hrefsToQuery);
     //CURL options for all requests
     $stdOptions = array(CURLOPT_RETURNTRANSFER => true, CURLOPT_USERAGENT => $this->userAgent, CURLOPT_SSL_VERIFYPEER => true, CURLOPT_SSL_CIPHER_LIST => 'TLSv1', CURLOPT_CAINFO => __DIR__ . '/cacert.pem', CURLOPT_HTTPHEADER => $header);
     $responses = array();
     $master = curl_multi_init();
     //setup the first batch of requests
     for ($i = 0; $i < $rollingWindow; $i++) {
         $href = $hrefsToQuery[$i];
         //echo time2s()."curl.multi  $href\n";
         $responses[$href] = $this->addHandleToMulti($master, $href, $stdOptions, $getAuthHeader, $header);
     }
     $running = false;
     do {
         //execute whichever handles need to be started
         do {
             $execrun = curl_multi_exec($master, $running);
         } while ($execrun == CURLM_CALL_MULTI_PERFORM);
         if ($execrun != CURLM_OK) {
             $crestExceptionClass = Config::getIveeClassName('IveeCrestException');
             throw new $crestExceptionClass("CURL Multi-GET error", $execrun);
         }
         //block until we have anything on at least one of the handles
         curl_multi_select($master);
         //a request returned, process it
         while ($done = curl_multi_info_read($master)) {
             //echo "curl_multi_info_read()...\n"; var_dump($done);
             $info = curl_getinfo($done['handle']);
             //find the Response object matching the URL
             $res = $responses[$info['url']];
             $url_short = str_replace(Config::getCrestBaseUrl(), '', $info['url']);
             //set info and content to Response object
             $res->setInfo($info);
             $res->setContent(curl_multi_getcontent($done['handle']));
             //execute the callbacks passing the response as argument
             if ($info['http_code'] == 200) {
                 //cache it if configured
                 if ($cache) {
                     $this->cache->setItem($res);
                 }
                 $callback($res);
                 if (isset($this->requeued[$info['url']])) {
                     $this->requeued[$info['url']] = NULL;
                     time2s() . ">>> recaptured " . $url_short . "\n";
                 }
                 echo time2s() . "got   " . $url_short . "\n";
             } elseif (isset($errCallback)) {
                 echo time2s() . "cw.asyncMultiGet(): curl_multi, http " . $info['http_code'] . "\n";
                 echo time2s() . "requeueing " . $url_short . "\n";
                 $errCallback($res);
                 $hrefsToQuery[] = $info['url'];
                 //put back on queue
                 $this->requeued[$info['url']] = true;
             }
             //remove the reference to response to conserve memory on large batches
             $responses[$info['url']] = null;
             //start a new request (it's important to do this before removing the old one)
             if ($i < count($hrefsToQuery)) {
                 $href = $hrefsToQuery[$i++];
                 //echo time2s()."curl.multi  $href\n";
                 $responses[$href] = $this->addHandleToMulti($master, $href, $stdOptions, $getAuthHeader, $header);
             }
             //remove the curl handle that just completed
             curl_multi_remove_handle($master, $done['handle']);
         }
         //don't waste too many CPU cycles on looping
         usleep(1000);
     } while ($running > 0);
     curl_multi_close($master);
 }
Exemplo n.º 2
0
 public function printCache()
 {
     $this->cache->printCache();
 }