Example #1
0
 /**
  * Uses cURL multi to simultaneously process our requests and run our
  * callback function after each request has completed.
  */
 private static function processRequests()
 {
     if (self::$totalRequests > self::$curlOptions[CURLOPT_MAXCONNECTS]) {
         $maxConnections = self::$curlOptions[CURLOPT_MAXCONNECTS];
     } else {
         $maxConnections = self::$totalRequests;
     }
     $multiHandler = curl_multi_init();
     $options = self::$curlOptions;
     $now = time();
     $secondSleep = FALSE;
     $hourSleep = FALSE;
     $isThrottledPerSecond = self::isThrottledPerSecond();
     $isThrottledPerHour = self::isThrottledPerHour();
     for ($i = 0; $i < $maxConnections; $i++) {
         $requestHandle = curl_init();
         $requestUrl = self::formatRequestUrl(self::$requests[$i]->getUrl());
         $options[CURLOPT_URL] = $requestUrl;
         self::$totalRequestsThisSecond++;
         self::$totalRequestsThisHour++;
         curl_setopt_array($requestHandle, $options);
         curl_multi_add_handle($multiHandler, $requestHandle);
     }
     do {
         while (($haveRequests = curl_multi_exec($multiHandler, $running)) == CURLM_CALL_MULTI_PERFORM) {
         }
         if ($haveRequests != CURLM_OK) {
             break;
         }
         while ($theRequest = curl_multi_info_read($multiHandler)) {
             $requestInfo = curl_getinfo($theRequest['handle']);
             $response = curl_multi_getcontent($theRequest['handle']);
             if (self::haveCallback()) {
                 call_user_func_array(self::getCallback(), array($requestInfo['url'], $response, $requestInfo, $theRequest['handle']));
             }
             if ($i < self::$totalRequests) {
                 if ($isThrottledPerSecond) {
                     if ($secondSleep) {
                         $secondSleep = FALSE;
                         sleep(1);
                         $now = time();
                         self::$totalRequestsThisSecond = 0;
                     }
                     if (time() > $now) {
                         $now = time();
                         self::$totalRequestsThisSecond = 0;
                     }
                     if (self::$totalRequestsThisSecond == self::$maxRequestsPerSecond) {
                         $secondSleep = TRUE;
                         self::$totalRequestsThisSecond = 0;
                     }
                     self::$totalRequestsThisSecond++;
                 }
                 if ($isThrottledPerHour) {
                     if ($hourSleep) {
                         $hourSleep = FALSE;
                         sleep(3600);
                         $now = time();
                         self::$totalRequestsThisHour = 0;
                     }
                     if (self::$totalRequestsThisHour == self::$maxRequestsPerHour) {
                         $hourSleep = TRUE;
                         self::$totalRequestsThisHour = 0;
                     }
                     self::$totalRequestsThisHour++;
                 }
                 $requestHandle = curl_init();
                 $options[CURLOPT_URL] = self::formatRequestUrl(self::$requests[$i++]->getUrl());
                 curl_setopt_array($requestHandle, $options);
                 curl_multi_add_handle($multiHandler, $requestHandle);
             }
             curl_multi_remove_handle($multiHandler, $theRequest['handle']);
         }
     } while ($running);
     curl_multi_close($multiHandler);
 }