/** * Uses cURL multi to simultaneously process our requests and run our * callback function after each request has completed. */ private static function processRequests() { if (self::$totalRequests > self::$curlOptions[CURLOPT_MAXCONNECTS]) { $maxConnections = self::$curlOptions[CURLOPT_MAXCONNECTS]; } else { $maxConnections = self::$totalRequests; } $multiHandler = curl_multi_init(); $options = self::$curlOptions; $now = time(); $secondSleep = FALSE; $hourSleep = FALSE; $isThrottledPerSecond = self::isThrottledPerSecond(); $isThrottledPerHour = self::isThrottledPerHour(); for ($i = 0; $i < $maxConnections; $i++) { $requestHandle = curl_init(); $requestUrl = self::formatRequestUrl(self::$requests[$i]->getUrl()); $options[CURLOPT_URL] = $requestUrl; self::$totalRequestsThisSecond++; self::$totalRequestsThisHour++; curl_setopt_array($requestHandle, $options); curl_multi_add_handle($multiHandler, $requestHandle); } do { while (($haveRequests = curl_multi_exec($multiHandler, $running)) == CURLM_CALL_MULTI_PERFORM) { } if ($haveRequests != CURLM_OK) { break; } while ($theRequest = curl_multi_info_read($multiHandler)) { $requestInfo = curl_getinfo($theRequest['handle']); $response = curl_multi_getcontent($theRequest['handle']); if (self::haveCallback()) { call_user_func_array(self::getCallback(), array($requestInfo['url'], $response, $requestInfo, $theRequest['handle'])); } if ($i < self::$totalRequests) { if ($isThrottledPerSecond) { if ($secondSleep) { $secondSleep = FALSE; sleep(1); $now = time(); self::$totalRequestsThisSecond = 0; } if (time() > $now) { $now = time(); self::$totalRequestsThisSecond = 0; } if (self::$totalRequestsThisSecond == self::$maxRequestsPerSecond) { $secondSleep = TRUE; self::$totalRequestsThisSecond = 0; } self::$totalRequestsThisSecond++; } if ($isThrottledPerHour) { if ($hourSleep) { $hourSleep = FALSE; sleep(3600); $now = time(); self::$totalRequestsThisHour = 0; } if (self::$totalRequestsThisHour == self::$maxRequestsPerHour) { $hourSleep = TRUE; self::$totalRequestsThisHour = 0; } self::$totalRequestsThisHour++; } $requestHandle = curl_init(); $options[CURLOPT_URL] = self::formatRequestUrl(self::$requests[$i++]->getUrl()); curl_setopt_array($requestHandle, $options); curl_multi_add_handle($multiHandler, $requestHandle); } curl_multi_remove_handle($multiHandler, $theRequest['handle']); } } while ($running); curl_multi_close($multiHandler); }