/** * @return resource */ protected function getCurlMulti() { if (!$this->multiHandle) { $cmh = curl_multi_init(); if (function_exists('curl_multi_setopt')) { // PHP 5.5 curl_multi_setopt($cmh, CURLMOPT_PIPELINING, (int) $this->usePipelining); curl_multi_setopt($cmh, CURLMOPT_MAXCONNECTS, (int) $this->maxConnsPerHost); } $this->multiHandle = $cmh; } return $this->multiHandle; }
public function execute() { $mh = curl_multi_init(); $chs = $this->chs(); $stillRunning = 0; $result = array(); if (function_exists('curl_multi_setopt')) { curl_multi_setopt($mh, CURLMOPT_PIPELINING, 1); curl_multi_setopt($mh, CURLMOPT_MAXCONNECTS, $this->maxConnectsSize()); } foreach ($chs as $ch) { curl_multi_add_handle($mh, $ch); } do { $execReturnValue = curl_multi_exec($mh, $stillRunning); curl_multi_select($mh); } while ($stillRunning > 0); foreach ($chs as $i => $ch) { $curlError = curl_error($ch); if ($curlError === "") { $result[$i] = curl_multi_getcontent($ch); } else { throw new PubnubException("Curl error on handle {$i}: {$curlError}\n"); } curl_multi_remove_handle($mh, $ch); curl_close($ch); } curl_multi_close($mh); $this->requests = array(); if ($execReturnValue != CURLM_OK) { throw new PubnubException(curl_multi_strerror($execReturnValue)); } return $result; }
/** * Set cURL option * * @param array $options * * @throws GrasshopperException */ public function setOptions($options) { foreach ($options as $key => $value) { $res = curl_multi_setopt($this->mh, $key, $value); if (!$res) { throw new GrasshopperException('curl_setopt_array failed', Grasshopper::ERROR_M_SETOPTIONS); } } }
/** * Constructor. * Initialize cURL multi handle. * @param CoOption $options */ public function __construct(CoOption $options) { $this->mh = curl_multi_init(); $flags = (int) $options['pipeline'] + (int) $options['multiplex'] * 2; curl_multi_setopt($this->mh, CURLMOPT_PIPELINING, $flags); $this->options = $options; $this->scheduler = $options['autoschedule'] ? new AutoScheduler($options, $this->mh) : new ManualScheduler($options, $this->mh); $this->delayer = new Delayer(); }
/** * Class Constructor, init the multi curl handler * */ public function __construct() { $this->multi_handler = curl_multi_init(); Utils::getPHPVersion(); if (PHP_MINOR_VERSION >= 5) { // curl_multi_setopt only for (PHP 5 >= 5.5.0) Default 10 curl_multi_setopt($this->multi_handler, CURLMOPT_MAXCONNECTS, 50); } }
/** * @return resource */ private function handlerMulti() { if (!$this->_pool_master) { $this->_pool_master = curl_multi_init(); if (function_exists('curl_multi_setopt')) { curl_multi_setopt($this->_pool_master, CURLMOPT_MAXCONNECTS, $this->simultaneousLimit); } } return $this->_pool_master; }
public function __construct($options = null) { $this->options = $options; $curl_opts = null; $this->curl_opts = $curl_opts ? $curl_opts : []; // TODO: cURL "eventization" with libevent/libev/libuv and integration with AsyncSteps loop $this->curl_mh = curl_multi_init(); # Missing on HHVM if (function_exists('curl_multi_setopt')) { curl_multi_setopt($this->curl_mh, CURLMOPT_PIPELINING, 1); curl_multi_setopt($this->curl_mh, CURLMOPT_MAXCONNECTS, isset($curl_opts[CURLOPT_MAXCONNECTS]) ? $curl_opts[CURLOPT_MAXCONNECTS] : 8); } }
public function setOption($options, $value = null) { if (is_int($options) && $value !== null) { curl_multi_setopt($this->_mh, $options, $value); } elseif (is_array($options)) { foreach ($options as $option => $value) { $this->setOption($option, $value); } } else { throw new \InvalidArgumentException('$option type must be int or array'); } return $this; }
/** * Set an option for the cURL multi handle * * @param int $option One of the CURLMOPT_* constants. * @param mixed $value The value to be set on option. * @return self */ public final function curlMultiSetOpt($option, $value) { curl_multi_setopt($this->curl_multi_handle, $option, $value); return $this; }
/** * @param $curlHandles * @throws RoutingMachineException * @return array */ private function runMultiCurlsBlockWise($curlHandles) { //we run curl_multi only on a BLOCKSIZE so we don't generate a flood $curlMultiHandle = curl_multi_init(); curl_multi_setopt($curlMultiHandle, CURLMOPT_PIPELINING, 1); curl_multi_setopt($curlMultiHandle, CURLMOPT_MAXCONNECTS, self::BLOCK_SIZE); $responses = array(); $blockCount = 0; // count where we are in the list so we can break up the runs into smaller blocks $blockResults = array(); // to accumulate the curl_handles for each group we'll run simultaneously foreach ($curlHandles as $hashKey => $curlHandle) { $blockCount++; // add the handle to the curl_multi_handle and to our tracking "block" curl_multi_add_handle($curlMultiHandle, $curlHandle); $blockResults[$hashKey] = $curlHandle; if ($blockCount % self::BLOCK_SIZE === 0 or $blockCount === count($curlHandles)) { $running = NULL; do { // track the previous loop's number of handles still running so we can tell if it changes $runningBefore = $running; // run the block or check on the running block and get the number of sites still running in $running curl_multi_exec($curlMultiHandle, $running); // if the number of sites still running changed, print out a message with the number of sites that are still running. if ($running != $runningBefore) { //waiting for running sites to finish } } while ($running > 0); //once the number still running is 0, curl_multi_ is done, so check the results foreach ($blockResults as $hashKeyBlock => $result) { // HTTP response code $code = curl_getinfo($result, CURLINFO_HTTP_CODE); // cURL error number $curlErrno = curl_errno($result); // cURL error message $curlError = curl_error($result); // output if there was an error if ($curlError) { throw new RoutingMachineException("OSRM cURL error: ({$curlErrno}) {$curlError}\n"); } // fill results from our requests we get $response = curl_multi_getcontent($result); $responses[$hashKeyBlock] = $response; // remove the (used) handle from the curl_multi_handle curl_multi_remove_handle($curlMultiHandle, $result); } // reset the block to empty, since we've run its curl_handles $blockResults = array(); } } // close the curl_multi_handle once we're done curl_multi_close($curlMultiHandle); return $responses; }
/** * @param Package\PackageInterface[] $packages * @param array $pluginConfig * @return void */ public function download(array $packages, array $pluginConfig) { $mh = curl_multi_init(); $unused = array(); $maxConns = $pluginConfig['maxConnections']; for ($i = 0; $i < $maxConns; ++$i) { $unused[] = curl_init(); } /// @codeCoverageIgnoreStart if (function_exists('curl_share_init')) { $sh = curl_share_init(); curl_share_setopt($sh, CURLSHOPT_SHARE, CURL_LOCK_DATA_SSL_SESSION); foreach ($unused as $ch) { curl_setopt($ch, CURLOPT_SHARE, $sh); } } if (function_exists('curl_multi_setopt')) { if ($pluginConfig['pipeline']) { curl_multi_setopt($mh, CURLMOPT_PIPELINING, true); } } /// @codeCoverageIgnoreEnd $cachedir = rtrim($this->config->get('cache-files-dir'), '\\/'); $chFpMap = array(); $running = 0; //ref type $remains = 0; //ref type $this->totalCnt = count($packages); $this->successCnt = 0; $this->failureCnt = 0; $this->io->write(" Prefetch start: <comment>success: {$this->successCnt}, failure: {$this->failureCnt}, total: {$this->totalCnt}</comment>"); do { // prepare curl resources while ($unused && $packages) { $package = array_pop($packages); $filepath = $cachedir . DIRECTORY_SEPARATOR . static::getCacheKey($package); if (file_exists($filepath)) { ++$this->successCnt; continue; } $ch = array_pop($unused); // make file resource $fp = CurlRemoteFilesystem::createFile($filepath); $chFpMap[(int) $ch] = compact('fp', 'filepath'); // make url $url = $package->getDistUrl(); $request = new Aspects\HttpGetRequest(parse_url($url, PHP_URL_HOST), $url, $this->io); $request->verbose = $pluginConfig['verbose']; if (in_array($package->getName(), $pluginConfig['privatePackages'])) { $request->maybePublic = false; } else { $request->maybePublic = preg_match('%^(?:https|git)://github\\.com%', $package->getSourceUrl()); } $onPreDownload = Factory::getPreEvent($request); $onPreDownload->notify(); $opts = $request->getCurlOpts(); unset($opts[CURLOPT_ENCODING]); unset($opts[CURLOPT_USERPWD]); curl_setopt_array($ch, $opts); curl_setopt($ch, CURLOPT_FILE, $fp); curl_multi_add_handle($mh, $ch); } // start multi download do { $stat = curl_multi_exec($mh, $running); } while ($stat === CURLM_CALL_MULTI_PERFORM); // wait for any event do { switch (curl_multi_select($mh, 5)) { case -1: usleep(10); do { $stat = curl_multi_exec($mh, $running); } while ($stat === CURLM_CALL_MULTI_PERFORM); continue 2; case 0: continue 2; default: do { $stat = curl_multi_exec($mh, $running); } while ($stat === CURLM_CALL_MULTI_PERFORM); do { if ($raised = curl_multi_info_read($mh, $remains)) { $ch = $raised['handle']; $errno = curl_errno($ch); $info = curl_getinfo($ch); curl_setopt($ch, CURLOPT_FILE, STDOUT); $index = (int) $ch; $fileinfo = $chFpMap[$index]; unset($chFpMap[$index]); $fp = $fileinfo['fp']; $filepath = $fileinfo['filepath']; fclose($fp); if (CURLE_OK === $errno && 200 === $info['http_code']) { ++$this->successCnt; } else { ++$this->failureCnt; unlink($filepath); } $this->io->write($this->makeDownloadingText($info['url'])); curl_multi_remove_handle($mh, $ch); $unused[] = $ch; } } while ($remains); if ($packages) { break 2; } } } while ($running); } while ($packages); $this->io->write(" Finished: <comment>success: {$this->successCnt}, failure: {$this->failureCnt}, total: {$this->totalCnt}</comment>"); foreach ($unused as $ch) { curl_close($ch); } curl_multi_close($mh); }
/** * Max request in Asynchron query * @param $max int default:10 * @return void */ public function setMaxRequest($max) { $this->maxRequest = $max; // PHP 5 >= 5.5.0 if (function_exists('curl_multi_setopt')) { curl_multi_setopt($this->mh, CURLMOPT_MAXCONNECTS, $max); } }
/** * Set an option for the cURL multi handle * * @param integer $option * @param mixed $value * @return boolean */ public function setopt($option, $value) { return curl_multi_setopt($this->_mh, $option, $value); }
<?php $mh = curl_multi_init(); var_dump(curl_multi_setopt($mh, CURLMOPT_PIPELINING, 0)); var_dump(curl_multi_setopt($mh, -1, 0));
/** * @since 1.0 * * @param string $name * @param mixed $value */ public function setOption($name, $value) { $this->options[$name] = $value; curl_multi_setopt($this->handle, $name, $value); }
/** * Constructor. * Initialize cURL multi handle. * @param CoOption $options * @param resource $mh curl_multi */ public function __construct(CoOption $options, $mh) { curl_multi_setopt($mh, CURLMOPT_MAX_TOTAL_CONNECTIONS, $options['concurrency']); $this->mh = $mh; $this->options = $options; }
/** * Get (and create if required) the Curl Multi handle. * * @return Resource $curl_multi_handle a Curl Multi handle as might be returned by curl_multi_init() **/ private function getCurlMultiHandle() { if (!isset($this->curl_multi_handle)) { $this->curl_multi_handle = curl_multi_init(); if (function_exists('curl_multi_setopt')) { // More efficient if this is available; not an error if not. curl_multi_setopt($this->curl_multi_handle, CURLMOPT_PIPELINING, true); } } return $this->curl_multi_handle; }
public function _setMultiOption() { if ((double) phpversion() >= 5.5) { foreach ($this->multiOptions as $optKey => $optValue) { printf('%s => %s', $optKey, $optValue); curl_multi_setopt($this->mh, $optKey, $optValue); } } }
/** * @param Package\PackageInterface[] $packages * @param array $pluginConfig * @return void */ public function download(array $packages, array $pluginConfig) { $mh = curl_multi_init(); $unused = array(); $maxConns = $pluginConfig['maxConnections']; for ($i = 0; $i < $maxConns; ++$i) { $unused[] = curl_init(); } // @codeCoverageIgnoreStart if (function_exists('curl_share_init')) { $sh = curl_share_init(); curl_share_setopt($sh, CURLSHOPT_SHARE, CURL_LOCK_DATA_SSL_SESSION); foreach ($unused as $ch) { curl_setopt($ch, CURLOPT_SHARE, $sh); } } if (function_exists('curl_multi_setopt')) { if ($pluginConfig['pipeline']) { curl_multi_setopt($mh, CURLMOPT_PIPELINING, true); } } // @codeCoverageIgnoreEnd $cachedir = rtrim($this->config->get('cache-files-dir'), '\\/'); $chFpMap = array(); $running = 0; //ref type $remains = 0; //ref type $this->totalCnt = count($packages); $this->successCnt = 0; $this->failureCnt = 0; $this->io->write(" Prefetch start: <comment>success: {$this->successCnt}, failure: {$this->failureCnt}, total: {$this->totalCnt}</comment>"); EVENTLOOP: // prepare curl resources while (count($unused) > 0 && count($packages) > 0) { $package = array_pop($packages); $filepath = $cachedir . DIRECTORY_SEPARATOR . static::getCacheKey($package); if (file_exists($filepath)) { ++$this->successCnt; continue; } $ch = array_pop($unused); // make file resource $chFpMap[(int) $ch] = $outputFile = new OutputFile($filepath); // make url $url = $package->getDistUrls(); if (count($url) > 0) { $url = $url[0]; } else { $url = $package->getDistUrl(); } $host = parse_url($url, PHP_URL_HOST) ?: ''; $request = new Aspects\HttpGetRequest($host, $url, $this->io); $request->verbose = $pluginConfig['verbose']; if (in_array($package->getName(), $pluginConfig['privatePackages'])) { $request->maybePublic = false; } else { $request->maybePublic = (bool) preg_match('%^(?:https|git)://github\\.com%', $package->getSourceUrl()); } $onPreDownload = Factory::getPreEvent($request); $onPreDownload->notify(); $opts = $request->getCurlOpts(); if ($pluginConfig['insecure']) { $opts[CURLOPT_SSL_VERIFYPEER] = false; } if (!empty($pluginConfig['userAgent'])) { $opts[CURLOPT_USERAGENT] = $pluginConfig['userAgent']; } if (!empty($pluginConfig['capath'])) { $opts[CURLOPT_CAPATH] = $pluginConfig['capath']; } unset($opts[CURLOPT_ENCODING]); unset($opts[CURLOPT_USERPWD]); // ParallelDownloader doesn't support private packages. curl_setopt_array($ch, $opts); curl_setopt($ch, CURLOPT_FILE, $outputFile->getPointer()); curl_multi_add_handle($mh, $ch); } // wait for any event do { $runningBefore = $running; while (CURLM_CALL_MULTI_PERFORM === curl_multi_exec($mh, $running)) { } SELECT: $eventCount = curl_multi_select($mh, 5); if ($eventCount === -1) { usleep(200 * 1000); continue; } if ($eventCount === 0) { continue; } while (CURLM_CALL_MULTI_PERFORM === curl_multi_exec($mh, $running)) { } if ($running > 0 && $running === $runningBefore) { goto SELECT; } do { if ($raised = curl_multi_info_read($mh, $remains)) { $ch = $raised['handle']; $errno = curl_errno($ch); $info = curl_getinfo($ch); curl_setopt($ch, CURLOPT_FILE, STDOUT); $index = (int) $ch; $outputFile = $chFpMap[$index]; unset($chFpMap[$index]); if (CURLE_OK === $errno && 200 === $info['http_code']) { ++$this->successCnt; } else { ++$this->failureCnt; $outputFile->setFailure(); } unset($outputFile); $this->io->write($this->makeDownloadingText($info['url'])); curl_multi_remove_handle($mh, $ch); $unused[] = $ch; } } while ($remains > 0); if (count($packages) > 0) { goto EVENTLOOP; } } while ($running > 0); $this->io->write(" Finished: <comment>success: {$this->successCnt}, failure: {$this->failureCnt}, total: {$this->totalCnt}</comment>"); foreach ($unused as $ch) { curl_close($ch); } curl_multi_close($mh); }
/** * @param RequestsCollection|RequestInterface $requests * * @return ResponsesCollection * @throws Exception */ public function exec($requests) { if ($requests instanceof RequestInterface) { $request = $requests; $requests = new RequestsCollection(); $requests->add($request); } if (false == $requests instanceof RequestsCollection) { throw new Exception('Invalid requests! Param $requests must be an instance of ' . 'RequestInterface or a RequestsCollection.'); } $curlMultiHandler = curl_multi_init(); if (true == $this->usePipelining) { curl_multi_setopt($curlMultiHandler, CURLMOPT_PIPELINING, 1); } $curlHandlers = []; $totalRequest = count($requests); $auxCtr = 0; $this->logger->debug(sprintf('Adding curl handlers for %s request...', $totalRequest)); /** @var RequestInterface $request */ foreach ($requests as $request) { $auxCtr++; $requestCurlHandler = new PhpCurlHandler(); $requestCurlHandler->setOption(CURLOPT_URL, $request->getUrl()); $requestCurlHandler->setOption(CURLOPT_RETURNTRANSFER, true); $requestCurlHandler->setOption(CURLOPT_FOLLOWLOCATION, false); $requestCurlHandler->setOption(CURLOPT_CUSTOMREQUEST, strtoupper($request->getRequestMethod())); $requestCurlHandler->setOption(CURLOPT_TIMEOUT, $request->getTimeout()); $requestCurlHandler->setOption(CURLOPT_CONNECTTIMEOUT, $request->getConnectTimeout()); $requestCurlHandler->setOption(CURLOPT_SSL_VERIFYPEER, $request->peersSSLCertificateVerificationIsRequired()); // Set headers if (true == $request->hasHeaders()) { $requestCurlHandler->setOption(CURLOPT_HTTPHEADER, $request->getHeaders()); } // Set request user and pass if needed if (true == $request->hasAuth()) { $requestCurlHandler->setOption(CURLOPT_HTTPAUTH, CURLAUTH_BASIC); $requestCurlHandler->setOption(CURLOPT_USERPWD, $request->getUser() . ':' . $request->getPass()); } // Set user agent if (true == $request->hasUserAgent()) { $requestCurlHandler->setOption(CURLOPT_USERAGENT, $request->getUserAgent()); } // Set request params if (true == $request->hasParams()) { $requestCurlHandler->setOption(CURLOPT_POST, 1); $requestCurlHandler->setOption(CURLOPT_POSTFIELDS, $request->getEncodedParams()); } $this->logger->debug(sprintf('Adding curl handler for request %s of %s (id:%s): [%s] %s ', $auxCtr, $totalRequest, $request->getId(), $request->getRequestMethod(), $request->getUrl()), ['params' => $request->getParams(), 'headers' => $request->getHeaders()]); $curlHandlers[$request->getId()] = $requestCurlHandler->getHandler(); curl_multi_add_handle($curlMultiHandler, $requestCurlHandler->getHandler()); } $this->logger->debug(sprintf('Executing curl multi request (%s requests)...', $totalRequest)); $initTime = microtime(true); $active = null; do { $status = curl_multi_exec($curlMultiHandler, $active); $info = curl_multi_info_read($curlMultiHandler); if ($status > 0) { throw new Exception(sprintf('Curl error "%s"!', $status)); } if (false !== $info) { if (isset($info['result']) && 0 < $info['result']) { $url = ''; $errorMsg = ''; if (isset($info['handle'])) { $errorMsg = curl_error($info['handle']); $url = curl_getinfo($info['handle'], CURLINFO_EFFECTIVE_URL); } throw new Exception(sprintf('Error [%s] calling URL "%s" (%s)!', $info['result'], $url, $errorMsg)); } } } while ($status === CURLM_CALL_MULTI_PERFORM || $active); $this->logger->debug(sprintf('Executed %s requests in %ss...', $totalRequest, microtime(true) - $initTime)); $responses = new ResponsesCollection(); $this->logger->debug('Composing responses...'); $auxCtr = 0; foreach ($curlHandlers as $requestId => $curlHandler) { $auxCtr++; $this->logger->debug(sprintf('Composing response for request %s (%s of %s)...', $requestId, $auxCtr, $totalRequest)); $httpCode = curl_getinfo($curlHandler, CURLINFO_HTTP_CODE); $contents = curl_multi_getcontent($curlHandler); $this->logger->debug(sprintf('Response for request %s [code: %s]: %s', $requestId, $httpCode, $this->mutiLineStringToOneLineString($contents))); $responses->add($requestId, $this->getResponsesFactory()->create((string) $contents, $httpCode)); // Remove handler from curl multi handler curl_multi_remove_handle($curlMultiHandler, $curlHandler); } // Close curl multi handler curl_multi_close($curlMultiHandler); $this->logger->debug(sprintf('Total execution time: %ss)...', microtime(true) - $initTime)); return $responses; }
<?php $transfers = 1; $callback = function () use(&$transfers) { return CURL_PUSH_DENY; $transfers++; }; $mh = curl_multi_init(); curl_multi_setopt($mh, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX); curl_multi_setopt($mh, CURLMOPT_PUSHFUNCTION, $callback); $ch = curl_init(); curl_setopt($ch, CURLOPT_VERBOSE, 1); //curl_setopt($ch, CURLOPT_URL, $_SERVER['argv'][1]); curl_setopt($ch, CURLOPT_URL, 'https://localhost:8443/index.html'); curl_setopt($ch, CURLOPT_HTTP_VERSION, 3); curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0); curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0); curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); curl_multi_add_handle($mh, $ch); $active = null; do { $status = curl_multi_exec($mh, $active); do { $info = curl_multi_info_read($mh); if (false !== $info && $info['msg'] == CURLMSG_DONE) { $handle = $info['handle']; if ($handle !== null) { $transfers--; $out = curl_multi_getcontent($info['handle']); var_dump($out); curl_multi_remove_handle($mh, $handle);
/** * Run all queued requests * * @return void */ public function execute() { $master = curl_multi_init(); foreach ($this->multicurlOptions as $multiOption => $multiValue) { curl_multi_setopt($master, $multiOption, $multiValue); } // start the first batch of requests $firstBatch = $this->getNextPendingRequests($this->getSimultaneousLimit()); // what a silly "error" if (count($firstBatch) == 0) { return; } foreach ($firstBatch as $request) { // setup the curl request, queue it up, and put it in the active array $ch = curl_init(); $options = $this->prepareRequestOptions($request); curl_setopt_array($ch, $options); curl_multi_add_handle($master, $ch); $this->activeRequests[(int) $ch] = $request; } $active = null; // Use a shorter select timeout when there is something to do between calls $idleCallback = $this->idleCallback; $selectTimeout = $idleCallback ? 8.0 : 10.0; do { // ensure we're running $status = curl_multi_exec($master, $active); // see if there is anything to read while ($transfer = curl_multi_info_read($master)) { // get the request object back and put the curl response into it $key = (int) $transfer['handle']; $request = $this->activeRequests[$key]; $request->setResponseText(curl_multi_getcontent($transfer['handle'])); $request->setResponseErrno(curl_errno($transfer['handle'])); $request->setResponseError(curl_error($transfer['handle'])); $request->setResponseInfo(curl_getinfo($transfer['handle'])); // remove the request from the list of active requests unset($this->activeRequests[$key]); // move the request to the completed set $this->completedRequests[] = $request; $this->completedRequestCount++; // start a new request (it's important to do this before removing the old one) if ($nextRequest = $this->getNextPendingRequest()) { // setup the curl request, queue it up, and put it in the active array $ch = curl_init(); $options = $this->prepareRequestOptions($nextRequest); curl_setopt_array($ch, $options); curl_multi_add_handle($master, $ch); $this->activeRequests[(int) $ch] = $nextRequest; } // remove the curl handle that just completed curl_multi_remove_handle($master, $transfer['handle']); // if there is a callback, run it if (is_callable($this->callback)) { $callback = $this->callback; $callback($request, $this); } // if something was requeued, this will get it running/update our loop check values $status = curl_multi_exec($master, $active); } // Error detection -- this is very, very rare $err = null; switch ($status) { case CURLM_BAD_EASY_HANDLE: $err = 'CURLM_BAD_EASY_HANDLE'; break; case CURLM_OUT_OF_MEMORY: $err = 'CURLM_OUT_OF_MEMORY'; break; case CURLM_INTERNAL_ERROR: $err = 'CURLM_INTERNAL_ERROR'; break; case CURLM_BAD_HANDLE: $err = 'CURLM_BAD_HANDLE'; break; } if ($err) { throw new \Exception("curl_multi_exec failed with error code ({$status}) const ({$err})"); } // Block until *something* happens to avoid burning CPU cycles for naught while (0 == curl_multi_select($master, $selectTimeout) && $idleCallback) { $idleCallback($this); } // see if we're done yet or not } while ($status === CURLM_CALL_MULTI_PERFORM || $active); curl_multi_close($master); }
function GetNextCharacter(&$characterNames) { global $db, $caughtKill, $ownerRealms; $sellerRealms = array_keys($characterNames); do { $sellerRealm = array_pop($sellerRealms); if (count($characterNames[$sellerRealm]) == 0) { unset($characterNames[$sellerRealm]); $sellerRealm = '~'; } } while (!isset($ownerRealms[$sellerRealm]) && count($sellerRealms)); if (!isset($ownerRealms[$sellerRealm]) && count($sellerRealms) == 0) { if (count($characterNames)) { DebugMessage('The following realms were not matched:' . "\n\"" . implode('", "', array_keys($characterNames)) . "\"\n against \"" . implode('", "', array_keys($ownerRealms)) . '"', E_USER_WARNING); } $characterNames = []; return; } unset($sellerRealms); $realmRow = $ownerRealms[$sellerRealm]; $charsToFetch = []; while (count($charsToFetch) < 5 && count($characterNames[$sellerRealm])) { reset($characterNames[$sellerRealm]); $character = key($characterNames[$sellerRealm]); unset($characterNames[$sellerRealm][$character]); $c = 0; $stmt = $db->prepare('select count(*) from tblCharacter where name=? and realm=? and scanned > timestampadd(week, -1, now())'); $stmt->bind_param('si', $character, $realmRow['id']); $stmt->execute(); $stmt->bind_result($c); $stmt->fetch(); $stmt->close(); if ($c == 0) { $charsToFetch[] = $character; } } if (count($charsToFetch) == 0) { DebugMessage("No more characters on {$realmRow['name']} ({$totalChars} remaining)"); return; } $guildsToFetch = []; $totalChars = 0; foreach (array_keys($characterNames) as $k) { $totalChars += count($characterNames[$k]); } DebugMessage("Getting characters " . implode(',', $charsToFetch) . " on {$realmRow['name']} ({$totalChars} remaining)"); $curlOpts = [CURLOPT_RETURNTRANSFER => true, CURLOPT_FOLLOWLOCATION => true, CURLOPT_MAXREDIRS => 2, CURLOPT_TIMEOUT => 10, CURLOPT_ENCODING => 'gzip']; $mh = curl_multi_init(); curl_multi_setopt($mh, CURLMOPT_PIPELINING, 3); $curls = []; foreach ($charsToFetch as $character) { $curls[$character] = curl_init(GetBattleNetURL($realmRow['region'], "wow/character/" . $realmRow['slug'] . "/" . rawurlencode($character) . "?fields=guild")); curl_setopt_array($curls[$character], $curlOpts); curl_multi_add_handle($mh, $curls[$character]); } $active = false; while (CURLM_CALL_MULTI_PERFORM == ($mrc = curl_multi_exec($mh, $active))) { } while ($active && $mrc == CURLM_OK) { if (curl_multi_select($mh) != -1) { while (CURLM_CALL_MULTI_PERFORM == ($mrc = curl_multi_exec($mh, $active))) { } } usleep(100000); } foreach ($charsToFetch as $character) { curl_multi_remove_handle($mh, $curls[$character]); $dta = json_decode(curl_multi_getcontent($curls[$character]), true); if (json_last_error() != JSON_ERROR_NONE || !isset($dta['name'])) { $stmt = $db->prepare('insert into tblCharacter (name, realm, scanned) values (?, ?, NOW()) on duplicate key update scanned=values(scanned), lastmodified=null'); $stmt->bind_param('si', $character, $realmRow['id']); $stmt->execute(); $stmt->close(); } else { $dta['gender']++; // line up with db enum $stmt = $db->prepare('insert into tblCharacter (name, realm, scanned, race, class, gender, level) values (?, ?, NOW(), ?, ?, ?, ?) on duplicate key update scanned=values(scanned), race=values(race), class=values(class), gender=values(gender), level=values(level), lastmodified=null'); $stmt->bind_param('siiiii', $dta['name'], $realmRow['id'], $dta['race'], $dta['class'], $dta['gender'], $dta['level']); $stmt->execute(); $stmt->close(); if (isset($dta['guild']) && isset($dta['guild']['name']) && $dta['guild']['name']) { $guildsToFetch[md5($dta['guild']['name']) . md5($dta['guild']['realm'])] = [$dta['guild']['name'], $dta['guild']['realm']]; } } curl_close($curls[$character]); } curl_multi_close($mh); foreach ($guildsToFetch as $params) { heartbeat(); if ($caughtKill) { return; } GetGuild($characterNames, $params[0], $params[1]); } }
function FetchURLBatch($urls, $curlOpts = []) { if (!$urls) { return []; } global $connectionTracking; $curlOpts = [CURLOPT_RETURNTRANSFER => true, CURLOPT_FOLLOWLOCATION => true, CURLOPT_MAXREDIRS => 2, CURLOPT_TIMEOUT => 10, CURLOPT_ENCODING => 'gzip'] + $curlOpts; static $mh = false; if ($mh === false) { $mh = curl_multi_init(); // old curl forces pipelining on one connection if we ask for it and the server supports it // this is slower than just opening multiple connections like we want to with curl_multi // also, old curl doesn't interpret the http2 flag properly, and thinks we want pipelining if we just set "2" here curl_multi_setopt($mh, CURLMOPT_PIPELINING, 0); } $results = []; $curls = []; foreach ($urls as $k => $url) { $curls[$k] = curl_init($url); curl_setopt_array($curls[$k], $curlOpts); curl_multi_add_handle($mh, $curls[$k]); } $active = false; do { while (CURLM_CALL_MULTI_PERFORM == ($mrc = curl_multi_exec($mh, $active))) { } if ($active) { usleep(100000); } } while ($active && $mrc == CURLM_OK); foreach ($urls as $k => $url) { $results[$k] = curl_multi_getcontent($curls[$k]); $connectionTracking['created'] += curl_getinfo($curls[$k], CURLINFO_NUM_CONNECTS); $connectionTracking['requests']++; curl_multi_remove_handle($mh, $curls[$k]); curl_close($curls[$k]); } return $results; }
function download_chunks($chunklist, $effective_url) { global $db, $prefix, $filetime; $mh = curl_multi_init(); $connections = array(); $files = array(); curl_multi_setopt($mh, CURLMOPT_PIPELINING, 1); curl_multi_setopt($mh, CURLMOPT_MAXCONNECTS, 1); foreach ($chunklist as $chunk) { if (db_is_file_downloaded($db, $chunk["filename"])) { // this file is already downloaded, skip downloading l("skip downloading: : " . $chunk["filename"]); // fill in the corresponding array elements to make these arrays synchronized with $chunklist $connections[] = null; $files[] = null; continue; } l("going to download: " . $chunk["filename"]); $url = str_replace("playlist.m3u8", $chunk["filename"], $effective_url); $c = new_curl($url); $f = fopen(DATA_DIR . $chunk["filename"], "w"); curl_setopt($c, CURLOPT_FILE, $f); curl_multi_add_handle($mh, $c); $connections[] = $c; $files[] = $f; } //execute the handles $still_running; do { $mrc = curl_multi_exec($mh, $still_running); curl_multi_select($mh, 5); } while ($still_running > 0); l("all downloads finished"); $t = $filetime; for ($i = 0; $i < count($connections); $i++) { $c = $connections[$i]; $chunk = $chunklist[$i]; if ($c) { curl_multi_remove_handle($mh, $c); $status = curl_getinfo($c, CURLINFO_HTTP_CODE); if ($status == 200) { db_add_file($db, $t, $chunk["filename"], $chunk["duration"]); } else { l("chunk download failed, status: " . $status . ", filename: " . $chunk["filename"]); } $t += $chunk["duration"]; } } $filetime = $t; foreach ($files as $f) { if ($f) { fclose($f); } } curl_multi_close($mh); db_save($db); }
<?php $callback = function ($data) { $args = func_get_args(); var_dump($args); }; $mh = curl_multi_init(); curl_multi_setopt($mh, CURLMOPT_PUSHFUNCTION, $callback); $data = []; curl_multi_setopt($mh, CURLMOPT_PUSHDATA, $data); $ch = curl_init(); curl_setopt($ch, CURLOPT_URL, 'http://0.0.0.0:8080/curl'); curl_setopt($ch, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0); curl_multi_add_handle($mh, $ch); $active = null; //execute the handles do { $mrc = curl_multi_exec($mh, $active); } while ($mrc == CURLM_CALL_MULTI_PERFORM); while ($active && $mrc == CURLM_OK) { if (curl_multi_select($mh) != -1) { do { $mrc = curl_multi_exec($mh, $active); } while ($mrc == CURLM_CALL_MULTI_PERFORM); } } curl_multi_close($mh);
/** * @see curl_multi_setopt * @param $option * @param $value * @return bool */ public function setMultiCurlOpt($option, $value) { return curl_multi_setopt($this->multiCurlHandle, $option, $value); }
/** * Starts a session by sending out the added requests. * * @param reference $success **OPTIONAL. OUTPUT.** After the method is called with this parameter provided, the * parameter's value tells whether the session was successful. * * @return void */ public function start(&$success = null) { $success = true; if ($this->m_hasError) { $success = false; return; } if (CArray::isEmpty($this->m_requestRecordsQueue)) { // Nothing to do. return; } // Current policy is to disable HTTP pipelining. $res = curl_multi_setopt($this->m_multiCurl, CURLMOPT_PIPELINING, 0); if (!$res) { // Should never get in here as long as cURL options are being set correctly, hence the assertion. assert('false', vs(isset($this), get_defined_vars())); $this->m_hasError = true; $this->m_errorMessage = "The 'curl_multi_setopt' function failed."; $success = false; $this->finalize(); return; } $anySuccessfulRequests = false; // Disable the script's execution timeout before getting into the session. $timeoutPause = new CTimeoutPause(); $numRunningRequests = 0; // also the index of the next request to send while (true) { // From the request queue, add as many normal cURL handles to the multi cURL handle as it is allowed by the // maximum number of concurrent requests, priorly setting internal options for every request. while ($numRunningRequests < CArray::length($this->m_requestRecordsQueue) && $numRunningRequests < $this->m_maxNumConcurrentRequests) { $requestRecord = $this->m_requestRecordsQueue[$numRunningRequests]; $request = $requestRecord[0]; $onCompleteCallback = $requestRecord[1]; $newCookieSession = $requestRecord[2]; $requestCurl = $request->curl(); // Set cURL options for the normal cURL handle, having created a temporary file for cookie storage if // needed. $requestSetOptSuccess; if ($this->m_cookiesAreEnabled && $request->isHttp()) { if (!isset($this->m_cookiesFp)) { $this->m_cookiesFp = CFile::createTemporary(); } $request->setInternalOptions($requestSetOptSuccess, $this->m_cookiesFp, $newCookieSession); } else { $request->setInternalOptions($requestSetOptSuccess); } if (!$requestSetOptSuccess) { if (isset($onCompleteCallback)) { call_user_func($onCompleteCallback, false, "", $request, $this); } CArray::remove($this->m_requestRecordsQueue, $numRunningRequests); continue; } // Add the normal cURL handle to the multi cURL handle. $res = curl_multi_add_handle($this->m_multiCurl, $requestCurl); if ($res != 0) { $this->m_hasError = true; $curlError = curl_multi_strerror($res); $this->m_errorMessage = is_cstring($curlError) && !CString::isEmpty($curlError) ? $curlError : "The 'curl_multi_add_handle' function failed."; $success = false; $timeoutPause->end(); $this->finalize(); return; } $numRunningRequests++; } if ($numRunningRequests == 0) { break; } // Process the currently added requests until complete or no more data is available. Although // `CURLM_CALL_MULTI_PERFORM` is deprecated since libcurl 7.20, keep it for compatibility reasons. $numRunningTransfers; do { $multiExecRes = curl_multi_exec($this->m_multiCurl, $numRunningTransfers); } while ($multiExecRes == CURLM_CALL_MULTI_PERFORM); if ($multiExecRes != CURLM_OK) { $this->m_hasError = true; $curlError = curl_multi_strerror($multiExecRes); $this->m_errorMessage = is_cstring($curlError) && !CString::isEmpty($curlError) ? $curlError : "The 'curl_multi_exec' function failed."; $success = false; $timeoutPause->end(); $this->finalize(); return; } // Check for completed requests, call the callback function for any completed one (if such a function is // defined), finalize completed requests, and remove completed requests from the queue. while (true) { $completedRequestInfo = curl_multi_info_read($this->m_multiCurl); if (!is_cmap($completedRequestInfo)) { break; } // A request has completed. assert('$completedRequestInfo["msg"] == CURLMSG_DONE', vs(isset($this), get_defined_vars())); $requestCurl = $completedRequestInfo["handle"]; $requestRes = $completedRequestInfo["result"]; $requestRecordPos; $found = CArray::find($this->m_requestRecordsQueue, $requestCurl, function ($requestRecord, $requestCurl) { $request = $requestRecord[0]; return $request->curl() == $requestCurl; }, $requestRecordPos); assert('$found', vs(isset($this), get_defined_vars())); $requestRecord = $this->m_requestRecordsQueue[$requestRecordPos]; $request = $requestRecord[0]; $onCompleteCallback = $requestRecord[1]; // Remove the normal cURL handle from the multi cURL handle. $res = curl_multi_remove_handle($this->m_multiCurl, $requestCurl); if ($res != 0) { $this->m_hasError = true; $curlError = curl_multi_strerror($res); $this->m_errorMessage = is_cstring($curlError) && !CString::isEmpty($curlError) ? $curlError : "The 'curl_multi_remove_handle' function failed."; $success = false; $timeoutPause->end(); $this->finalize(); return; } if ($requestRes == CURLE_OK) { // The request has succeeded. if (isset($onCompleteCallback)) { $response; if ($request->isReturnTransferSet()) { $response = curl_multi_getcontent($requestCurl); assert('is_cstring($response)', vs(isset($this), get_defined_vars())); } else { $response = ""; } $request->onRequestCompleteOk(); // also close the normal cURL handle call_user_func($onCompleteCallback, true, $response, $request, $this); } else { $request->onRequestCompleteOk(); // also close the normal cURL handle } $anySuccessfulRequests = true; } else { // The request has failed. $curlError = curl_strerror($requestRes); if (!is_cstring($curlError)) { $curlError = ""; } $request->onRequestCompleteWithError($curlError); // also close the normal cURL handle if (isset($onCompleteCallback)) { call_user_func($onCompleteCallback, false, "", $request, $this); } } CArray::remove($this->m_requestRecordsQueue, $requestRecordPos); $numRunningRequests--; } assert('$numRunningRequests == $numRunningTransfers', vs(isset($this), get_defined_vars())); if ($numRunningTransfers > 0) { // Some requests are still being processed (by remote machines). Wait for more data to appear on // sockets, without getting hard on the CPU. do { $multiSelectRes = curl_multi_select($this->m_multiCurl); } while ($multiSelectRes == -1); } else { // No requests are being processed. Check if any requests are pending. if (CArray::isEmpty($this->m_requestRecordsQueue)) { // No requests are pending. break; } } } // Set the script's execution time limit like the session has never happened. $timeoutPause->end(); if (!$anySuccessfulRequests) { $this->m_hasError = true; $this->m_errorMessage = "None of the session's requests succeeded."; $success = false; } $this->finalize(); }
/** * @see curl_multi_setopt * * @param int $opt * @param mixed $val * @return boolean */ public function setOpt($opt, $val) { return curl_multi_setopt($this->handle, $opt, $val); }
/** * Constructor * * @link http://www.php.net/manual/en/function.curl-init.php curl_init() * @link http://www.php.net/manual/en/function.curl-multi-init.php curl_multi_init() * * @throws \RuntimeException If curl_init() or curl_multi_init() fails. * @throws \BLW\Model\InvalidArgumentException If <code>$Handles</code> is not an integer. * * @param integer $Handles * Maximum handles to create for parralell operations. */ public function __construct($Handles) { // Validate $Handles if (!is_int($Handles)) { throw new InvalidArgumentException(0); } // curl_multi handle $this->MainHandle = curl_multi_init(); // @codeCoverageIgnoreStart // Check results if (!is_resource($this->MainHandle)) { throw new RuntimeException('Unable to initialize cURL multi session'); } // Enable pipelining if (is_callable('curl_multi_setopt')) { curl_multi_setopt($this->MainHandle, CURLMOPT_PIPELINING, 1); } // @codeCoverageIgnoreEnd // curl_init handles foreach (range(0, $Handles) as $i) { // Create handle $this->Handles[$i] = curl_init(); // Check result. Add handle to free handles. if (is_resource($this->Handles[$i])) { $this->FreeHandles[] = $this->Handles[$i]; // @codeCoverageIgnoreStart // Unable to create handle? Exception. } else { throw new RuntimeException('Unable to initialize cURL session'); } // @codeCoverageIgnoreEnd } }