/**
* @return resource
*/
protected function getCurlMulti()
{
if (!$this->multiHandle) {
$cmh = curl_multi_init();
if (function_exists('curl_multi_setopt')) {
// PHP 5.5
curl_multi_setopt($cmh, CURLMOPT_PIPELINING, (int) $this->usePipelining);
curl_multi_setopt($cmh, CURLMOPT_MAXCONNECTS, (int) $this->maxConnsPerHost);
}
$this->multiHandle = $cmh;
}
return $this->multiHandle;
}
public function execute()
{
$mh = curl_multi_init();
$chs = $this->chs();
$stillRunning = 0;
$result = array();
if (function_exists('curl_multi_setopt')) {
curl_multi_setopt($mh, CURLMOPT_PIPELINING, 1);
curl_multi_setopt($mh, CURLMOPT_MAXCONNECTS, $this->maxConnectsSize());
}
foreach ($chs as $ch) {
curl_multi_add_handle($mh, $ch);
}
do {
$execReturnValue = curl_multi_exec($mh, $stillRunning);
curl_multi_select($mh);
} while ($stillRunning > 0);
foreach ($chs as $i => $ch) {
$curlError = curl_error($ch);
if ($curlError === "") {
$result[$i] = curl_multi_getcontent($ch);
} else {
throw new PubnubException("Curl error on handle {$i}: {$curlError}\n");
}
curl_multi_remove_handle($mh, $ch);
curl_close($ch);
}
curl_multi_close($mh);
$this->requests = array();
if ($execReturnValue != CURLM_OK) {
throw new PubnubException(curl_multi_strerror($execReturnValue));
}
return $result;
}
/**
* Set cURL option
*
* @param array $options
*
* @throws GrasshopperException
*/
public function setOptions($options)
{
foreach ($options as $key => $value) {
$res = curl_multi_setopt($this->mh, $key, $value);
if (!$res) {
throw new GrasshopperException('curl_setopt_array failed', Grasshopper::ERROR_M_SETOPTIONS);
}
}
}
/**
* Constructor.
* Initialize cURL multi handle.
* @param CoOption $options
*/
public function __construct(CoOption $options)
{
$this->mh = curl_multi_init();
$flags = (int) $options['pipeline'] + (int) $options['multiplex'] * 2;
curl_multi_setopt($this->mh, CURLMOPT_PIPELINING, $flags);
$this->options = $options;
$this->scheduler = $options['autoschedule'] ? new AutoScheduler($options, $this->mh) : new ManualScheduler($options, $this->mh);
$this->delayer = new Delayer();
}
/**
* Class Constructor, init the multi curl handler
*
*/
public function __construct()
{
$this->multi_handler = curl_multi_init();
Utils::getPHPVersion();
if (PHP_MINOR_VERSION >= 5) {
// curl_multi_setopt only for (PHP 5 >= 5.5.0) Default 10
curl_multi_setopt($this->multi_handler, CURLMOPT_MAXCONNECTS, 50);
}
}
/**
* @return resource
*/
private function handlerMulti()
{
if (!$this->_pool_master) {
$this->_pool_master = curl_multi_init();
if (function_exists('curl_multi_setopt')) {
curl_multi_setopt($this->_pool_master, CURLMOPT_MAXCONNECTS, $this->simultaneousLimit);
}
}
return $this->_pool_master;
}
public function __construct($options = null)
{
$this->options = $options;
$curl_opts = null;
$this->curl_opts = $curl_opts ? $curl_opts : [];
// TODO: cURL "eventization" with libevent/libev/libuv and integration with AsyncSteps loop
$this->curl_mh = curl_multi_init();
# Missing on HHVM
if (function_exists('curl_multi_setopt')) {
curl_multi_setopt($this->curl_mh, CURLMOPT_PIPELINING, 1);
curl_multi_setopt($this->curl_mh, CURLMOPT_MAXCONNECTS, isset($curl_opts[CURLOPT_MAXCONNECTS]) ? $curl_opts[CURLOPT_MAXCONNECTS] : 8);
}
}
public function setOption($options, $value = null)
{
if (is_int($options) && $value !== null) {
curl_multi_setopt($this->_mh, $options, $value);
} elseif (is_array($options)) {
foreach ($options as $option => $value) {
$this->setOption($option, $value);
}
} else {
throw new \InvalidArgumentException('$option type must be int or array');
}
return $this;
}
/**
* Set an option for the cURL multi handle
*
* @param int $option One of the CURLMOPT_* constants.
* @param mixed $value The value to be set on option.
* @return self
*/
public final function curlMultiSetOpt($option, $value)
{
curl_multi_setopt($this->curl_multi_handle, $option, $value);
return $this;
}
/**
* @param $curlHandles
* @throws RoutingMachineException
* @return array
*/
private function runMultiCurlsBlockWise($curlHandles)
{
//we run curl_multi only on a BLOCKSIZE so we don't generate a flood
$curlMultiHandle = curl_multi_init();
curl_multi_setopt($curlMultiHandle, CURLMOPT_PIPELINING, 1);
curl_multi_setopt($curlMultiHandle, CURLMOPT_MAXCONNECTS, self::BLOCK_SIZE);
$responses = array();
$blockCount = 0;
// count where we are in the list so we can break up the runs into smaller blocks
$blockResults = array();
// to accumulate the curl_handles for each group we'll run simultaneously
foreach ($curlHandles as $hashKey => $curlHandle) {
$blockCount++;
// add the handle to the curl_multi_handle and to our tracking "block"
curl_multi_add_handle($curlMultiHandle, $curlHandle);
$blockResults[$hashKey] = $curlHandle;
if ($blockCount % self::BLOCK_SIZE === 0 or $blockCount === count($curlHandles)) {
$running = NULL;
do {
// track the previous loop's number of handles still running so we can tell if it changes
$runningBefore = $running;
// run the block or check on the running block and get the number of sites still running in $running
curl_multi_exec($curlMultiHandle, $running);
// if the number of sites still running changed, print out a message with the number of sites that are still running.
if ($running != $runningBefore) {
//waiting for running sites to finish
}
} while ($running > 0);
//once the number still running is 0, curl_multi_ is done, so check the results
foreach ($blockResults as $hashKeyBlock => $result) {
// HTTP response code
$code = curl_getinfo($result, CURLINFO_HTTP_CODE);
// cURL error number
$curlErrno = curl_errno($result);
// cURL error message
$curlError = curl_error($result);
// output if there was an error
if ($curlError) {
throw new RoutingMachineException("OSRM cURL error: ({$curlErrno}) {$curlError}\n");
}
// fill results from our requests we get
$response = curl_multi_getcontent($result);
$responses[$hashKeyBlock] = $response;
// remove the (used) handle from the curl_multi_handle
curl_multi_remove_handle($curlMultiHandle, $result);
}
// reset the block to empty, since we've run its curl_handles
$blockResults = array();
}
}
// close the curl_multi_handle once we're done
curl_multi_close($curlMultiHandle);
return $responses;
}
/**
* @param Package\PackageInterface[] $packages
* @param array $pluginConfig
* @return void
*/
public function download(array $packages, array $pluginConfig)
{
$mh = curl_multi_init();
$unused = array();
$maxConns = $pluginConfig['maxConnections'];
for ($i = 0; $i < $maxConns; ++$i) {
$unused[] = curl_init();
}
/// @codeCoverageIgnoreStart
if (function_exists('curl_share_init')) {
$sh = curl_share_init();
curl_share_setopt($sh, CURLSHOPT_SHARE, CURL_LOCK_DATA_SSL_SESSION);
foreach ($unused as $ch) {
curl_setopt($ch, CURLOPT_SHARE, $sh);
}
}
if (function_exists('curl_multi_setopt')) {
if ($pluginConfig['pipeline']) {
curl_multi_setopt($mh, CURLMOPT_PIPELINING, true);
}
}
/// @codeCoverageIgnoreEnd
$cachedir = rtrim($this->config->get('cache-files-dir'), '\\/');
$chFpMap = array();
$running = 0;
//ref type
$remains = 0;
//ref type
$this->totalCnt = count($packages);
$this->successCnt = 0;
$this->failureCnt = 0;
$this->io->write(" Prefetch start: <comment>success: {$this->successCnt}, failure: {$this->failureCnt}, total: {$this->totalCnt}</comment>");
do {
// prepare curl resources
while ($unused && $packages) {
$package = array_pop($packages);
$filepath = $cachedir . DIRECTORY_SEPARATOR . static::getCacheKey($package);
if (file_exists($filepath)) {
++$this->successCnt;
continue;
}
$ch = array_pop($unused);
// make file resource
$fp = CurlRemoteFilesystem::createFile($filepath);
$chFpMap[(int) $ch] = compact('fp', 'filepath');
// make url
$url = $package->getDistUrl();
$request = new Aspects\HttpGetRequest(parse_url($url, PHP_URL_HOST), $url, $this->io);
$request->verbose = $pluginConfig['verbose'];
if (in_array($package->getName(), $pluginConfig['privatePackages'])) {
$request->maybePublic = false;
} else {
$request->maybePublic = preg_match('%^(?:https|git)://github\\.com%', $package->getSourceUrl());
}
$onPreDownload = Factory::getPreEvent($request);
$onPreDownload->notify();
$opts = $request->getCurlOpts();
unset($opts[CURLOPT_ENCODING]);
unset($opts[CURLOPT_USERPWD]);
curl_setopt_array($ch, $opts);
curl_setopt($ch, CURLOPT_FILE, $fp);
curl_multi_add_handle($mh, $ch);
}
// start multi download
do {
$stat = curl_multi_exec($mh, $running);
} while ($stat === CURLM_CALL_MULTI_PERFORM);
// wait for any event
do {
switch (curl_multi_select($mh, 5)) {
case -1:
usleep(10);
do {
$stat = curl_multi_exec($mh, $running);
} while ($stat === CURLM_CALL_MULTI_PERFORM);
continue 2;
case 0:
continue 2;
default:
do {
$stat = curl_multi_exec($mh, $running);
} while ($stat === CURLM_CALL_MULTI_PERFORM);
do {
if ($raised = curl_multi_info_read($mh, $remains)) {
$ch = $raised['handle'];
$errno = curl_errno($ch);
$info = curl_getinfo($ch);
curl_setopt($ch, CURLOPT_FILE, STDOUT);
$index = (int) $ch;
$fileinfo = $chFpMap[$index];
unset($chFpMap[$index]);
$fp = $fileinfo['fp'];
$filepath = $fileinfo['filepath'];
fclose($fp);
if (CURLE_OK === $errno && 200 === $info['http_code']) {
++$this->successCnt;
} else {
++$this->failureCnt;
unlink($filepath);
}
$this->io->write($this->makeDownloadingText($info['url']));
curl_multi_remove_handle($mh, $ch);
$unused[] = $ch;
}
} while ($remains);
if ($packages) {
break 2;
}
}
} while ($running);
} while ($packages);
$this->io->write(" Finished: <comment>success: {$this->successCnt}, failure: {$this->failureCnt}, total: {$this->totalCnt}</comment>");
foreach ($unused as $ch) {
curl_close($ch);
}
curl_multi_close($mh);
}
/**
* Max request in Asynchron query
* @param $max int default:10
* @return void
*/
public function setMaxRequest($max)
{
$this->maxRequest = $max;
// PHP 5 >= 5.5.0
if (function_exists('curl_multi_setopt')) {
curl_multi_setopt($this->mh, CURLMOPT_MAXCONNECTS, $max);
}
}
/**
* Set an option for the cURL multi handle
*
* @param integer $option
* @param mixed $value
* @return boolean
*/
public function setopt($option, $value)
{
return curl_multi_setopt($this->_mh, $option, $value);
}
<?php $mh = curl_multi_init(); var_dump(curl_multi_setopt($mh, CURLMOPT_PIPELINING, 0)); var_dump(curl_multi_setopt($mh, -1, 0));
/**
* @since 1.0
*
* @param string $name
* @param mixed $value
*/
public function setOption($name, $value)
{
$this->options[$name] = $value;
curl_multi_setopt($this->handle, $name, $value);
}
/**
* Constructor.
* Initialize cURL multi handle.
* @param CoOption $options
* @param resource $mh curl_multi
*/
public function __construct(CoOption $options, $mh)
{
curl_multi_setopt($mh, CURLMOPT_MAX_TOTAL_CONNECTIONS, $options['concurrency']);
$this->mh = $mh;
$this->options = $options;
}
/**
* Get (and create if required) the Curl Multi handle.
*
* @return Resource $curl_multi_handle a Curl Multi handle as might be returned by curl_multi_init()
**/
private function getCurlMultiHandle()
{
if (!isset($this->curl_multi_handle)) {
$this->curl_multi_handle = curl_multi_init();
if (function_exists('curl_multi_setopt')) {
// More efficient if this is available; not an error if not.
curl_multi_setopt($this->curl_multi_handle, CURLMOPT_PIPELINING, true);
}
}
return $this->curl_multi_handle;
}
public function _setMultiOption()
{
if ((double) phpversion() >= 5.5) {
foreach ($this->multiOptions as $optKey => $optValue) {
printf('%s => %s', $optKey, $optValue);
curl_multi_setopt($this->mh, $optKey, $optValue);
}
}
}
/**
* @param Package\PackageInterface[] $packages
* @param array $pluginConfig
* @return void
*/
public function download(array $packages, array $pluginConfig)
{
$mh = curl_multi_init();
$unused = array();
$maxConns = $pluginConfig['maxConnections'];
for ($i = 0; $i < $maxConns; ++$i) {
$unused[] = curl_init();
}
// @codeCoverageIgnoreStart
if (function_exists('curl_share_init')) {
$sh = curl_share_init();
curl_share_setopt($sh, CURLSHOPT_SHARE, CURL_LOCK_DATA_SSL_SESSION);
foreach ($unused as $ch) {
curl_setopt($ch, CURLOPT_SHARE, $sh);
}
}
if (function_exists('curl_multi_setopt')) {
if ($pluginConfig['pipeline']) {
curl_multi_setopt($mh, CURLMOPT_PIPELINING, true);
}
}
// @codeCoverageIgnoreEnd
$cachedir = rtrim($this->config->get('cache-files-dir'), '\\/');
$chFpMap = array();
$running = 0;
//ref type
$remains = 0;
//ref type
$this->totalCnt = count($packages);
$this->successCnt = 0;
$this->failureCnt = 0;
$this->io->write(" Prefetch start: <comment>success: {$this->successCnt}, failure: {$this->failureCnt}, total: {$this->totalCnt}</comment>");
EVENTLOOP:
// prepare curl resources
while (count($unused) > 0 && count($packages) > 0) {
$package = array_pop($packages);
$filepath = $cachedir . DIRECTORY_SEPARATOR . static::getCacheKey($package);
if (file_exists($filepath)) {
++$this->successCnt;
continue;
}
$ch = array_pop($unused);
// make file resource
$chFpMap[(int) $ch] = $outputFile = new OutputFile($filepath);
// make url
$url = $package->getDistUrls();
if (count($url) > 0) {
$url = $url[0];
} else {
$url = $package->getDistUrl();
}
$host = parse_url($url, PHP_URL_HOST) ?: '';
$request = new Aspects\HttpGetRequest($host, $url, $this->io);
$request->verbose = $pluginConfig['verbose'];
if (in_array($package->getName(), $pluginConfig['privatePackages'])) {
$request->maybePublic = false;
} else {
$request->maybePublic = (bool) preg_match('%^(?:https|git)://github\\.com%', $package->getSourceUrl());
}
$onPreDownload = Factory::getPreEvent($request);
$onPreDownload->notify();
$opts = $request->getCurlOpts();
if ($pluginConfig['insecure']) {
$opts[CURLOPT_SSL_VERIFYPEER] = false;
}
if (!empty($pluginConfig['userAgent'])) {
$opts[CURLOPT_USERAGENT] = $pluginConfig['userAgent'];
}
if (!empty($pluginConfig['capath'])) {
$opts[CURLOPT_CAPATH] = $pluginConfig['capath'];
}
unset($opts[CURLOPT_ENCODING]);
unset($opts[CURLOPT_USERPWD]);
// ParallelDownloader doesn't support private packages.
curl_setopt_array($ch, $opts);
curl_setopt($ch, CURLOPT_FILE, $outputFile->getPointer());
curl_multi_add_handle($mh, $ch);
}
// wait for any event
do {
$runningBefore = $running;
while (CURLM_CALL_MULTI_PERFORM === curl_multi_exec($mh, $running)) {
}
SELECT:
$eventCount = curl_multi_select($mh, 5);
if ($eventCount === -1) {
usleep(200 * 1000);
continue;
}
if ($eventCount === 0) {
continue;
}
while (CURLM_CALL_MULTI_PERFORM === curl_multi_exec($mh, $running)) {
}
if ($running > 0 && $running === $runningBefore) {
goto SELECT;
}
do {
if ($raised = curl_multi_info_read($mh, $remains)) {
$ch = $raised['handle'];
$errno = curl_errno($ch);
$info = curl_getinfo($ch);
curl_setopt($ch, CURLOPT_FILE, STDOUT);
$index = (int) $ch;
$outputFile = $chFpMap[$index];
unset($chFpMap[$index]);
if (CURLE_OK === $errno && 200 === $info['http_code']) {
++$this->successCnt;
} else {
++$this->failureCnt;
$outputFile->setFailure();
}
unset($outputFile);
$this->io->write($this->makeDownloadingText($info['url']));
curl_multi_remove_handle($mh, $ch);
$unused[] = $ch;
}
} while ($remains > 0);
if (count($packages) > 0) {
goto EVENTLOOP;
}
} while ($running > 0);
$this->io->write(" Finished: <comment>success: {$this->successCnt}, failure: {$this->failureCnt}, total: {$this->totalCnt}</comment>");
foreach ($unused as $ch) {
curl_close($ch);
}
curl_multi_close($mh);
}
/**
* @param RequestsCollection|RequestInterface $requests
*
* @return ResponsesCollection
* @throws Exception
*/
public function exec($requests)
{
if ($requests instanceof RequestInterface) {
$request = $requests;
$requests = new RequestsCollection();
$requests->add($request);
}
if (false == $requests instanceof RequestsCollection) {
throw new Exception('Invalid requests! Param $requests must be an instance of ' . 'RequestInterface or a RequestsCollection.');
}
$curlMultiHandler = curl_multi_init();
if (true == $this->usePipelining) {
curl_multi_setopt($curlMultiHandler, CURLMOPT_PIPELINING, 1);
}
$curlHandlers = [];
$totalRequest = count($requests);
$auxCtr = 0;
$this->logger->debug(sprintf('Adding curl handlers for %s request...', $totalRequest));
/** @var RequestInterface $request */
foreach ($requests as $request) {
$auxCtr++;
$requestCurlHandler = new PhpCurlHandler();
$requestCurlHandler->setOption(CURLOPT_URL, $request->getUrl());
$requestCurlHandler->setOption(CURLOPT_RETURNTRANSFER, true);
$requestCurlHandler->setOption(CURLOPT_FOLLOWLOCATION, false);
$requestCurlHandler->setOption(CURLOPT_CUSTOMREQUEST, strtoupper($request->getRequestMethod()));
$requestCurlHandler->setOption(CURLOPT_TIMEOUT, $request->getTimeout());
$requestCurlHandler->setOption(CURLOPT_CONNECTTIMEOUT, $request->getConnectTimeout());
$requestCurlHandler->setOption(CURLOPT_SSL_VERIFYPEER, $request->peersSSLCertificateVerificationIsRequired());
// Set headers
if (true == $request->hasHeaders()) {
$requestCurlHandler->setOption(CURLOPT_HTTPHEADER, $request->getHeaders());
}
// Set request user and pass if needed
if (true == $request->hasAuth()) {
$requestCurlHandler->setOption(CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
$requestCurlHandler->setOption(CURLOPT_USERPWD, $request->getUser() . ':' . $request->getPass());
}
// Set user agent
if (true == $request->hasUserAgent()) {
$requestCurlHandler->setOption(CURLOPT_USERAGENT, $request->getUserAgent());
}
// Set request params
if (true == $request->hasParams()) {
$requestCurlHandler->setOption(CURLOPT_POST, 1);
$requestCurlHandler->setOption(CURLOPT_POSTFIELDS, $request->getEncodedParams());
}
$this->logger->debug(sprintf('Adding curl handler for request %s of %s (id:%s): [%s] %s ', $auxCtr, $totalRequest, $request->getId(), $request->getRequestMethod(), $request->getUrl()), ['params' => $request->getParams(), 'headers' => $request->getHeaders()]);
$curlHandlers[$request->getId()] = $requestCurlHandler->getHandler();
curl_multi_add_handle($curlMultiHandler, $requestCurlHandler->getHandler());
}
$this->logger->debug(sprintf('Executing curl multi request (%s requests)...', $totalRequest));
$initTime = microtime(true);
$active = null;
do {
$status = curl_multi_exec($curlMultiHandler, $active);
$info = curl_multi_info_read($curlMultiHandler);
if ($status > 0) {
throw new Exception(sprintf('Curl error "%s"!', $status));
}
if (false !== $info) {
if (isset($info['result']) && 0 < $info['result']) {
$url = '';
$errorMsg = '';
if (isset($info['handle'])) {
$errorMsg = curl_error($info['handle']);
$url = curl_getinfo($info['handle'], CURLINFO_EFFECTIVE_URL);
}
throw new Exception(sprintf('Error [%s] calling URL "%s" (%s)!', $info['result'], $url, $errorMsg));
}
}
} while ($status === CURLM_CALL_MULTI_PERFORM || $active);
$this->logger->debug(sprintf('Executed %s requests in %ss...', $totalRequest, microtime(true) - $initTime));
$responses = new ResponsesCollection();
$this->logger->debug('Composing responses...');
$auxCtr = 0;
foreach ($curlHandlers as $requestId => $curlHandler) {
$auxCtr++;
$this->logger->debug(sprintf('Composing response for request %s (%s of %s)...', $requestId, $auxCtr, $totalRequest));
$httpCode = curl_getinfo($curlHandler, CURLINFO_HTTP_CODE);
$contents = curl_multi_getcontent($curlHandler);
$this->logger->debug(sprintf('Response for request %s [code: %s]: %s', $requestId, $httpCode, $this->mutiLineStringToOneLineString($contents)));
$responses->add($requestId, $this->getResponsesFactory()->create((string) $contents, $httpCode));
// Remove handler from curl multi handler
curl_multi_remove_handle($curlMultiHandler, $curlHandler);
}
// Close curl multi handler
curl_multi_close($curlMultiHandler);
$this->logger->debug(sprintf('Total execution time: %ss)...', microtime(true) - $initTime));
return $responses;
}
<?php
$transfers = 1;
$callback = function () use(&$transfers) {
return CURL_PUSH_DENY;
$transfers++;
};
$mh = curl_multi_init();
curl_multi_setopt($mh, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
curl_multi_setopt($mh, CURLMOPT_PUSHFUNCTION, $callback);
$ch = curl_init();
curl_setopt($ch, CURLOPT_VERBOSE, 1);
//curl_setopt($ch, CURLOPT_URL, $_SERVER['argv'][1]);
curl_setopt($ch, CURLOPT_URL, 'https://localhost:8443/index.html');
curl_setopt($ch, CURLOPT_HTTP_VERSION, 3);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_multi_add_handle($mh, $ch);
$active = null;
do {
$status = curl_multi_exec($mh, $active);
do {
$info = curl_multi_info_read($mh);
if (false !== $info && $info['msg'] == CURLMSG_DONE) {
$handle = $info['handle'];
if ($handle !== null) {
$transfers--;
$out = curl_multi_getcontent($info['handle']);
var_dump($out);
curl_multi_remove_handle($mh, $handle);
/**
* Run all queued requests
*
* @return void
*/
public function execute()
{
$master = curl_multi_init();
foreach ($this->multicurlOptions as $multiOption => $multiValue) {
curl_multi_setopt($master, $multiOption, $multiValue);
}
// start the first batch of requests
$firstBatch = $this->getNextPendingRequests($this->getSimultaneousLimit());
// what a silly "error"
if (count($firstBatch) == 0) {
return;
}
foreach ($firstBatch as $request) {
// setup the curl request, queue it up, and put it in the active array
$ch = curl_init();
$options = $this->prepareRequestOptions($request);
curl_setopt_array($ch, $options);
curl_multi_add_handle($master, $ch);
$this->activeRequests[(int) $ch] = $request;
}
$active = null;
// Use a shorter select timeout when there is something to do between calls
$idleCallback = $this->idleCallback;
$selectTimeout = $idleCallback ? 8.0 : 10.0;
do {
// ensure we're running
$status = curl_multi_exec($master, $active);
// see if there is anything to read
while ($transfer = curl_multi_info_read($master)) {
// get the request object back and put the curl response into it
$key = (int) $transfer['handle'];
$request = $this->activeRequests[$key];
$request->setResponseText(curl_multi_getcontent($transfer['handle']));
$request->setResponseErrno(curl_errno($transfer['handle']));
$request->setResponseError(curl_error($transfer['handle']));
$request->setResponseInfo(curl_getinfo($transfer['handle']));
// remove the request from the list of active requests
unset($this->activeRequests[$key]);
// move the request to the completed set
$this->completedRequests[] = $request;
$this->completedRequestCount++;
// start a new request (it's important to do this before removing the old one)
if ($nextRequest = $this->getNextPendingRequest()) {
// setup the curl request, queue it up, and put it in the active array
$ch = curl_init();
$options = $this->prepareRequestOptions($nextRequest);
curl_setopt_array($ch, $options);
curl_multi_add_handle($master, $ch);
$this->activeRequests[(int) $ch] = $nextRequest;
}
// remove the curl handle that just completed
curl_multi_remove_handle($master, $transfer['handle']);
// if there is a callback, run it
if (is_callable($this->callback)) {
$callback = $this->callback;
$callback($request, $this);
}
// if something was requeued, this will get it running/update our loop check values
$status = curl_multi_exec($master, $active);
}
// Error detection -- this is very, very rare
$err = null;
switch ($status) {
case CURLM_BAD_EASY_HANDLE:
$err = 'CURLM_BAD_EASY_HANDLE';
break;
case CURLM_OUT_OF_MEMORY:
$err = 'CURLM_OUT_OF_MEMORY';
break;
case CURLM_INTERNAL_ERROR:
$err = 'CURLM_INTERNAL_ERROR';
break;
case CURLM_BAD_HANDLE:
$err = 'CURLM_BAD_HANDLE';
break;
}
if ($err) {
throw new \Exception("curl_multi_exec failed with error code ({$status}) const ({$err})");
}
// Block until *something* happens to avoid burning CPU cycles for naught
while (0 == curl_multi_select($master, $selectTimeout) && $idleCallback) {
$idleCallback($this);
}
// see if we're done yet or not
} while ($status === CURLM_CALL_MULTI_PERFORM || $active);
curl_multi_close($master);
}
function GetNextCharacter(&$characterNames)
{
global $db, $caughtKill, $ownerRealms;
$sellerRealms = array_keys($characterNames);
do {
$sellerRealm = array_pop($sellerRealms);
if (count($characterNames[$sellerRealm]) == 0) {
unset($characterNames[$sellerRealm]);
$sellerRealm = '~';
}
} while (!isset($ownerRealms[$sellerRealm]) && count($sellerRealms));
if (!isset($ownerRealms[$sellerRealm]) && count($sellerRealms) == 0) {
if (count($characterNames)) {
DebugMessage('The following realms were not matched:' . "\n\"" . implode('", "', array_keys($characterNames)) . "\"\n against \"" . implode('", "', array_keys($ownerRealms)) . '"', E_USER_WARNING);
}
$characterNames = [];
return;
}
unset($sellerRealms);
$realmRow = $ownerRealms[$sellerRealm];
$charsToFetch = [];
while (count($charsToFetch) < 5 && count($characterNames[$sellerRealm])) {
reset($characterNames[$sellerRealm]);
$character = key($characterNames[$sellerRealm]);
unset($characterNames[$sellerRealm][$character]);
$c = 0;
$stmt = $db->prepare('select count(*) from tblCharacter where name=? and realm=? and scanned > timestampadd(week, -1, now())');
$stmt->bind_param('si', $character, $realmRow['id']);
$stmt->execute();
$stmt->bind_result($c);
$stmt->fetch();
$stmt->close();
if ($c == 0) {
$charsToFetch[] = $character;
}
}
if (count($charsToFetch) == 0) {
DebugMessage("No more characters on {$realmRow['name']} ({$totalChars} remaining)");
return;
}
$guildsToFetch = [];
$totalChars = 0;
foreach (array_keys($characterNames) as $k) {
$totalChars += count($characterNames[$k]);
}
DebugMessage("Getting characters " . implode(',', $charsToFetch) . " on {$realmRow['name']} ({$totalChars} remaining)");
$curlOpts = [CURLOPT_RETURNTRANSFER => true, CURLOPT_FOLLOWLOCATION => true, CURLOPT_MAXREDIRS => 2, CURLOPT_TIMEOUT => 10, CURLOPT_ENCODING => 'gzip'];
$mh = curl_multi_init();
curl_multi_setopt($mh, CURLMOPT_PIPELINING, 3);
$curls = [];
foreach ($charsToFetch as $character) {
$curls[$character] = curl_init(GetBattleNetURL($realmRow['region'], "wow/character/" . $realmRow['slug'] . "/" . rawurlencode($character) . "?fields=guild"));
curl_setopt_array($curls[$character], $curlOpts);
curl_multi_add_handle($mh, $curls[$character]);
}
$active = false;
while (CURLM_CALL_MULTI_PERFORM == ($mrc = curl_multi_exec($mh, $active))) {
}
while ($active && $mrc == CURLM_OK) {
if (curl_multi_select($mh) != -1) {
while (CURLM_CALL_MULTI_PERFORM == ($mrc = curl_multi_exec($mh, $active))) {
}
}
usleep(100000);
}
foreach ($charsToFetch as $character) {
curl_multi_remove_handle($mh, $curls[$character]);
$dta = json_decode(curl_multi_getcontent($curls[$character]), true);
if (json_last_error() != JSON_ERROR_NONE || !isset($dta['name'])) {
$stmt = $db->prepare('insert into tblCharacter (name, realm, scanned) values (?, ?, NOW()) on duplicate key update scanned=values(scanned), lastmodified=null');
$stmt->bind_param('si', $character, $realmRow['id']);
$stmt->execute();
$stmt->close();
} else {
$dta['gender']++;
// line up with db enum
$stmt = $db->prepare('insert into tblCharacter (name, realm, scanned, race, class, gender, level) values (?, ?, NOW(), ?, ?, ?, ?) on duplicate key update scanned=values(scanned), race=values(race), class=values(class), gender=values(gender), level=values(level), lastmodified=null');
$stmt->bind_param('siiiii', $dta['name'], $realmRow['id'], $dta['race'], $dta['class'], $dta['gender'], $dta['level']);
$stmt->execute();
$stmt->close();
if (isset($dta['guild']) && isset($dta['guild']['name']) && $dta['guild']['name']) {
$guildsToFetch[md5($dta['guild']['name']) . md5($dta['guild']['realm'])] = [$dta['guild']['name'], $dta['guild']['realm']];
}
}
curl_close($curls[$character]);
}
curl_multi_close($mh);
foreach ($guildsToFetch as $params) {
heartbeat();
if ($caughtKill) {
return;
}
GetGuild($characterNames, $params[0], $params[1]);
}
}
function FetchURLBatch($urls, $curlOpts = [])
{
if (!$urls) {
return [];
}
global $connectionTracking;
$curlOpts = [CURLOPT_RETURNTRANSFER => true, CURLOPT_FOLLOWLOCATION => true, CURLOPT_MAXREDIRS => 2, CURLOPT_TIMEOUT => 10, CURLOPT_ENCODING => 'gzip'] + $curlOpts;
static $mh = false;
if ($mh === false) {
$mh = curl_multi_init();
// old curl forces pipelining on one connection if we ask for it and the server supports it
// this is slower than just opening multiple connections like we want to with curl_multi
// also, old curl doesn't interpret the http2 flag properly, and thinks we want pipelining if we just set "2" here
curl_multi_setopt($mh, CURLMOPT_PIPELINING, 0);
}
$results = [];
$curls = [];
foreach ($urls as $k => $url) {
$curls[$k] = curl_init($url);
curl_setopt_array($curls[$k], $curlOpts);
curl_multi_add_handle($mh, $curls[$k]);
}
$active = false;
do {
while (CURLM_CALL_MULTI_PERFORM == ($mrc = curl_multi_exec($mh, $active))) {
}
if ($active) {
usleep(100000);
}
} while ($active && $mrc == CURLM_OK);
foreach ($urls as $k => $url) {
$results[$k] = curl_multi_getcontent($curls[$k]);
$connectionTracking['created'] += curl_getinfo($curls[$k], CURLINFO_NUM_CONNECTS);
$connectionTracking['requests']++;
curl_multi_remove_handle($mh, $curls[$k]);
curl_close($curls[$k]);
}
return $results;
}
function download_chunks($chunklist, $effective_url)
{
global $db, $prefix, $filetime;
$mh = curl_multi_init();
$connections = array();
$files = array();
curl_multi_setopt($mh, CURLMOPT_PIPELINING, 1);
curl_multi_setopt($mh, CURLMOPT_MAXCONNECTS, 1);
foreach ($chunklist as $chunk) {
if (db_is_file_downloaded($db, $chunk["filename"])) {
// this file is already downloaded, skip downloading
l("skip downloading: : " . $chunk["filename"]);
// fill in the corresponding array elements to make these arrays synchronized with $chunklist
$connections[] = null;
$files[] = null;
continue;
}
l("going to download: " . $chunk["filename"]);
$url = str_replace("playlist.m3u8", $chunk["filename"], $effective_url);
$c = new_curl($url);
$f = fopen(DATA_DIR . $chunk["filename"], "w");
curl_setopt($c, CURLOPT_FILE, $f);
curl_multi_add_handle($mh, $c);
$connections[] = $c;
$files[] = $f;
}
//execute the handles
$still_running;
do {
$mrc = curl_multi_exec($mh, $still_running);
curl_multi_select($mh, 5);
} while ($still_running > 0);
l("all downloads finished");
$t = $filetime;
for ($i = 0; $i < count($connections); $i++) {
$c = $connections[$i];
$chunk = $chunklist[$i];
if ($c) {
curl_multi_remove_handle($mh, $c);
$status = curl_getinfo($c, CURLINFO_HTTP_CODE);
if ($status == 200) {
db_add_file($db, $t, $chunk["filename"], $chunk["duration"]);
} else {
l("chunk download failed, status: " . $status . ", filename: " . $chunk["filename"]);
}
$t += $chunk["duration"];
}
}
$filetime = $t;
foreach ($files as $f) {
if ($f) {
fclose($f);
}
}
curl_multi_close($mh);
db_save($db);
}
<?php
$callback = function ($data) {
$args = func_get_args();
var_dump($args);
};
$mh = curl_multi_init();
curl_multi_setopt($mh, CURLMOPT_PUSHFUNCTION, $callback);
$data = [];
curl_multi_setopt($mh, CURLMOPT_PUSHDATA, $data);
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'http://0.0.0.0:8080/curl');
curl_setopt($ch, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
curl_multi_add_handle($mh, $ch);
$active = null;
//execute the handles
do {
$mrc = curl_multi_exec($mh, $active);
} while ($mrc == CURLM_CALL_MULTI_PERFORM);
while ($active && $mrc == CURLM_OK) {
if (curl_multi_select($mh) != -1) {
do {
$mrc = curl_multi_exec($mh, $active);
} while ($mrc == CURLM_CALL_MULTI_PERFORM);
}
}
curl_multi_close($mh);
/**
* @see curl_multi_setopt
* @param $option
* @param $value
* @return bool
*/
public function setMultiCurlOpt($option, $value)
{
return curl_multi_setopt($this->multiCurlHandle, $option, $value);
}
/**
* Starts a session by sending out the added requests.
*
* @param reference $success **OPTIONAL. OUTPUT.** After the method is called with this parameter provided, the
* parameter's value tells whether the session was successful.
*
* @return void
*/
public function start(&$success = null)
{
$success = true;
if ($this->m_hasError) {
$success = false;
return;
}
if (CArray::isEmpty($this->m_requestRecordsQueue)) {
// Nothing to do.
return;
}
// Current policy is to disable HTTP pipelining.
$res = curl_multi_setopt($this->m_multiCurl, CURLMOPT_PIPELINING, 0);
if (!$res) {
// Should never get in here as long as cURL options are being set correctly, hence the assertion.
assert('false', vs(isset($this), get_defined_vars()));
$this->m_hasError = true;
$this->m_errorMessage = "The 'curl_multi_setopt' function failed.";
$success = false;
$this->finalize();
return;
}
$anySuccessfulRequests = false;
// Disable the script's execution timeout before getting into the session.
$timeoutPause = new CTimeoutPause();
$numRunningRequests = 0;
// also the index of the next request to send
while (true) {
// From the request queue, add as many normal cURL handles to the multi cURL handle as it is allowed by the
// maximum number of concurrent requests, priorly setting internal options for every request.
while ($numRunningRequests < CArray::length($this->m_requestRecordsQueue) && $numRunningRequests < $this->m_maxNumConcurrentRequests) {
$requestRecord = $this->m_requestRecordsQueue[$numRunningRequests];
$request = $requestRecord[0];
$onCompleteCallback = $requestRecord[1];
$newCookieSession = $requestRecord[2];
$requestCurl = $request->curl();
// Set cURL options for the normal cURL handle, having created a temporary file for cookie storage if
// needed.
$requestSetOptSuccess;
if ($this->m_cookiesAreEnabled && $request->isHttp()) {
if (!isset($this->m_cookiesFp)) {
$this->m_cookiesFp = CFile::createTemporary();
}
$request->setInternalOptions($requestSetOptSuccess, $this->m_cookiesFp, $newCookieSession);
} else {
$request->setInternalOptions($requestSetOptSuccess);
}
if (!$requestSetOptSuccess) {
if (isset($onCompleteCallback)) {
call_user_func($onCompleteCallback, false, "", $request, $this);
}
CArray::remove($this->m_requestRecordsQueue, $numRunningRequests);
continue;
}
// Add the normal cURL handle to the multi cURL handle.
$res = curl_multi_add_handle($this->m_multiCurl, $requestCurl);
if ($res != 0) {
$this->m_hasError = true;
$curlError = curl_multi_strerror($res);
$this->m_errorMessage = is_cstring($curlError) && !CString::isEmpty($curlError) ? $curlError : "The 'curl_multi_add_handle' function failed.";
$success = false;
$timeoutPause->end();
$this->finalize();
return;
}
$numRunningRequests++;
}
if ($numRunningRequests == 0) {
break;
}
// Process the currently added requests until complete or no more data is available. Although
// `CURLM_CALL_MULTI_PERFORM` is deprecated since libcurl 7.20, keep it for compatibility reasons.
$numRunningTransfers;
do {
$multiExecRes = curl_multi_exec($this->m_multiCurl, $numRunningTransfers);
} while ($multiExecRes == CURLM_CALL_MULTI_PERFORM);
if ($multiExecRes != CURLM_OK) {
$this->m_hasError = true;
$curlError = curl_multi_strerror($multiExecRes);
$this->m_errorMessage = is_cstring($curlError) && !CString::isEmpty($curlError) ? $curlError : "The 'curl_multi_exec' function failed.";
$success = false;
$timeoutPause->end();
$this->finalize();
return;
}
// Check for completed requests, call the callback function for any completed one (if such a function is
// defined), finalize completed requests, and remove completed requests from the queue.
while (true) {
$completedRequestInfo = curl_multi_info_read($this->m_multiCurl);
if (!is_cmap($completedRequestInfo)) {
break;
}
// A request has completed.
assert('$completedRequestInfo["msg"] == CURLMSG_DONE', vs(isset($this), get_defined_vars()));
$requestCurl = $completedRequestInfo["handle"];
$requestRes = $completedRequestInfo["result"];
$requestRecordPos;
$found = CArray::find($this->m_requestRecordsQueue, $requestCurl, function ($requestRecord, $requestCurl) {
$request = $requestRecord[0];
return $request->curl() == $requestCurl;
}, $requestRecordPos);
assert('$found', vs(isset($this), get_defined_vars()));
$requestRecord = $this->m_requestRecordsQueue[$requestRecordPos];
$request = $requestRecord[0];
$onCompleteCallback = $requestRecord[1];
// Remove the normal cURL handle from the multi cURL handle.
$res = curl_multi_remove_handle($this->m_multiCurl, $requestCurl);
if ($res != 0) {
$this->m_hasError = true;
$curlError = curl_multi_strerror($res);
$this->m_errorMessage = is_cstring($curlError) && !CString::isEmpty($curlError) ? $curlError : "The 'curl_multi_remove_handle' function failed.";
$success = false;
$timeoutPause->end();
$this->finalize();
return;
}
if ($requestRes == CURLE_OK) {
// The request has succeeded.
if (isset($onCompleteCallback)) {
$response;
if ($request->isReturnTransferSet()) {
$response = curl_multi_getcontent($requestCurl);
assert('is_cstring($response)', vs(isset($this), get_defined_vars()));
} else {
$response = "";
}
$request->onRequestCompleteOk();
// also close the normal cURL handle
call_user_func($onCompleteCallback, true, $response, $request, $this);
} else {
$request->onRequestCompleteOk();
// also close the normal cURL handle
}
$anySuccessfulRequests = true;
} else {
// The request has failed.
$curlError = curl_strerror($requestRes);
if (!is_cstring($curlError)) {
$curlError = "";
}
$request->onRequestCompleteWithError($curlError);
// also close the normal cURL handle
if (isset($onCompleteCallback)) {
call_user_func($onCompleteCallback, false, "", $request, $this);
}
}
CArray::remove($this->m_requestRecordsQueue, $requestRecordPos);
$numRunningRequests--;
}
assert('$numRunningRequests == $numRunningTransfers', vs(isset($this), get_defined_vars()));
if ($numRunningTransfers > 0) {
// Some requests are still being processed (by remote machines). Wait for more data to appear on
// sockets, without getting hard on the CPU.
do {
$multiSelectRes = curl_multi_select($this->m_multiCurl);
} while ($multiSelectRes == -1);
} else {
// No requests are being processed. Check if any requests are pending.
if (CArray::isEmpty($this->m_requestRecordsQueue)) {
// No requests are pending.
break;
}
}
}
// Set the script's execution time limit like the session has never happened.
$timeoutPause->end();
if (!$anySuccessfulRequests) {
$this->m_hasError = true;
$this->m_errorMessage = "None of the session's requests succeeded.";
$success = false;
}
$this->finalize();
}
/**
* @see curl_multi_setopt
*
* @param int $opt
* @param mixed $val
* @return boolean
*/
public function setOpt($opt, $val)
{
return curl_multi_setopt($this->handle, $opt, $val);
}
/**
* Constructor
*
* @link http://www.php.net/manual/en/function.curl-init.php curl_init()
* @link http://www.php.net/manual/en/function.curl-multi-init.php curl_multi_init()
*
* @throws \RuntimeException If curl_init() or curl_multi_init() fails.
* @throws \BLW\Model\InvalidArgumentException If <code>$Handles</code> is not an integer.
*
* @param integer $Handles
* Maximum handles to create for parralell operations.
*/
public function __construct($Handles)
{
// Validate $Handles
if (!is_int($Handles)) {
throw new InvalidArgumentException(0);
}
// curl_multi handle
$this->MainHandle = curl_multi_init();
// @codeCoverageIgnoreStart
// Check results
if (!is_resource($this->MainHandle)) {
throw new RuntimeException('Unable to initialize cURL multi session');
}
// Enable pipelining
if (is_callable('curl_multi_setopt')) {
curl_multi_setopt($this->MainHandle, CURLMOPT_PIPELINING, 1);
}
// @codeCoverageIgnoreEnd
// curl_init handles
foreach (range(0, $Handles) as $i) {
// Create handle
$this->Handles[$i] = curl_init();
// Check result. Add handle to free handles.
if (is_resource($this->Handles[$i])) {
$this->FreeHandles[] = $this->Handles[$i];
// @codeCoverageIgnoreStart
// Unable to create handle? Exception.
} else {
throw new RuntimeException('Unable to initialize cURL session');
}
// @codeCoverageIgnoreEnd
}
}
