function parseDetails($urls)
{
$requests = function ($arr) {
for ($i = 0; $i < count($arr); $i++) {
(yield new Request('GET', $arr[$i]));
}
};
$pool = new Pool($this->http, $requests($urls), ['concurrency' => 4, 'fulfilled' => function ($response, $index) use($urls) {
echo "{$index} loaded\n";
$this->html->load((string) $response->getBody());
$detail = $this->parseDetails_($this->html);
echo "Details parsed\n";
$id = $this->parseId($urls[$index]);
$this->details[$id] = $detail;
$this->details[$id]['url'] = $urls[$index];
$this->details[$id]['status'] = 'ok';
//var_dump($this->details);
}, 'rejected' => function ($reason, $index) {
// this is delivered each failed request
echo "{$index} rejected\n";
$id = $this->parseId($urls[$index]);
$this->details[$id]['url'] = $urls[$index];
$this->details[$id]['status'] = $reason;
}]);
// Initiate the transfers and create a promise
$promise = $pool->promise();
// Force the pool of requests to complete.
$promise->wait();
return $this->details;
}
public function handle()
{
$this->totalPageCount = count($this->users);
$client = new Client();
$requests = function ($total) use($client) {
foreach ($this->users as $key => $user) {
$uri = 'https://api.github.com/users/' . $user;
(yield function () use($client, $uri) {
return $client->getAsync($uri);
});
}
};
$pool = new Pool($client, $requests($this->totalPageCount), ['concurrency' => $this->concurrency, 'fulfilled' => function ($response, $index) {
$res = json_decode($response->getBody()->getContents());
$this->info("请求第 {$index} 个请求，用户 " . $this->users[$index] . " 的 Github ID 为：" . $res->id);
$this->countedAndCheckEnded();
}, 'rejected' => function ($reason, $index) {
$this->error("rejected");
$this->error("rejected reason: " . $reason);
$this->countedAndCheckEnded();
}]);
// 开始发送请求
$promise = $pool->promise();
$promise->wait();
}
/**
* Make all the requests and push responses (as objects if success) to self::responses["public" or "oauth"]
*/
public static function run()
{
// Create Public and Oauth Clients
self::$clientPublic = new Client(Endpoint::headers());
self::$clientOauth = new Client(Endpoint::headers(TRUE));
// Public
$poolPublic = new Pool(self::$clientPublic, self::$requests["public"], ["concurrency" => 20, "fulfilled" => function (Response $response, $index) {
// On success we json_decode response
$data = json_decode($response->getBody()->getContents(), TRUE);
// Create object and push it to responses success
$factory = OpenCrest::getFactory();
$object = $factory->create(self::$objects["public"][$index], $data, $response);
array_push(self::$responses["public"]["success"], $object);
}, "rejected" => function ($reason, $index) {
// On failure we push it to responses rejected
array_push(self::$responses["public"]["rejected"], ["reason" => $reason, "index" => $index]);
}]);
self::$promise["public"] = $poolPublic->promise();
self::$promise["public"]->wait();
// Oauth
$poolOauth = new Pool(self::$clientOauth, self::$requests["oauth"], ["concurrency" => 20, "fulfilled" => function (Response $response, $index) {
// On success we json_decode response
$data = json_decode($response->getBody()->getContents(), TRUE);
// Create object and push it to responses success
$factory = OpenCrest::getFactory();
$object = $factory->create(self::$objects["oauth"][$index], $data, $response);
array_push(self::$responses["oauth"]["success"], $object);
}, "rejected" => function ($reason, $index) {
// On failure we push it to responses rejected
array_push(self::$responses["oauth"]["rejected"], ["reason" => $reason, "index" => $index]);
}]);
self::$promise["oauth"] = $poolOauth->promise();
self::$promise["oauth"]->wait();
}
/**
* @param $pathToImportFile
* @return array
* @throws Exception
*/
public function import($pathToImportFile)
{
$masterAccountsToImport = [];
$subAccountsToImport = [];
if (($handle = fopen($pathToImportFile, "r")) !== FALSE) {
$this->validateImportFile($pathToImportFile);
$failureLogName = tempnam(getcwd() . "/log_output", "account_import_failures");
$failureLog = fopen($failureLogName, "w");
$successLogName = tempnam(getcwd() . "/log_output", "account_import_successes");
$successLog = fopen($successLogName, "w");
$returnData = ['successes' => 0, 'failures' => 0, 'failure_log_name' => $failureLogName, 'success_log_name' => $successLogName];
while (($data = fgetcsv($handle, 8096, ",")) !== FALSE) {
try {
$payload = $this->buildPayload($data);
} catch (Exception $e) {
throw new Exception("When building a payload for the following account, the payload generation failed with {$e->getMessage()}: " . implode(",", $data));
}
if (array_key_exists("sub_accounts", $payload)) {
array_push($subAccountsToImport, $data);
continue;
}
array_push($masterAccountsToImport, $data);
}
$allAccounts = array_merge($masterAccountsToImport, $subAccountsToImport);
$requests = function () use($allAccounts) {
foreach ($allAccounts as $account) {
(yield new Request("POST", $this->uri . "/api/v1/accounts", ['Content-Type' => 'application/json; charset=UTF8', 'timeout' => 30, 'Authorization' => 'Basic ' . base64_encode($this->username . ':' . $this->password)], json_encode($this->buildPayload($account))));
}
};
$pool = new Pool($this->client, $requests(), ['concurrency' => 10, 'fulfilled' => function ($response, $index) use(&$returnData, $successLog, $failureLog, $allAccounts) {
$statusCode = $response->getStatusCode();
if ($statusCode > 201) {
$body = json_decode($response->getBody()->getContents());
$line = $allAccounts[$index];
array_push($line, $body);
fputcsv($failureLog, $line);
$returnData['failures'] += 1;
} else {
$returnData['successes'] += 1;
fwrite($successLog, "Import succeeded for account ID {$allAccounts[$index][0]}" . "\n");
}
}, 'rejected' => function ($reason, $index) use(&$returnData, $failureLog, $allAccounts) {
$response = $reason->getResponse();
$body = json_decode($response->getBody()->getContents());
$returnMessage = implode(", ", (array) $body->error->message);
$line = $allAccounts[$index];
array_push($line, $returnMessage);
fputcsv($failureLog, $line);
$returnData['failures'] += 1;
}]);
$promise = $pool->promise();
$promise->wait();
} else {
throw new InvalidArgumentException("File could not be opened.");
}
fclose($failureLog);
fclose($successLog);
return $returnData;
}
/**
* @param $pathToImportFile
* @param bool $validateAddress
* @return array
*/
public function import($pathToImportFile, $validateAddress = false)
{
if (($handle = fopen($pathToImportFile, "r")) !== FALSE) {
$this->validateImportFile($pathToImportFile);
$failureLogName = tempnam(getcwd() . "/log_output", "account_secondary_address_import_failures");
$failureLog = fopen($failureLogName, "w");
$successLogName = tempnam(getcwd() . "/log_output", "account_secondary_address_import_successes");
$successLog = fopen($successLogName, "w");
$returnData = ['successes' => 0, 'failures' => 0, 'failure_log_name' => $failureLogName, 'success_log_name' => $successLogName];
$validData = [];
while (($data = fgetcsv($handle, 8096, ",")) !== FALSE) {
try {
$this->buildPayload($data, (bool) $validateAddress);
} catch (Exception $e) {
echo "Got an exception - {$e->getMessage()}\n";
continue;
}
array_push($validData, $data);
}
$requests = function () use($validData, $validateAddress) {
foreach ($validData as $validDatum) {
if (count($validDatum[0]) > 0) {
(yield new Request("POST", $this->uri . "/api/v1/accounts/" . (int) trim($validDatum[0]) . "/addresses", ['Content-Type' => 'application/json; charset=UTF8', 'timeout' => 30, 'Authorization' => 'Basic ' . base64_encode($this->username . ':' . $this->password)], json_encode($this->buildPayload($validDatum, (bool) $validateAddress))));
}
}
};
$pool = new Pool($this->client, $requests(), ['concurrency' => 10, 'fulfilled' => function ($response, $index) use(&$returnData, $successLog, $failureLog, $validData) {
$statusCode = $response->getStatusCode();
if ($statusCode > 201) {
$body = json_decode($response->getBody()->getContents());
$line = $validData[$index];
array_push($line, $body);
fputcsv($failureLog, $line);
$returnData['failures'] += 1;
} else {
$returnData['successes'] += 1;
fwrite($successLog, "Secondary address import succeeded for account ID {$validData[$index][0]}" . "\n");
}
}, 'rejected' => function ($reason, $index) use(&$returnData, $failureLog, $validData) {
$response = $reason->getResponse();
$body = json_decode($response->getBody()->getContents());
$returnMessage = implode(", ", (array) $body->error->message);
$line = $validData[$index];
array_push($line, $returnMessage);
fputcsv($failureLog, $line);
$returnData['failures'] += 1;
}]);
$promise = $pool->promise();
$promise->wait();
} else {
throw new InvalidArgumentException("File could not be opened.");
}
fclose($failureLog);
fclose($successLog);
return $returnData;
}
public function executeRequests(array $requests)
{
set_time_limit(Settings::TIMEOUT_LIMIT_IN_SECONDS);
ini_set('memory_limit', Settings::MEMORY_LIMIT);
$this->resetFileHandles();
$pool = new Pool($this->client, $requests, ['complete' => function (CompleteEvent $event) {
$this->saveResponseToFile($event->getResponse());
}, 'pool_size' => Settings::NB_OF_PARALLEL_REQUESTS]);
$pool->wait();
return $this->fileHandles;
}
/**
* {@inheritdoc}
*/
public static function getImagesInfo(array $urls, array $config = null)
{
$client = isset($config['client']) ? $config['client'] : new Client(['defaults' => static::$config]);
$result = [];
// Build parallel requests
$requests = [];
foreach ($urls as $url) {
if (strpos($url['value'], 'data:') === 0) {
if ($info = static::getEmbeddedImageInfo($url['value'])) {
$result[] = array_merge($url, $info);
}
continue;
}
$requests[] = $client->createRequest('GET', $url['value']);
}
// Execute in parallel
$responses = Pool::batch($client, $requests);
// Build result set
foreach ($responses as $i => $response) {
if ($response instanceof RequestException) {
continue;
}
if (($size = getimagesizefromstring($response->getBody())) !== false) {
$result[] = ['width' => $size[0], 'height' => $size[1], 'size' => $size[0] * $size[1], 'mime' => $size['mime']] + $urls[$i];
}
}
return $result;
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$config = $this->setConfiguraton($input);
$client = new \GuzzleHttp\Client(['defaults' => ['allow_redirects' => false, 'timeout' => 5, 'connect_timeout' => 5]]);
/** @var Instance $instance */
foreach ($config->getInstances() as $instance) {
$requests[] = $client->createRequest('HEAD', $instance->getUrl());
}
$options = [];
Pool::send($client, $requests, ['complete' => function (CompleteEvent $event) {
}, 'error' => function (ErrorEvent $event) use($config) {
$instance = $config->findInstanceByUrl($event->getRequest()->getUrl());
if ($instance == null) {
throw new \RuntimeException('Instance not found');
}
if (!$event->getException()->hasResponse()) {
$raven = new \Raven_Client($instance->getSentryDsn());
$event_id = $raven->getIdent($raven->captureMessage(sprintf('The website %s with url -> %s is down or has a problem', $instance->getName(), $event->getRequest()->getUrl()), [], \Raven_Client::FATAL));
if ($raven->getLastError() !== null) {
printf('There was an error sending the event to Sentry: %s', $raven->getLastError());
}
$error_handler = new \Raven_ErrorHandler($raven);
$error_handler->registerExceptionHandler();
$error_handler->registerErrorHandler();
$error_handler->registerShutdownFunction();
}
}]);
}
/**
* @inheritdoc
*/
public function multiRequest(array $urls)
{
$client = new Client();
$requests = array();
foreach ($urls as $urlName => $urlData) {
if (is_string($urlData)) {
$urlData = array($urlData, array());
}
$urlOptions = new Options($urlData[1]);
$method = $urlOptions->get('method', 'GET', 'up');
$args = $urlOptions->get('args');
$url = 'GET' === $method ? Url::addArg((array) $args, $urlData[0]) : $urlData[0];
$requests[$urlName] = $client->createRequest($method, $url, $this->_getClientOptions($urlOptions, $method, $args));
}
$httpResults = Pool::batch($client, $requests);
/** @var string $resName */
/** @var Response $httpResult */
$result = array();
$index = 0;
$keys = array_keys($urls);
foreach ($keys as $resName) {
$httpResult = $httpResults->offsetGet($index++);
$result[$resName] = array($httpResult->getStatusCode(), $httpResult->getHeaders(), $httpResult->getBody()->getContents());
}
return $result;
}
/**
* Download all wallpapers in list.
*
* @param string $directory Where to download wallpapers.
*
* @throws DownloadException Thrown if the download directory cannot be created.
*/
public function downloadAll($directory)
{
if (!file_exists($directory)) {
if (!@mkdir($directory, null, true)) {
throw new DownloadException("The download directory cannot be created.");
}
}
$client = new Client();
$requests = [];
foreach ($this->wallpapers as $w) {
$url = $w->getImageUrl(true);
$requests[] = $client->createRequest('GET', $url, ['save_to' => $directory . '/' . basename($url)]);
}
$results = Pool::batch($client, $requests);
// Retry with PNG
$retryRequests = [];
foreach ($results->getFailures() as $e) {
// Delete failed files
unlink($directory . '/' . basename($e->getRequest()->getUrl()));
$urlPng = str_replace('.jpg', '.png', $e->getRequest()->getUrl());
$statusCode = $e->getResponse()->getStatusCode();
if ($statusCode == 404) {
$retryRequests[] = $client->createRequest('GET', $urlPng, ['save_to' => $directory . '/' . basename($urlPng)]);
}
}
Pool::batch($client, $retryRequests);
}
/**
* 通过guzzle发送并行请求
*
* @param array $request_info 多个请求的地址
* @return array 按请求的顺序以数字键为key，返回对应内容
*/
public static function postConcurrent($request_info)
{
$ret = [];
$requests = [];
$client = self::getInstance();
foreach ($request_info as $v) {
//print_r($v['params']) . '<br>';
$requests[] = $client->createRequest('post', $v['url'], isset($v['params']) ? $v['params'] : array());
}
// Results is a GuzzleHttp\BatchResults object.
$results = Pool::batch($client, $requests);
foreach ($results as $key => $response) {
if (get_class($response) == 'GuzzleHttp\\Message\\Response') {
$headers = $response->getHeaders();
$contentType = $headers['Content-Type'][0];
$contentType = explode(';', $contentType);
$ret[$key]['contentType'] = $contentType[0];
$ret[$key]['statusCode'] = $response->getStatusCode();
if ($contentType[0] == 'application/json') {
$ret[$key]['contentType'] = 'json';
}
$ret[$key]['body'] = $response->getbody();
} else {
$ret[$key]['contentType'] = 'text/html';
$ret[$key]['statusCode'] = $response->getCode();
$ret[$key]['body'] = $response->getMessage();
}
}
return $ret;
}
/**
* @param string $url
* @return array|mixed|null
*/
public function get($url)
{
// Aenderungen an der Konfiguration invalidieren den Cache
$cache_key = md5($url . $this->baseCacheKey);
if (!filter_var($url, FILTER_VALIDATE_URL)) {
return null;
}
if ($this->cache->hasItem($cache_key)) {
return json_decode($this->cache->getItem($cache_key), true);
}
if (!$this->isValidDomain($url)) {
return null;
}
$requests = array_map(function ($service) use($url) {
/** @var ServiceInterface $service */
return $service->getRequest($url);
}, $this->services);
$results = Pool::batch($this->client, $requests);
$counts = array();
$i = 0;
foreach ($this->services as $service) {
if (method_exists($results[$i], "json")) {
try {
$counts[$service->getName()] = (int) $service->extractCount($results[$i]->json());
} catch (\Exception $e) {
// Skip service if broken
}
}
$i++;
}
$this->cache->setItem($cache_key, json_encode($counts));
return $counts;
}
/**
* Fetch and process a paged response from JoindIn.
*
* @param $initial
* The initial URL to fetch. That URL may result in others getting
* fetched as well.
* @param callable $processor
* A callable that will be used to process each page of results. Its signature
* must be: (\SplQueue $pages, ResponseInterface $response, $index)
* @param int $concurrency
* The number of concurrent requests to send. In practice this kinda has to
* be 1, or else Guzzle will conclude itself before getting to later-added
* entries.
*/
function fetchPages($initial, callable $processor, $concurrency = 1)
{
$client = getClient();
$pages = new \SplQueue();
$pages->enqueue($initial);
$pageProcessor = partial($processor, $pages);
$pageRequestGenerator = function (\SplQueue $pages) {
foreach ($pages as $page) {
(yield new Request('GET', $page));
}
};
$pool = new Pool($client, $pageRequestGenerator($pages), ['concurrency' => $concurrency, 'fulfilled' => $pageProcessor]);
// Initiate the transfers and create a promise
$promise = $pool->promise();
// Force the pool of requests to complete.
$promise->wait();
}
/**
* {@inheritdoc}
*/
protected function doSendInternalRequests(array $internalRequests, $success, $error)
{
$requests = array();
foreach ($internalRequests as $internalRequest) {
$requests[] = $this->createRequest($internalRequest, $success, $error);
}
class_exists('GuzzleHttp\\Pool') ? Pool::batch($this->client, $requests) : \GuzzleHttp\batch($this->client, $requests);
}
/**
* @inheritdoc
*/
public function sendBatch(BatchRequest $batchRequest) : array
{
$requests = array_map([$this, 'createRequest'], $batchRequest->getRequests());
$batchResults = Pool::batch($this->guzzleClient, $requests);
if ($batchResults->getFailures()) {
throw new BatchRequestFailureException($batchRequest, $batchResults);
}
return $batchResults->getSuccessful();
}
public static function concurrent($request_info)
{
$requests = [];
$client = new GuzzleHttp\Client();
foreach ($request_info as $v) {
$requests[] = $client->createRequest($v[0], $v[1], $v[2]);
}
return Pool::batch($client, $requests);
}
/**
* @issue https://github.com/guzzle/guzzle/issues/867
*/
public function testDoesNotFailInEventSystemForNetworkError()
{
$c = new Client();
$r = $c->createRequest('GET', Server::$url, ['timeout' => 1, 'connect_timeout' => 1, 'proxy' => 'http://127.0.0.1:123/foo']);
$events = [];
$fn = function (AbstractTransferEvent $event) use(&$events) {
$events[] = [get_class($event), $event->hasResponse(), $event->getResponse()];
};
$pool = new Pool($c, [$r], ['error' => $fn, 'end' => $fn]);
$pool->wait();
$this->assertCount(2, $events);
$this->assertEquals('GuzzleHttp\\Event\\ErrorEvent', $events[0][0]);
$this->assertFalse($events[0][1]);
$this->assertNull($events[0][2]);
$this->assertEquals('GuzzleHttp\\Event\\EndEvent', $events[1][0]);
$this->assertFalse($events[1][1]);
$this->assertNull($events[1][2]);
}
/**
* @param \Destiny\DestinyRequest[]|\Destiny\DestinyRequest $requests
* @throws \DestinyException
* @throws \Exception
*
* @return array
*/
public function request($requests)
{
$multi = true;
if ($requests instanceof DestinyRequest) {
$multi = false;
$requests = [$requests];
}
$batch = $responses = [];
foreach ($requests as $key => $request) {
if (!$request instanceof DestinyRequest) {
throw new Exception('Invalid request');
}
if (!CACHE_ENABLED || $request->raw) {
Cache::forget($request->key);
}
if ($request->cache && Cache::has($request->key)) {
$responses[$key] = Cache::get($request->key);
} else {
$req = $this->createRequest('GET', $request->url);
$req->getEmitter()->attach($request);
$batch[$key] = $req;
}
}
if (count($batch)) {
$keys = array_keys($batch);
foreach (Pool::batch($this, $batch) as $i => $result) {
$key = $keys[$i];
$request = $requests[$key];
if ($request instanceof DestinyRequest && $request->raw) {
$responses[$key] = $result;
continue;
}
if ($result instanceof Exception) {
Cache::forget($request->key);
throw new DestinyException($result->getMessage(), $result->getCode(), $result);
}
if ($result instanceof Response) {
$response = json_decode($result->getBody()->getContents(), true);
if (array_get($response, 'ErrorStatus') !== 'Success') {
Cache::forget($request->key);
Bugsnag::setMetaData(['bungie' => $response]);
throw new DestinyException(array_get($response, 'Message'), array_get($response, 'ErrorCode'));
}
$response = array_get($response, 'Response');
if (empty($response)) {
Cache::forget($request->key);
}
if ($request->cache) {
Cache::put($request->key, $response, $request->cache);
}
$responses[$key] = $response;
}
}
}
return $multi ? $responses : array_shift($responses);
}
public function sendRequestBatch(array $requests)
{
$responses = Pool::batch($this->client, $requests);
foreach ($responses as $response) {
if ($response instanceof PhpException) {
throw $response;
}
}
return $responses;
}
/**
* @param array $data array($url=>$savePath)
*/
public function downloadFiles(array $data)
{
trigger_error('Not yet implemented', E_USER_WARNING);
return;
$client = new Client();
$requests = array();
foreach ($data as $url => $path) {
$requests[] = new Request('GET', HttpAdapterInterface::BASE_URL . $url, ['save_to' => $path]);
}
$pool = new Pool($client, $requests, ['concurrency' => 5, 'fulfilled' => function ($response, $index) {
// this is delivered each successful response
}, 'rejected' => function ($reason, $index) {
// this is delivered each failed request
//TODO error handling
}]);
// Initiate the transfers and create a promise
$promise = $pool->promise();
// Force the pool of requests to complete.
$promise->wait();
}
public function requestsBatch($method, $uris)
{
$requests = array_map(function ($eventUri) use($method) {
$this->validateMethod($method);
return new Request($method, $this->uri($eventUri), ['Accept' => [self::FORMAT_EVENT_STORE_ATOM]]);
}, $uris);
$responses = Pool::batch($this->guzzle, $requests);
return array_map(function (ResponseInterface $response) {
return json_decode($response->getBody()->getContents(), true);
}, $responses);
}
/**
* {@inheritdoc}
*/
protected function sendInternalRequests(array $internalRequests, $success, $error)
{
$requests = array();
foreach ($internalRequests as $key => $internalRequest) {
$requests[$key] = $this->createRequest($internalRequest);
}
$httpAdapter = $this;
$pool = new Pool($this->client, $requests, array_merge($this->createOptions(), array('fulfilled' => function ($response, $index) use($success, $internalRequests, $httpAdapter) {
$response = $httpAdapter->getConfiguration()->getMessageFactory()->createResponse((int) $response->getStatusCode(), $response->getProtocolVersion(), $response->getHeaders(), BodyNormalizer::normalize(function () use($response) {
return $response->getBody()->detach();
}, $internalRequests[$index]->getMethod()));
$response = $response->withParameter('request', $internalRequests[$index]);
call_user_func($success, $response);
}, 'rejected' => function ($exception, $index) use($error, $internalRequests, $httpAdapter) {
$exception = HttpAdapterException::cannotFetchUri($exception->getRequest()->getUri(), $httpAdapter->getName(), $exception->getMessage());
$exception->setRequest($internalRequests[$index]);
call_user_func($error, $exception);
})));
$pool->promise()->wait();
}
public static function prepare(Event $event)
{
$composer = $event->getComposer();
$io = $event->getIO();
// TODO: Remove this hack when composer supports loading autoload.files for plugins
$vendorDir = $composer->getConfig()->get('vendor-dir');
$composerDir = $vendorDir . '/composer';
$includeFiles = (require $composerDir . '/autoload_files.php');
foreach ($includeFiles as $file) {
require $file;
}
// End of hack
/** @var \Composer\Package\Package $package */
$package = $composer->getPackage();
$dependencies = [];
$requires = $package->getRequires();
/** @var \Composer\Package\Link $require */
foreach ($requires as $require) {
$dependencies[$require->getTarget()] = $require->getPrettyConstraint();
}
$devRequires = $package->getDevRequires();
/** @var \Composer\Package\Link $require */
foreach ($devRequires as $devRequire) {
$dependencies[$devRequire->getTarget()] = $devRequire->getPrettyConstraint();
}
$extra = $package->getExtra();
$proxyUrls = isset($extra['composer-proxy']['url']) ? $extra['composer-proxy']['url'] : null;
if (!$proxyUrls) {
$io->write('<warning>No composer proxy url defined in composer.json. You might not use your proxy when fetching packages.</warning>');
return;
}
$io->write('<info>Sending data over to the defined proxies in order to cache your packages. Please wait, this may take a while...</info>');
$client = new GuzzleClient();
$requests = [];
foreach ($proxyUrls as $proxyUrl) {
$requests[] = $client->createRequest('POST', $proxyUrl . '/packages', ['body' => $dependencies]);
}
$pool = new Pool($client, $requests);
$res = $pool->wait();
$io->write('<info>Proxies caching completed!</info>');
}
/**
* execute batch requests (post).
*
* @throws BatchRequestException
*
* @return Models\Responses\OrtcResponse
*/
public function batchExecute()
{
$guzzleRequests = $this->createBatchPostRequests();
$results = Pool::batch($this->guzzleClient, $guzzleRequests, ['pool_size' => $this->request->getOrtcConfig()->getBatchPoolSize()]);
if (count($results->getFailures()) > 0) {
$batchRequestException = new BatchRequestException();
$batchRequestException->setResults($results);
throw $batchRequestException;
}
$handler = $this->request->getResponseHandler();
return $handler->handle($results);
}
public function fetch($responseFunction, $isMultithreaded = false, $threadCount = 1)
{
if ($isMultithreaded) {
$requestList = [];
foreach ($this->uriList as $uri) {
$requestList[] = new Request('GET', $uri);
}
$uriPool = new Pool($this->guzzle, $requestList, ['concurrency' => $threadCount, 'fulfilled' => function (Response $response, $index) use($responseFunction) {
$responseFunction($response->getBody()->getContents(), $index);
}, 'rejected' => function () {
}]);
$promise = $uriPool->promise();
$promise->wait();
} else {
$parsedPageCounter = 0;
foreach ($this->uriList as $uri) {
$response = $this->buzz->get($uri);
$responseFunction($response->getContent(), ++$parsedPageCounter);
}
}
}
/**
* run.
*
* @return array
*/
public function run()
{
$urls = $this->urlProvider->getUrls();
$client = $this->client;
//This is a bit messie, need a refacto
$resultCollection = new ResultCollection();
$requests = function () use($urls, $client, $resultCollection) {
foreach ($urls as $url) {
(yield function () use($client, $url, $resultCollection) {
return $client->sendAsync(new Request($url->getMethod(), $url->getUrl(), $url->getHeaders()), ['timeout' => $url->getTimeout(), 'connect_timeout' => $url->getTimeout(), 'on_stats' => function (TransferStats $tranferStats) use($url, $resultCollection) {
$handlerError = null;
$validatorError = null;
$validatorResult = null;
if ($tranferStats->hasResponse()) {
$validatorResult = $url->getValidator()->check((string) $tranferStats->getResponse()->getBody());
if (false === $validatorResult) {
$validatorError = $url->getValidator()->getError();
}
$statusCode = $tranferStats->getResponse()->getStatusCode();
$transferTime = $tranferStats->getTransferTime();
} else {
// If we have a connection error
$statusCode = 400;
$transferTime = 0;
$handlerError = curl_strerror($tranferStats->getHandlerErrorData());
}
$resultCollection->offsetSet($url->getName(), new Result($url, $statusCode, $transferTime, $handlerError, $validatorResult, $validatorError));
}]);
});
}
};
$pool = new Pool($this->client, $requests(), ['concurrency' => 5]);
$promise = $pool->promise();
$promise->wait();
return $resultCollection;
}
/**
* @param array $data array($url=>$savePath)
*/
public function downloadFiles(array $data)
{
$client = new Client();
$requests = array();
foreach ($data as $url => $path) {
$requests[] = $client->createRequest('GET', HttpAdapterInterface::BASE_URL . $url, ['save_to' => $path]);
}
// Results is a GuzzleHttp\BatchResults object.
$results = Pool::batch($client, $requests, ['pool_size' => 5]);
// Retrieve all failures.
foreach ($results->getFailures() as $requestException) {
//TODO error handling
throw new \Exception($requestException->getMessage());
}
}
public function getCounts($url)
{
$requests = [];
/** @var Provider $provider */
foreach ($this->providers as $name => $provider) {
$requests[$name] = $provider->getRequest($url);
}
$responses = Pool::batch($this->client, $requests);
foreach ($requests as $name => $request) {
$result = $responses->getResult($request);
if ($result instanceof \Exception) {
$requests[$name] = false;
} else {
$requests[$name] = $this->providers[$name]->getCount($result->getBody()->getContents(), $result);
}
}
return $requests;
}
/**
* @inheritDoc
*/
public function batchLoad($requests)
{
/** @var IApiRequest[] $originals */
$originals = [];
$batchRequests = [];
foreach ($requests as $i => $request) {
if ($request instanceof IApiRequest) {
$originals[$i] = $request;
$batchRequests[$i] = $this->_makeRequest($request);
}
}
$options = [];
$responses = Pool::batch($this->_client, $batchRequests, $options);
foreach ($responses as $id => $response) {
$originals[$id]->setRawResult($this->_getResult($response));
}
return $this;
}
/**
* @return array Array with 'ok' and 'failed' keys
*/
public function send()
{
$responses = Pool::batch(new HttpClient(), $this->requests);
$result = ['ok' => [], 'failed' => []];
/** @var ResponseInterface $response */
foreach ($responses as $key => $response) {
if ($response instanceof RequestException) {
/** @var RequestException $response */
$result['failed'][] = ['exception' => $response->getMessage(), 'url' => $response->hasResponse() ? $response->getResponse()->getEffectiveUrl() : $response->getRequest()->getUrl(), 'object' => $this->objects[$key], 'status_code' => $response->hasResponse() ? $response->getResponse()->getStatusCode() : '0', 'reason' => $response->hasResponse() ? $response->getResponse()->getReasonPhrase() : ''];
continue;
}
if (in_array($response->getStatusCode(), $this->validResponses)) {
$result['ok'][] = $this->objects[$key];
continue;
}
$result['failed'][] = ['url' => $response->getEffectiveUrl(), 'object' => $this->objects[$key], 'status_code' => $response->getStatusCode(), 'reason' => $response->getReasonPhrase()];
}
return $result;
}
