/**
* @inheritdoc
*/
public function multiRequest(array $urls)
{
$client = new Client();
$requests = array();
foreach ($urls as $urlName => $urlData) {
if (is_string($urlData)) {
$urlData = array($urlData, array());
}
$urlOptions = new Options($urlData[1]);
$method = $urlOptions->get('method', 'GET', 'up');
$args = $urlOptions->get('args');
$url = 'GET' === $method ? Url::addArg((array) $args, $urlData[0]) : $urlData[0];
$requests[$urlName] = $client->createRequest($method, $url, $this->_getClientOptions($urlOptions, $method, $args));
}
$httpResults = Pool::batch($client, $requests);
/** @var string $resName */
/** @var Response $httpResult */
$result = array();
$index = 0;
$keys = array_keys($urls);
foreach ($keys as $resName) {
$httpResult = $httpResults->offsetGet($index++);
$result[$resName] = array($httpResult->getStatusCode(), $httpResult->getHeaders(), $httpResult->getBody()->getContents());
}
return $result;
}
/**
* 通过guzzle发送并行请求
*
* @param array $request_info 多个请求的地址
* @return array 按请求的顺序以数字键为key，返回对应内容
*/
public static function postConcurrent($request_info)
{
$ret = [];
$requests = [];
$client = self::getInstance();
foreach ($request_info as $v) {
//print_r($v['params']) . '<br>';
$requests[] = $client->createRequest('post', $v['url'], isset($v['params']) ? $v['params'] : array());
}
// Results is a GuzzleHttp\BatchResults object.
$results = Pool::batch($client, $requests);
foreach ($results as $key => $response) {
if (get_class($response) == 'GuzzleHttp\\Message\\Response') {
$headers = $response->getHeaders();
$contentType = $headers['Content-Type'][0];
$contentType = explode(';', $contentType);
$ret[$key]['contentType'] = $contentType[0];
$ret[$key]['statusCode'] = $response->getStatusCode();
if ($contentType[0] == 'application/json') {
$ret[$key]['contentType'] = 'json';
}
$ret[$key]['body'] = $response->getbody();
} else {
$ret[$key]['contentType'] = 'text/html';
$ret[$key]['statusCode'] = $response->getCode();
$ret[$key]['body'] = $response->getMessage();
}
}
return $ret;
}
/**
* {@inheritdoc}
*/
public static function getImagesInfo(array $urls, array $config = null)
{
$client = isset($config['client']) ? $config['client'] : new Client(['defaults' => static::$config]);
$result = [];
// Build parallel requests
$requests = [];
foreach ($urls as $url) {
if (strpos($url['value'], 'data:') === 0) {
if ($info = static::getEmbeddedImageInfo($url['value'])) {
$result[] = array_merge($url, $info);
}
continue;
}
$requests[] = $client->createRequest('GET', $url['value']);
}
// Execute in parallel
$responses = Pool::batch($client, $requests);
// Build result set
foreach ($responses as $i => $response) {
if ($response instanceof RequestException) {
continue;
}
if (($size = getimagesizefromstring($response->getBody())) !== false) {
$result[] = ['width' => $size[0], 'height' => $size[1], 'size' => $size[0] * $size[1], 'mime' => $size['mime']] + $urls[$i];
}
}
return $result;
}
/**
* @param string $url
* @return array|mixed|null
*/
public function get($url)
{
// Aenderungen an der Konfiguration invalidieren den Cache
$cache_key = md5($url . $this->baseCacheKey);
if (!filter_var($url, FILTER_VALIDATE_URL)) {
return null;
}
if ($this->cache->hasItem($cache_key)) {
return json_decode($this->cache->getItem($cache_key), true);
}
if (!$this->isValidDomain($url)) {
return null;
}
$requests = array_map(function ($service) use($url) {
/** @var ServiceInterface $service */
return $service->getRequest($url);
}, $this->services);
$results = Pool::batch($this->client, $requests);
$counts = array();
$i = 0;
foreach ($this->services as $service) {
if (method_exists($results[$i], "json")) {
try {
$counts[$service->getName()] = (int) $service->extractCount($results[$i]->json());
} catch (\Exception $e) {
// Skip service if broken
}
}
$i++;
}
$this->cache->setItem($cache_key, json_encode($counts));
return $counts;
}
/**
* Download all wallpapers in list.
*
* @param string $directory Where to download wallpapers.
*
* @throws DownloadException Thrown if the download directory cannot be created.
*/
public function downloadAll($directory)
{
if (!file_exists($directory)) {
if (!@mkdir($directory, null, true)) {
throw new DownloadException("The download directory cannot be created.");
}
}
$client = new Client();
$requests = [];
foreach ($this->wallpapers as $w) {
$url = $w->getImageUrl(true);
$requests[] = $client->createRequest('GET', $url, ['save_to' => $directory . '/' . basename($url)]);
}
$results = Pool::batch($client, $requests);
// Retry with PNG
$retryRequests = [];
foreach ($results->getFailures() as $e) {
// Delete failed files
unlink($directory . '/' . basename($e->getRequest()->getUrl()));
$urlPng = str_replace('.jpg', '.png', $e->getRequest()->getUrl());
$statusCode = $e->getResponse()->getStatusCode();
if ($statusCode == 404) {
$retryRequests[] = $client->createRequest('GET', $urlPng, ['save_to' => $directory . '/' . basename($urlPng)]);
}
}
Pool::batch($client, $retryRequests);
}
/**
* {@inheritdoc}
*/
protected function doSendInternalRequests(array $internalRequests, $success, $error)
{
$requests = array();
foreach ($internalRequests as $internalRequest) {
$requests[] = $this->createRequest($internalRequest, $success, $error);
}
class_exists('GuzzleHttp\\Pool') ? Pool::batch($this->client, $requests) : \GuzzleHttp\batch($this->client, $requests);
}
public static function concurrent($request_info)
{
$requests = [];
$client = new GuzzleHttp\Client();
foreach ($request_info as $v) {
$requests[] = $client->createRequest($v[0], $v[1], $v[2]);
}
return Pool::batch($client, $requests);
}
/**
* @inheritdoc
*/
public function sendBatch(BatchRequest $batchRequest) : array
{
$requests = array_map([$this, 'createRequest'], $batchRequest->getRequests());
$batchResults = Pool::batch($this->guzzleClient, $requests);
if ($batchResults->getFailures()) {
throw new BatchRequestFailureException($batchRequest, $batchResults);
}
return $batchResults->getSuccessful();
}
/**
* @param \Destiny\DestinyRequest[]|\Destiny\DestinyRequest $requests
* @throws \DestinyException
* @throws \Exception
*
* @return array
*/
public function request($requests)
{
$multi = true;
if ($requests instanceof DestinyRequest) {
$multi = false;
$requests = [$requests];
}
$batch = $responses = [];
foreach ($requests as $key => $request) {
if (!$request instanceof DestinyRequest) {
throw new Exception('Invalid request');
}
if (!CACHE_ENABLED || $request->raw) {
Cache::forget($request->key);
}
if ($request->cache && Cache::has($request->key)) {
$responses[$key] = Cache::get($request->key);
} else {
$req = $this->createRequest('GET', $request->url);
$req->getEmitter()->attach($request);
$batch[$key] = $req;
}
}
if (count($batch)) {
$keys = array_keys($batch);
foreach (Pool::batch($this, $batch) as $i => $result) {
$key = $keys[$i];
$request = $requests[$key];
if ($request instanceof DestinyRequest && $request->raw) {
$responses[$key] = $result;
continue;
}
if ($result instanceof Exception) {
Cache::forget($request->key);
throw new DestinyException($result->getMessage(), $result->getCode(), $result);
}
if ($result instanceof Response) {
$response = json_decode($result->getBody()->getContents(), true);
if (array_get($response, 'ErrorStatus') !== 'Success') {
Cache::forget($request->key);
Bugsnag::setMetaData(['bungie' => $response]);
throw new DestinyException(array_get($response, 'Message'), array_get($response, 'ErrorCode'));
}
$response = array_get($response, 'Response');
if (empty($response)) {
Cache::forget($request->key);
}
if ($request->cache) {
Cache::put($request->key, $response, $request->cache);
}
$responses[$key] = $response;
}
}
}
return $multi ? $responses : array_shift($responses);
}
public function sendRequestBatch(array $requests)
{
$responses = Pool::batch($this->client, $requests);
foreach ($responses as $response) {
if ($response instanceof PhpException) {
throw $response;
}
}
return $responses;
}
public function requestsBatch($method, $uris)
{
$requests = array_map(function ($eventUri) use($method) {
$this->validateMethod($method);
return new Request($method, $this->uri($eventUri), ['Accept' => [self::FORMAT_EVENT_STORE_ATOM]]);
}, $uris);
$responses = Pool::batch($this->guzzle, $requests);
return array_map(function (ResponseInterface $response) {
return json_decode($response->getBody()->getContents(), true);
}, $responses);
}
/**
* execute batch requests (post).
*
* @throws BatchRequestException
*
* @return Models\Responses\OrtcResponse
*/
public function batchExecute()
{
$guzzleRequests = $this->createBatchPostRequests();
$results = Pool::batch($this->guzzleClient, $guzzleRequests, ['pool_size' => $this->request->getOrtcConfig()->getBatchPoolSize()]);
if (count($results->getFailures()) > 0) {
$batchRequestException = new BatchRequestException();
$batchRequestException->setResults($results);
throw $batchRequestException;
}
$handler = $this->request->getResponseHandler();
return $handler->handle($results);
}
/**
* @param array $data array($url=>$savePath)
*/
public function downloadFiles(array $data)
{
$client = new Client();
$requests = array();
foreach ($data as $url => $path) {
$requests[] = $client->createRequest('GET', HttpAdapterInterface::BASE_URL . $url, ['save_to' => $path]);
}
// Results is a GuzzleHttp\BatchResults object.
$results = Pool::batch($client, $requests, ['pool_size' => 5]);
// Retrieve all failures.
foreach ($results->getFailures() as $requestException) {
//TODO error handling
throw new \Exception($requestException->getMessage());
}
}
/**
* @inheritDoc
*/
public function batchLoad($requests)
{
/** @var IApiRequest[] $originals */
$originals = [];
$batchRequests = [];
foreach ($requests as $i => $request) {
if ($request instanceof IApiRequest) {
$originals[$i] = $request;
$batchRequests[$i] = $this->_makeRequest($request);
}
}
$options = [];
$responses = Pool::batch($this->_client, $batchRequests, $options);
foreach ($responses as $id => $response) {
$originals[$id]->setRawResult($this->_getResult($response));
}
return $this;
}
public function getCounts($url)
{
$requests = [];
/** @var Provider $provider */
foreach ($this->providers as $name => $provider) {
$requests[$name] = $provider->getRequest($url);
}
$responses = Pool::batch($this->client, $requests);
foreach ($requests as $name => $request) {
$result = $responses->getResult($request);
if ($result instanceof \Exception) {
$requests[$name] = false;
} else {
$requests[$name] = $this->providers[$name]->getCount($result->getBody()->getContents(), $result);
}
}
return $requests;
}
/**
* @return array Array with 'ok' and 'failed' keys
*/
public function send()
{
$responses = Pool::batch(new HttpClient(), $this->requests);
$result = ['ok' => [], 'failed' => []];
/** @var ResponseInterface $response */
foreach ($responses as $key => $response) {
if ($response instanceof RequestException) {
/** @var RequestException $response */
$result['failed'][] = ['exception' => $response->getMessage(), 'url' => $response->hasResponse() ? $response->getResponse()->getEffectiveUrl() : $response->getRequest()->getUrl(), 'object' => $this->objects[$key], 'status_code' => $response->hasResponse() ? $response->getResponse()->getStatusCode() : '0', 'reason' => $response->hasResponse() ? $response->getResponse()->getReasonPhrase() : ''];
continue;
}
if (in_array($response->getStatusCode(), $this->validResponses)) {
$result['ok'][] = $this->objects[$key];
continue;
}
$result['failed'][] = ['url' => $response->getEffectiveUrl(), 'object' => $this->objects[$key], 'status_code' => $response->getStatusCode(), 'reason' => $response->getReasonPhrase()];
}
return $result;
}
private function _getDataPartial($filter, $page)
{
$requests = [];
$return = [];
$current_page = $page;
for ($i = 1; $i <= $this->batch_size; $i++) {
$current_page = $page + $i;
$requests[] = $request = $this->client->createRequest('GET', $this->path, ['auth' => [env('FRESHDESK_API_KEY'), 'X']]);
$query = $request->getQuery();
$query['page'] = $current_page;
}
$results = Pool::batch($this->client, $requests);
try {
foreach ($results->getSuccessful() as $result) {
$dataset = array_filter($result->json(), $filter);
$return = array_merge($return, $dataset);
}
} catch (ParseException $e) {
throw new DataControllerException($e->getMessage(), $e->getCode(), $e);
}
return ['current_page' => $current_page, 'data' => $return];
}
private function findThumbnail($url)
{
$client = new Client();
$response = $client->get($url);
$crawler = new Crawler($response->getBody());
$requests = $crawler->filter('img')->each(function (Crawler $node, $i) use($client, $url) {
$src = $node->attr('src');
if (starts_with($src, '//')) {
$src = 'http:' . $src;
}
if (!starts_with($src, ['http://', 'https://'])) {
$src = parse_url($url, PHP_URL_HOST) . $src;
}
return $client->createRequest('GET', $src);
});
$results = Pool::batch($client, $requests);
$images = [];
foreach ($results->getSuccessful() as $response) {
list($x, $y) = @getimagesize($response->getBody()->getUri());
$images[$response->getEffectiveUrl()] = $x * $y;
}
return current(array_keys($images, max($images)));
}
public function testDoesNotThrowInErrorEvent()
{
$client = new Client();
$responses = [new Response(404)];
$client->getEmitter()->attach(new Mock($responses));
$requests = [$client->createRequest('GET', 'http://foo.com/baz')];
$result = Pool::batch($client, $requests);
$this->assertCount(1, $result);
$this->assertInstanceOf('GuzzleHttp\\Exception\\ClientException', $result[0]);
}
/**
* Loads details about multiple packages.
*
* @param array $packages
* @return array
*/
protected function getMultiplePackageDetails($packages)
{
$requests = [];
foreach ($packages as $package) {
$requests[] = $this->client->createRequest('GET', $this->getPackageDetailsUrl($package));
}
$result = Pool::batch($this->client, $requests);
$json = [];
foreach ($result->getSuccessful() as $item) {
$json[] = $item->json()['package'];
}
return $json;
}
public function testBatchesResultsWithCallbacks()
{
$requests = [new Request('GET', 'http://foo.com/200'), new Request('GET', 'http://foo.com/201')];
$mock = new MockHandler([function (RequestInterface $request) {
return new Response(substr($request->getUri()->getPath(), 1));
}]);
$client = new Client(['handler' => $mock]);
$results = Pool::batch($client, $requests, ['fulfilled' => function ($value) use(&$called) {
$called = true;
}]);
$this->assertCount(2, $results);
$this->assertTrue($called);
}
/**
* @param array $requests
* @return array <Clickbus\DataTransfer\Response\Search\SearchInterface>
*/
public function sendSearchRequests(array $requests)
{
$responses = array();
// Results is a GuzzleHttp\BatchResults object.
$results = Pool::batch($this->httpClient, $requests);
foreach ($results as $result) {
$domXml = new \DOMDocument();
$domXml->loadXML($result->getBody()->getContents());
$schedules = $domXml->getElementsByTagName('Table');
$responses[] = ScheduleFactory::create($schedules);
}
return $responses;
}
public function execute()
{
$responses = Pool::batch($this->client->getHttpClient(), $this->requests);
return $this->parseResponse($responses);
}
/**
* Creates a new Request to this Endpoint.
*
* @param string[][] $queries
* @param null $url
* @param string $method
* @param array $options
* @return ApiResponse[]
* @throws ApiException
* @throws Response
*/
protected function requestMany(array $queries = [], $url = null, $method = 'GET', $options = [])
{
$requests = [];
$responses = [];
foreach ($queries as $query) {
$request = $this->createRequest($query, $url, $method, $options);
foreach ($this->handlers as $handler) {
$request = $handler->onRequest($request);
}
$requests[] = $request;
}
$results = Pool::batch($this->getClient(), $requests, ['pool_size' => 128]);
foreach ($results as $i => $response) {
/** @var Response|RequestException|\Exception $response */
$request = $requests[$i];
if ($response instanceof \Exception) {
if ($response instanceof RequestException && $response->hasResponse()) {
$response = $response->getResponse();
foreach ($this->handlers as $handler) {
$handler->onError($response, $request);
}
return $this->handleUnhandledError($response);
}
throw $response;
}
foreach ($this->handlers as $handler) {
$handler->onResponse($response, $request);
}
$responses[] = new ApiResponse($response);
}
return $responses;
}
public function performBatch()
{
$client = new \GuzzleHttp\Client(array(), array());
$time = microtime();
$time = explode(' ', $time);
$time = $time[1] + $time[0];
$start = $time;
$time = microtime();
$time = explode(' ', $time);
$time = $time[1] + $time[0];
$start = $time;
// does the batch operations
$responses = array();
$requests = array();
$this->logger->log(sprintf("starting batch"), DestinyLogger::debug);
foreach ($this->batch_items as $xlc) {
$headers = $xlc->xlcObject->headers;
$url = $xlc->url;
$post_data = $xlc->post_data;
if ($post_data) {
$this->logger->log(sprintf("adding post '%s' to batch", $url), DestinyLogger::debug);
$req = $client->createRequest('POST', $url, array('future' => false, 'debug' => false, 'body' => $post_data));
} else {
$this->logger->log(sprintf("adding get '%s' to batch", $url), DestinyLogger::debug);
$req = $client->createRequest('GET', $url, array('future' => false, 'debug' => false));
}
foreach ($headers as $header => $value) {
$req->setHeader($header, $value);
}
$requests[] = $req;
}
$time = microtime();
$time = explode(' ', $time);
$time = $time[1] + $time[0];
$start = $time;
$time = microtime();
$time = explode(' ', $time);
$time = $time[1] + $time[0];
$start = $time;
$results = \GuzzleHttp\Pool::batch($client, $requests);
$time = microtime();
$time = explode(' ', $time);
$time = $time[1] + $time[0];
$finish = $time;
$total_time = round($finish - $start, 3);
$this->logger->log(sprintf("batch of %s took %ss", count($requests), $total_time), DestinyLogger::debug);
foreach ($results->getSuccessful() as $response) {
$responses[] = array('body' => $response->getBody(), 'request' => $response->getEffectiveUrl());
}
foreach ($results->getFailures() as $failures) {
dpm($failures);
}
unset($this->batch_items);
return $responses;
}
/**
* @inheritdoc
*/
public function deleteObjects(array $objects)
{
$requests = [];
/** @var SwiftObject $object */
foreach ($objects as $object) {
if ($object->isPseudoDir()) {
continue;
}
$requests[] = $this->createRequest('delete', $object->getPath());
$this->logger->debug(sprintf('Deleting "%s"', $object->getPath()));
}
$numRemoved = 0;
if (!empty($requests)) {
$results = Pool::batch($this->client, $requests);
$error = false;
foreach ($results as $index => $response) {
if ($response instanceof ResponseInterface) {
if ($this->assertResponse($response, [204 => true])) {
++$numRemoved;
}
} elseif ($response instanceof \Exception) {
// skip 404s
if ($response instanceof BadResponseException && ($inner = $response->getResponse()) && $inner->getStatusCode() === 404) {
continue;
}
$error = true;
$this->logger->error(sprintf('Error deleting: %s', $response->getMessage()));
}
}
if ($error) {
throw new SwiftException('Could not delete all objects.');
}
}
$this->logger->info(sprintf('Deleted %d objects', $numRemoved));
return $numRemoved;
}
