Ejemplo n.º 1
0
 function parseDetails($urls)
 {
     $requests = function ($arr) {
         for ($i = 0; $i < count($arr); $i++) {
             (yield new Request('GET', $arr[$i]));
         }
     };
     $pool = new Pool($this->http, $requests($urls), ['concurrency' => 4, 'fulfilled' => function ($response, $index) use($urls) {
         echo "{$index} loaded\n";
         $this->html->load((string) $response->getBody());
         $detail = $this->parseDetails_($this->html);
         echo "Details parsed\n";
         $id = $this->parseId($urls[$index]);
         $this->details[$id] = $detail;
         $this->details[$id]['url'] = $urls[$index];
         $this->details[$id]['status'] = 'ok';
         //var_dump($this->details);
     }, 'rejected' => function ($reason, $index) {
         // this is delivered each failed request
         echo "{$index} rejected\n";
         $id = $this->parseId($urls[$index]);
         $this->details[$id]['url'] = $urls[$index];
         $this->details[$id]['status'] = $reason;
     }]);
     // Initiate the transfers and create a promise
     $promise = $pool->promise();
     // Force the pool of requests to complete.
     $promise->wait();
     return $this->details;
 }
 public function handle()
 {
     $this->totalPageCount = count($this->users);
     $client = new Client();
     $requests = function ($total) use($client) {
         foreach ($this->users as $key => $user) {
             $uri = 'https://api.github.com/users/' . $user;
             (yield function () use($client, $uri) {
                 return $client->getAsync($uri);
             });
         }
     };
     $pool = new Pool($client, $requests($this->totalPageCount), ['concurrency' => $this->concurrency, 'fulfilled' => function ($response, $index) {
         $res = json_decode($response->getBody()->getContents());
         $this->info("请求第 {$index} 个请求,用户 " . $this->users[$index] . " 的 Github ID 为:" . $res->id);
         $this->countedAndCheckEnded();
     }, 'rejected' => function ($reason, $index) {
         $this->error("rejected");
         $this->error("rejected reason: " . $reason);
         $this->countedAndCheckEnded();
     }]);
     // 开始发送请求
     $promise = $pool->promise();
     $promise->wait();
 }
Ejemplo n.º 3
0
 /**
  * Make all the requests and push responses (as objects if success) to self::responses["public" or "oauth"]
  */
 public static function run()
 {
     // Create Public and Oauth Clients
     self::$clientPublic = new Client(Endpoint::headers());
     self::$clientOauth = new Client(Endpoint::headers(TRUE));
     // Public
     $poolPublic = new Pool(self::$clientPublic, self::$requests["public"], ["concurrency" => 20, "fulfilled" => function (Response $response, $index) {
         // On success we json_decode response
         $data = json_decode($response->getBody()->getContents(), TRUE);
         // Create object and push it to responses success
         $factory = OpenCrest::getFactory();
         $object = $factory->create(self::$objects["public"][$index], $data, $response);
         array_push(self::$responses["public"]["success"], $object);
     }, "rejected" => function ($reason, $index) {
         // On failure we push it to responses rejected
         array_push(self::$responses["public"]["rejected"], ["reason" => $reason, "index" => $index]);
     }]);
     self::$promise["public"] = $poolPublic->promise();
     self::$promise["public"]->wait();
     // Oauth
     $poolOauth = new Pool(self::$clientOauth, self::$requests["oauth"], ["concurrency" => 20, "fulfilled" => function (Response $response, $index) {
         // On success we json_decode response
         $data = json_decode($response->getBody()->getContents(), TRUE);
         // Create object and push it to responses success
         $factory = OpenCrest::getFactory();
         $object = $factory->create(self::$objects["oauth"][$index], $data, $response);
         array_push(self::$responses["oauth"]["success"], $object);
     }, "rejected" => function ($reason, $index) {
         // On failure we push it to responses rejected
         array_push(self::$responses["oauth"]["rejected"], ["reason" => $reason, "index" => $index]);
     }]);
     self::$promise["oauth"] = $poolOauth->promise();
     self::$promise["oauth"]->wait();
 }
Ejemplo n.º 4
0
 /**
  * @param $pathToImportFile
  * @return array
  * @throws Exception
  */
 public function import($pathToImportFile)
 {
     $masterAccountsToImport = [];
     $subAccountsToImport = [];
     if (($handle = fopen($pathToImportFile, "r")) !== FALSE) {
         $this->validateImportFile($pathToImportFile);
         $failureLogName = tempnam(getcwd() . "/log_output", "account_import_failures");
         $failureLog = fopen($failureLogName, "w");
         $successLogName = tempnam(getcwd() . "/log_output", "account_import_successes");
         $successLog = fopen($successLogName, "w");
         $returnData = ['successes' => 0, 'failures' => 0, 'failure_log_name' => $failureLogName, 'success_log_name' => $successLogName];
         while (($data = fgetcsv($handle, 8096, ",")) !== FALSE) {
             try {
                 $payload = $this->buildPayload($data);
             } catch (Exception $e) {
                 throw new Exception("When building a payload for the following account, the payload generation failed with {$e->getMessage()}: " . implode(",", $data));
             }
             if (array_key_exists("sub_accounts", $payload)) {
                 array_push($subAccountsToImport, $data);
                 continue;
             }
             array_push($masterAccountsToImport, $data);
         }
         $allAccounts = array_merge($masterAccountsToImport, $subAccountsToImport);
         $requests = function () use($allAccounts) {
             foreach ($allAccounts as $account) {
                 (yield new Request("POST", $this->uri . "/api/v1/accounts", ['Content-Type' => 'application/json; charset=UTF8', 'timeout' => 30, 'Authorization' => 'Basic ' . base64_encode($this->username . ':' . $this->password)], json_encode($this->buildPayload($account))));
             }
         };
         $pool = new Pool($this->client, $requests(), ['concurrency' => 10, 'fulfilled' => function ($response, $index) use(&$returnData, $successLog, $failureLog, $allAccounts) {
             $statusCode = $response->getStatusCode();
             if ($statusCode > 201) {
                 $body = json_decode($response->getBody()->getContents());
                 $line = $allAccounts[$index];
                 array_push($line, $body);
                 fputcsv($failureLog, $line);
                 $returnData['failures'] += 1;
             } else {
                 $returnData['successes'] += 1;
                 fwrite($successLog, "Import succeeded for account ID {$allAccounts[$index][0]}" . "\n");
             }
         }, 'rejected' => function ($reason, $index) use(&$returnData, $failureLog, $allAccounts) {
             $response = $reason->getResponse();
             $body = json_decode($response->getBody()->getContents());
             $returnMessage = implode(", ", (array) $body->error->message);
             $line = $allAccounts[$index];
             array_push($line, $returnMessage);
             fputcsv($failureLog, $line);
             $returnData['failures'] += 1;
         }]);
         $promise = $pool->promise();
         $promise->wait();
     } else {
         throw new InvalidArgumentException("File could not be opened.");
     }
     fclose($failureLog);
     fclose($successLog);
     return $returnData;
 }
 /**
  * @param $pathToImportFile
  * @param bool $validateAddress
  * @return array
  */
 public function import($pathToImportFile, $validateAddress = false)
 {
     if (($handle = fopen($pathToImportFile, "r")) !== FALSE) {
         $this->validateImportFile($pathToImportFile);
         $failureLogName = tempnam(getcwd() . "/log_output", "account_secondary_address_import_failures");
         $failureLog = fopen($failureLogName, "w");
         $successLogName = tempnam(getcwd() . "/log_output", "account_secondary_address_import_successes");
         $successLog = fopen($successLogName, "w");
         $returnData = ['successes' => 0, 'failures' => 0, 'failure_log_name' => $failureLogName, 'success_log_name' => $successLogName];
         $validData = [];
         while (($data = fgetcsv($handle, 8096, ",")) !== FALSE) {
             try {
                 $this->buildPayload($data, (bool) $validateAddress);
             } catch (Exception $e) {
                 echo "Got an exception - {$e->getMessage()}\n";
                 continue;
             }
             array_push($validData, $data);
         }
         $requests = function () use($validData, $validateAddress) {
             foreach ($validData as $validDatum) {
                 if (count($validDatum[0]) > 0) {
                     (yield new Request("POST", $this->uri . "/api/v1/accounts/" . (int) trim($validDatum[0]) . "/addresses", ['Content-Type' => 'application/json; charset=UTF8', 'timeout' => 30, 'Authorization' => 'Basic ' . base64_encode($this->username . ':' . $this->password)], json_encode($this->buildPayload($validDatum, (bool) $validateAddress))));
                 }
             }
         };
         $pool = new Pool($this->client, $requests(), ['concurrency' => 10, 'fulfilled' => function ($response, $index) use(&$returnData, $successLog, $failureLog, $validData) {
             $statusCode = $response->getStatusCode();
             if ($statusCode > 201) {
                 $body = json_decode($response->getBody()->getContents());
                 $line = $validData[$index];
                 array_push($line, $body);
                 fputcsv($failureLog, $line);
                 $returnData['failures'] += 1;
             } else {
                 $returnData['successes'] += 1;
                 fwrite($successLog, "Secondary address import succeeded for account ID {$validData[$index][0]}" . "\n");
             }
         }, 'rejected' => function ($reason, $index) use(&$returnData, $failureLog, $validData) {
             $response = $reason->getResponse();
             $body = json_decode($response->getBody()->getContents());
             $returnMessage = implode(", ", (array) $body->error->message);
             $line = $validData[$index];
             array_push($line, $returnMessage);
             fputcsv($failureLog, $line);
             $returnData['failures'] += 1;
         }]);
         $promise = $pool->promise();
         $promise->wait();
     } else {
         throw new InvalidArgumentException("File could not be opened.");
     }
     fclose($failureLog);
     fclose($successLog);
     return $returnData;
 }
 public function executeRequests(array $requests)
 {
     set_time_limit(Settings::TIMEOUT_LIMIT_IN_SECONDS);
     ini_set('memory_limit', Settings::MEMORY_LIMIT);
     $this->resetFileHandles();
     $pool = new Pool($this->client, $requests, ['complete' => function (CompleteEvent $event) {
         $this->saveResponseToFile($event->getResponse());
     }, 'pool_size' => Settings::NB_OF_PARALLEL_REQUESTS]);
     $pool->wait();
     return $this->fileHandles;
 }
Ejemplo n.º 7
0
 /**
  * {@inheritdoc}
  */
 public static function getImagesInfo(array $urls, array $config = null)
 {
     $client = isset($config['client']) ? $config['client'] : new Client(['defaults' => static::$config]);
     $result = [];
     // Build parallel requests
     $requests = [];
     foreach ($urls as $url) {
         if (strpos($url['value'], 'data:') === 0) {
             if ($info = static::getEmbeddedImageInfo($url['value'])) {
                 $result[] = array_merge($url, $info);
             }
             continue;
         }
         $requests[] = $client->createRequest('GET', $url['value']);
     }
     // Execute in parallel
     $responses = Pool::batch($client, $requests);
     // Build result set
     foreach ($responses as $i => $response) {
         if ($response instanceof RequestException) {
             continue;
         }
         if (($size = getimagesizefromstring($response->getBody())) !== false) {
             $result[] = ['width' => $size[0], 'height' => $size[1], 'size' => $size[0] * $size[1], 'mime' => $size['mime']] + $urls[$i];
         }
     }
     return $result;
 }
Ejemplo n.º 8
0
 protected function execute(InputInterface $input, OutputInterface $output)
 {
     $config = $this->setConfiguraton($input);
     $client = new \GuzzleHttp\Client(['defaults' => ['allow_redirects' => false, 'timeout' => 5, 'connect_timeout' => 5]]);
     /** @var Instance $instance */
     foreach ($config->getInstances() as $instance) {
         $requests[] = $client->createRequest('HEAD', $instance->getUrl());
     }
     $options = [];
     Pool::send($client, $requests, ['complete' => function (CompleteEvent $event) {
     }, 'error' => function (ErrorEvent $event) use($config) {
         $instance = $config->findInstanceByUrl($event->getRequest()->getUrl());
         if ($instance == null) {
             throw new \RuntimeException('Instance not found');
         }
         if (!$event->getException()->hasResponse()) {
             $raven = new \Raven_Client($instance->getSentryDsn());
             $event_id = $raven->getIdent($raven->captureMessage(sprintf('The website %s with url -> %s is down or has a problem', $instance->getName(), $event->getRequest()->getUrl()), [], \Raven_Client::FATAL));
             if ($raven->getLastError() !== null) {
                 printf('There was an error sending the event to Sentry: %s', $raven->getLastError());
             }
             $error_handler = new \Raven_ErrorHandler($raven);
             $error_handler->registerExceptionHandler();
             $error_handler->registerErrorHandler();
             $error_handler->registerShutdownFunction();
         }
     }]);
 }
Ejemplo n.º 9
0
 /**
  * @inheritdoc
  */
 public function multiRequest(array $urls)
 {
     $client = new Client();
     $requests = array();
     foreach ($urls as $urlName => $urlData) {
         if (is_string($urlData)) {
             $urlData = array($urlData, array());
         }
         $urlOptions = new Options($urlData[1]);
         $method = $urlOptions->get('method', 'GET', 'up');
         $args = $urlOptions->get('args');
         $url = 'GET' === $method ? Url::addArg((array) $args, $urlData[0]) : $urlData[0];
         $requests[$urlName] = $client->createRequest($method, $url, $this->_getClientOptions($urlOptions, $method, $args));
     }
     $httpResults = Pool::batch($client, $requests);
     /** @var string $resName */
     /** @var Response $httpResult */
     $result = array();
     $index = 0;
     $keys = array_keys($urls);
     foreach ($keys as $resName) {
         $httpResult = $httpResults->offsetGet($index++);
         $result[$resName] = array($httpResult->getStatusCode(), $httpResult->getHeaders(), $httpResult->getBody()->getContents());
     }
     return $result;
 }
Ejemplo n.º 10
0
 /**
  * Download all wallpapers in list.
  *
  * @param string $directory Where to download wallpapers.
  *
  * @throws DownloadException Thrown if the download directory cannot be created.
  */
 public function downloadAll($directory)
 {
     if (!file_exists($directory)) {
         if (!@mkdir($directory, null, true)) {
             throw new DownloadException("The download directory cannot be created.");
         }
     }
     $client = new Client();
     $requests = [];
     foreach ($this->wallpapers as $w) {
         $url = $w->getImageUrl(true);
         $requests[] = $client->createRequest('GET', $url, ['save_to' => $directory . '/' . basename($url)]);
     }
     $results = Pool::batch($client, $requests);
     // Retry with PNG
     $retryRequests = [];
     foreach ($results->getFailures() as $e) {
         // Delete failed files
         unlink($directory . '/' . basename($e->getRequest()->getUrl()));
         $urlPng = str_replace('.jpg', '.png', $e->getRequest()->getUrl());
         $statusCode = $e->getResponse()->getStatusCode();
         if ($statusCode == 404) {
             $retryRequests[] = $client->createRequest('GET', $urlPng, ['save_to' => $directory . '/' . basename($urlPng)]);
         }
     }
     Pool::batch($client, $retryRequests);
 }
Ejemplo n.º 11
0
 /**
  * 通过guzzle发送并行请求
  *
  * @param  array $request_info 多个请求的地址
  * @return array  按请求的顺序以数字键为key,返回对应内容
  */
 public static function postConcurrent($request_info)
 {
     $ret = [];
     $requests = [];
     $client = self::getInstance();
     foreach ($request_info as $v) {
         //print_r($v['params']) . '<br>';
         $requests[] = $client->createRequest('post', $v['url'], isset($v['params']) ? $v['params'] : array());
     }
     // Results is a GuzzleHttp\BatchResults object.
     $results = Pool::batch($client, $requests);
     foreach ($results as $key => $response) {
         if (get_class($response) == 'GuzzleHttp\\Message\\Response') {
             $headers = $response->getHeaders();
             $contentType = $headers['Content-Type'][0];
             $contentType = explode(';', $contentType);
             $ret[$key]['contentType'] = $contentType[0];
             $ret[$key]['statusCode'] = $response->getStatusCode();
             if ($contentType[0] == 'application/json') {
                 $ret[$key]['contentType'] = 'json';
             }
             $ret[$key]['body'] = $response->getbody();
         } else {
             $ret[$key]['contentType'] = 'text/html';
             $ret[$key]['statusCode'] = $response->getCode();
             $ret[$key]['body'] = $response->getMessage();
         }
     }
     return $ret;
 }
Ejemplo n.º 12
0
 /**
  * @param string $url
  * @return array|mixed|null
  */
 public function get($url)
 {
     // Aenderungen an der Konfiguration invalidieren den Cache
     $cache_key = md5($url . $this->baseCacheKey);
     if (!filter_var($url, FILTER_VALIDATE_URL)) {
         return null;
     }
     if ($this->cache->hasItem($cache_key)) {
         return json_decode($this->cache->getItem($cache_key), true);
     }
     if (!$this->isValidDomain($url)) {
         return null;
     }
     $requests = array_map(function ($service) use($url) {
         /** @var ServiceInterface $service */
         return $service->getRequest($url);
     }, $this->services);
     $results = Pool::batch($this->client, $requests);
     $counts = array();
     $i = 0;
     foreach ($this->services as $service) {
         if (method_exists($results[$i], "json")) {
             try {
                 $counts[$service->getName()] = (int) $service->extractCount($results[$i]->json());
             } catch (\Exception $e) {
                 // Skip service if broken
             }
         }
         $i++;
     }
     $this->cache->setItem($cache_key, json_encode($counts));
     return $counts;
 }
Ejemplo n.º 13
0
/**
 * Fetch and process a paged response from JoindIn.
 *
 * @param $initial
 *   The initial URL to fetch. That URL may result in others getting
 *   fetched as well.
 * @param callable $processor
 *   A callable that will be used to process each page of results. Its signature
 *   must be: (\SplQueue $pages, ResponseInterface $response, $index)
 * @param int $concurrency
 *   The number of concurrent requests to send. In practice this kinda has to
 *   be 1, or else Guzzle will conclude itself before getting to later-added
 *   entries.
 */
function fetchPages($initial, callable $processor, $concurrency = 1)
{
    $client = getClient();
    $pages = new \SplQueue();
    $pages->enqueue($initial);
    $pageProcessor = partial($processor, $pages);
    $pageRequestGenerator = function (\SplQueue $pages) {
        foreach ($pages as $page) {
            (yield new Request('GET', $page));
        }
    };
    $pool = new Pool($client, $pageRequestGenerator($pages), ['concurrency' => $concurrency, 'fulfilled' => $pageProcessor]);
    // Initiate the transfers and create a promise
    $promise = $pool->promise();
    // Force the pool of requests to complete.
    $promise->wait();
}
Ejemplo n.º 14
0
 /**
  * {@inheritdoc}
  */
 protected function doSendInternalRequests(array $internalRequests, $success, $error)
 {
     $requests = array();
     foreach ($internalRequests as $internalRequest) {
         $requests[] = $this->createRequest($internalRequest, $success, $error);
     }
     class_exists('GuzzleHttp\\Pool') ? Pool::batch($this->client, $requests) : \GuzzleHttp\batch($this->client, $requests);
 }
Ejemplo n.º 15
0
 /**
  * @inheritdoc
  */
 public function sendBatch(BatchRequest $batchRequest) : array
 {
     $requests = array_map([$this, 'createRequest'], $batchRequest->getRequests());
     $batchResults = Pool::batch($this->guzzleClient, $requests);
     if ($batchResults->getFailures()) {
         throw new BatchRequestFailureException($batchRequest, $batchResults);
     }
     return $batchResults->getSuccessful();
 }
Ejemplo n.º 16
0
 public static function concurrent($request_info)
 {
     $requests = [];
     $client = new GuzzleHttp\Client();
     foreach ($request_info as $v) {
         $requests[] = $client->createRequest($v[0], $v[1], $v[2]);
     }
     return Pool::batch($client, $requests);
 }
 /**
  * @issue https://github.com/guzzle/guzzle/issues/867
  */
 public function testDoesNotFailInEventSystemForNetworkError()
 {
     $c = new Client();
     $r = $c->createRequest('GET', Server::$url, ['timeout' => 1, 'connect_timeout' => 1, 'proxy' => 'http://127.0.0.1:123/foo']);
     $events = [];
     $fn = function (AbstractTransferEvent $event) use(&$events) {
         $events[] = [get_class($event), $event->hasResponse(), $event->getResponse()];
     };
     $pool = new Pool($c, [$r], ['error' => $fn, 'end' => $fn]);
     $pool->wait();
     $this->assertCount(2, $events);
     $this->assertEquals('GuzzleHttp\\Event\\ErrorEvent', $events[0][0]);
     $this->assertFalse($events[0][1]);
     $this->assertNull($events[0][2]);
     $this->assertEquals('GuzzleHttp\\Event\\EndEvent', $events[1][0]);
     $this->assertFalse($events[1][1]);
     $this->assertNull($events[1][2]);
 }
Ejemplo n.º 18
0
 /**
  * @param \Destiny\DestinyRequest[]|\Destiny\DestinyRequest $requests
  * @throws \DestinyException
  * @throws \Exception
  *
  * @return array
  */
 public function request($requests)
 {
     $multi = true;
     if ($requests instanceof DestinyRequest) {
         $multi = false;
         $requests = [$requests];
     }
     $batch = $responses = [];
     foreach ($requests as $key => $request) {
         if (!$request instanceof DestinyRequest) {
             throw new Exception('Invalid request');
         }
         if (!CACHE_ENABLED || $request->raw) {
             Cache::forget($request->key);
         }
         if ($request->cache && Cache::has($request->key)) {
             $responses[$key] = Cache::get($request->key);
         } else {
             $req = $this->createRequest('GET', $request->url);
             $req->getEmitter()->attach($request);
             $batch[$key] = $req;
         }
     }
     if (count($batch)) {
         $keys = array_keys($batch);
         foreach (Pool::batch($this, $batch) as $i => $result) {
             $key = $keys[$i];
             $request = $requests[$key];
             if ($request instanceof DestinyRequest && $request->raw) {
                 $responses[$key] = $result;
                 continue;
             }
             if ($result instanceof Exception) {
                 Cache::forget($request->key);
                 throw new DestinyException($result->getMessage(), $result->getCode(), $result);
             }
             if ($result instanceof Response) {
                 $response = json_decode($result->getBody()->getContents(), true);
                 if (array_get($response, 'ErrorStatus') !== 'Success') {
                     Cache::forget($request->key);
                     Bugsnag::setMetaData(['bungie' => $response]);
                     throw new DestinyException(array_get($response, 'Message'), array_get($response, 'ErrorCode'));
                 }
                 $response = array_get($response, 'Response');
                 if (empty($response)) {
                     Cache::forget($request->key);
                 }
                 if ($request->cache) {
                     Cache::put($request->key, $response, $request->cache);
                 }
                 $responses[$key] = $response;
             }
         }
     }
     return $multi ? $responses : array_shift($responses);
 }
 public function sendRequestBatch(array $requests)
 {
     $responses = Pool::batch($this->client, $requests);
     foreach ($responses as $response) {
         if ($response instanceof PhpException) {
             throw $response;
         }
     }
     return $responses;
 }
Ejemplo n.º 20
0
 /**
  * @param array $data array($url=>$savePath)
  */
 public function downloadFiles(array $data)
 {
     trigger_error('Not yet implemented', E_USER_WARNING);
     return;
     $client = new Client();
     $requests = array();
     foreach ($data as $url => $path) {
         $requests[] = new Request('GET', HttpAdapterInterface::BASE_URL . $url, ['save_to' => $path]);
     }
     $pool = new Pool($client, $requests, ['concurrency' => 5, 'fulfilled' => function ($response, $index) {
         // this is delivered each successful response
     }, 'rejected' => function ($reason, $index) {
         // this is delivered each failed request
         //TODO error handling
     }]);
     // Initiate the transfers and create a promise
     $promise = $pool->promise();
     // Force the pool of requests to complete.
     $promise->wait();
 }
Ejemplo n.º 21
0
 public function requestsBatch($method, $uris)
 {
     $requests = array_map(function ($eventUri) use($method) {
         $this->validateMethod($method);
         return new Request($method, $this->uri($eventUri), ['Accept' => [self::FORMAT_EVENT_STORE_ATOM]]);
     }, $uris);
     $responses = Pool::batch($this->guzzle, $requests);
     return array_map(function (ResponseInterface $response) {
         return json_decode($response->getBody()->getContents(), true);
     }, $responses);
 }
Ejemplo n.º 22
0
 /**
  * {@inheritdoc}
  */
 protected function sendInternalRequests(array $internalRequests, $success, $error)
 {
     $requests = array();
     foreach ($internalRequests as $key => $internalRequest) {
         $requests[$key] = $this->createRequest($internalRequest);
     }
     $httpAdapter = $this;
     $pool = new Pool($this->client, $requests, array_merge($this->createOptions(), array('fulfilled' => function ($response, $index) use($success, $internalRequests, $httpAdapter) {
         $response = $httpAdapter->getConfiguration()->getMessageFactory()->createResponse((int) $response->getStatusCode(), $response->getProtocolVersion(), $response->getHeaders(), BodyNormalizer::normalize(function () use($response) {
             return $response->getBody()->detach();
         }, $internalRequests[$index]->getMethod()));
         $response = $response->withParameter('request', $internalRequests[$index]);
         call_user_func($success, $response);
     }, 'rejected' => function ($exception, $index) use($error, $internalRequests, $httpAdapter) {
         $exception = HttpAdapterException::cannotFetchUri($exception->getRequest()->getUri(), $httpAdapter->getName(), $exception->getMessage());
         $exception->setRequest($internalRequests[$index]);
         call_user_func($error, $exception);
     })));
     $pool->promise()->wait();
 }
Ejemplo n.º 23
0
 public static function prepare(Event $event)
 {
     $composer = $event->getComposer();
     $io = $event->getIO();
     // TODO: Remove this hack when composer supports loading autoload.files for plugins
     $vendorDir = $composer->getConfig()->get('vendor-dir');
     $composerDir = $vendorDir . '/composer';
     $includeFiles = (require $composerDir . '/autoload_files.php');
     foreach ($includeFiles as $file) {
         require $file;
     }
     // End of hack
     /** @var \Composer\Package\Package $package */
     $package = $composer->getPackage();
     $dependencies = [];
     $requires = $package->getRequires();
     /** @var \Composer\Package\Link $require */
     foreach ($requires as $require) {
         $dependencies[$require->getTarget()] = $require->getPrettyConstraint();
     }
     $devRequires = $package->getDevRequires();
     /** @var \Composer\Package\Link $require */
     foreach ($devRequires as $devRequire) {
         $dependencies[$devRequire->getTarget()] = $devRequire->getPrettyConstraint();
     }
     $extra = $package->getExtra();
     $proxyUrls = isset($extra['composer-proxy']['url']) ? $extra['composer-proxy']['url'] : null;
     if (!$proxyUrls) {
         $io->write('<warning>No composer proxy url defined in composer.json. You might not use your proxy when fetching packages.</warning>');
         return;
     }
     $io->write('<info>Sending data over to the defined proxies in order to cache your packages. Please wait, this may take a while...</info>');
     $client = new GuzzleClient();
     $requests = [];
     foreach ($proxyUrls as $proxyUrl) {
         $requests[] = $client->createRequest('POST', $proxyUrl . '/packages', ['body' => $dependencies]);
     }
     $pool = new Pool($client, $requests);
     $res = $pool->wait();
     $io->write('<info>Proxies caching completed!</info>');
 }
Ejemplo n.º 24
0
 /**
  * execute batch requests (post).
  *
  * @throws BatchRequestException
  *
  * @return Models\Responses\OrtcResponse
  */
 public function batchExecute()
 {
     $guzzleRequests = $this->createBatchPostRequests();
     $results = Pool::batch($this->guzzleClient, $guzzleRequests, ['pool_size' => $this->request->getOrtcConfig()->getBatchPoolSize()]);
     if (count($results->getFailures()) > 0) {
         $batchRequestException = new BatchRequestException();
         $batchRequestException->setResults($results);
         throw $batchRequestException;
     }
     $handler = $this->request->getResponseHandler();
     return $handler->handle($results);
 }
 public function fetch($responseFunction, $isMultithreaded = false, $threadCount = 1)
 {
     if ($isMultithreaded) {
         $requestList = [];
         foreach ($this->uriList as $uri) {
             $requestList[] = new Request('GET', $uri);
         }
         $uriPool = new Pool($this->guzzle, $requestList, ['concurrency' => $threadCount, 'fulfilled' => function (Response $response, $index) use($responseFunction) {
             $responseFunction($response->getBody()->getContents(), $index);
         }, 'rejected' => function () {
         }]);
         $promise = $uriPool->promise();
         $promise->wait();
     } else {
         $parsedPageCounter = 0;
         foreach ($this->uriList as $uri) {
             $response = $this->buzz->get($uri);
             $responseFunction($response->getContent(), ++$parsedPageCounter);
         }
     }
 }
Ejemplo n.º 26
0
 /**
  * run.
  *
  * @return array
  */
 public function run()
 {
     $urls = $this->urlProvider->getUrls();
     $client = $this->client;
     //This is a bit messie, need a refacto
     $resultCollection = new ResultCollection();
     $requests = function () use($urls, $client, $resultCollection) {
         foreach ($urls as $url) {
             (yield function () use($client, $url, $resultCollection) {
                 return $client->sendAsync(new Request($url->getMethod(), $url->getUrl(), $url->getHeaders()), ['timeout' => $url->getTimeout(), 'connect_timeout' => $url->getTimeout(), 'on_stats' => function (TransferStats $tranferStats) use($url, $resultCollection) {
                     $handlerError = null;
                     $validatorError = null;
                     $validatorResult = null;
                     if ($tranferStats->hasResponse()) {
                         $validatorResult = $url->getValidator()->check((string) $tranferStats->getResponse()->getBody());
                         if (false === $validatorResult) {
                             $validatorError = $url->getValidator()->getError();
                         }
                         $statusCode = $tranferStats->getResponse()->getStatusCode();
                         $transferTime = $tranferStats->getTransferTime();
                     } else {
                         // If we have a connection error
                         $statusCode = 400;
                         $transferTime = 0;
                         $handlerError = curl_strerror($tranferStats->getHandlerErrorData());
                     }
                     $resultCollection->offsetSet($url->getName(), new Result($url, $statusCode, $transferTime, $handlerError, $validatorResult, $validatorError));
                 }]);
             });
         }
     };
     $pool = new Pool($this->client, $requests(), ['concurrency' => 5]);
     $promise = $pool->promise();
     $promise->wait();
     return $resultCollection;
 }
Ejemplo n.º 27
0
 /**
  * @param array $data array($url=>$savePath)
  */
 public function downloadFiles(array $data)
 {
     $client = new Client();
     $requests = array();
     foreach ($data as $url => $path) {
         $requests[] = $client->createRequest('GET', HttpAdapterInterface::BASE_URL . $url, ['save_to' => $path]);
     }
     // Results is a GuzzleHttp\BatchResults object.
     $results = Pool::batch($client, $requests, ['pool_size' => 5]);
     // Retrieve all failures.
     foreach ($results->getFailures() as $requestException) {
         //TODO error handling
         throw new \Exception($requestException->getMessage());
     }
 }
Ejemplo n.º 28
0
 public function getCounts($url)
 {
     $requests = [];
     /** @var Provider $provider */
     foreach ($this->providers as $name => $provider) {
         $requests[$name] = $provider->getRequest($url);
     }
     $responses = Pool::batch($this->client, $requests);
     foreach ($requests as $name => $request) {
         $result = $responses->getResult($request);
         if ($result instanceof \Exception) {
             $requests[$name] = false;
         } else {
             $requests[$name] = $this->providers[$name]->getCount($result->getBody()->getContents(), $result);
         }
     }
     return $requests;
 }
Ejemplo n.º 29
0
 /**
  * @inheritDoc
  */
 public function batchLoad($requests)
 {
     /** @var IApiRequest[] $originals */
     $originals = [];
     $batchRequests = [];
     foreach ($requests as $i => $request) {
         if ($request instanceof IApiRequest) {
             $originals[$i] = $request;
             $batchRequests[$i] = $this->_makeRequest($request);
         }
     }
     $options = [];
     $responses = Pool::batch($this->_client, $batchRequests, $options);
     foreach ($responses as $id => $response) {
         $originals[$id]->setRawResult($this->_getResult($response));
     }
     return $this;
 }
Ejemplo n.º 30
0
 /**
  * @return array Array with 'ok' and 'failed' keys
  */
 public function send()
 {
     $responses = Pool::batch(new HttpClient(), $this->requests);
     $result = ['ok' => [], 'failed' => []];
     /** @var ResponseInterface $response */
     foreach ($responses as $key => $response) {
         if ($response instanceof RequestException) {
             /** @var RequestException $response */
             $result['failed'][] = ['exception' => $response->getMessage(), 'url' => $response->hasResponse() ? $response->getResponse()->getEffectiveUrl() : $response->getRequest()->getUrl(), 'object' => $this->objects[$key], 'status_code' => $response->hasResponse() ? $response->getResponse()->getStatusCode() : '0', 'reason' => $response->hasResponse() ? $response->getResponse()->getReasonPhrase() : ''];
             continue;
         }
         if (in_array($response->getStatusCode(), $this->validResponses)) {
             $result['ok'][] = $this->objects[$key];
             continue;
         }
         $result['failed'][] = ['url' => $response->getEffectiveUrl(), 'object' => $this->objects[$key], 'status_code' => $response->getStatusCode(), 'reason' => $response->getReasonPhrase()];
     }
     return $result;
 }