/**
* Awaits all the requests attaching callbacks before.
*
* @return mixed
*/
public function unwrap()
{
//Deals with whole pool cache
if ($this->getCachePool() && $this->getCacheKey() && ($cacheItem = $this->getCachePool()->getItem($this->getCacheKey())) && $cacheItem->isHit()) {
return $cacheItem->get();
}
$promises = $this->flushPromises();
if (empty($promises)) {
return [];
}
foreach ($promises as $key => $promise) {
$promises[$key] = $this->attachTransforms($promise);
}
$res = \GuzzleHttp\Promise\unwrap($promises);
$res = array_merge($this->cacheData, $res);
//saves pool cache
if ($this->getCacheKey()) {
$cacheItem = $this->getCachePool()->getItem($this->getCacheKey());
$cacheItem->set($res);
}
// clearing so the pool can be reused.
$this->promises = [];
$this->cacheData = [];
$this->queries = [];
return $res;
}
public function testUnwrapsPromisesWithKeys()
{
$promises = ['foo' => new FulfilledPromise('a'), 'bar' => new FulfilledPromise('b')];
$this->assertEquals(['foo' => 'a', 'bar' => 'b'], \GuzzleHttp\Promise\unwrap($promises));
}
public static function purgeReValidation()
{
\GuzzleHttp\Promise\unwrap(static::$waitingRevalidate);
}
public function batch($requests, $options = [])
{
$this->lastError = null;
$this->lastRequest = null;
$this->lastResponse = null;
if (empty($this->accessKey)) {
$this->lastError = "No access key set. Please set an access key before attempting to fetch forecast data.";
}
if (!$this->httpClient) {
$this->httpClient = new \GuzzleHttp\Client();
}
$baseURL = rtrim(self::ENDPOINT, '/') . '/' . $this->accessKey . '/';
$httpRequestPromises = [];
foreach ($requests as $idx => $request) {
$url = $baseURL . $request['latitude'] . ',' . $request['longitude'];
if (!empty($request['time'])) {
$url .= ',' . $request['time'];
}
if ($options) {
$url .= '?' . http_build_query($options);
}
$requests[$idx]['url'] = $url;
$httpRequestPromises[$idx] = $this->httpClient->getAsync($url, ['allow_redirects' => true, 'timeout' => $this->timeout, 'connect_timeout' => $this->connectTimeout, 'headers' => ['User-Agent' => 'Tave/ForecastIOWrapper']]);
}
try {
$httpResults = \GuzzleHttp\Promise\unwrap($httpRequestPromises);
foreach ($httpResults as $idx => $httpResponse) {
if ($httpResponse instanceof \GuzzleHttp\Exception\RequestException) {
$this->lastRequest = $requests[$idx];
$this->lastResponse = $httpResponse;
$this->lastError = $httpResults[$resultIdx]->getMessage();
$requests[$idx]['error'] = $this->lastError;
} else {
$requests[$idx]['forecast'] = json_decode($httpResponse->getBody(), true);
}
}
return $requests;
} catch (\GuzzleHttp\Exception\RequestException $e) {
$this->lastRequest = $e->getRequest();
$this->lastResponse = $e->hasResponse() ? $e->getResponse() : null;
$this->lastError = $e->getMessage();
return false;
}
}
require __DIR__ . '/vendor/autoload.php';
use GuzzleHttp\Client;
use GuzzleHttp\TransferStats;
use GuzzleHttp\HandlerStack;
$stats = function (TransferStats $stats) {
echo $stats->getTransferTime() . " " . $stats->getEffectiveUri() . "\n";
};
// if you are going to use multiple clients, and want to make requests async among all of them
// you have to pass a handler into the client, so they all run in the same loop
$handler = HandlerStack::create();
$usernameClient = new Client(['base_uri' => 'http://localhost:8000/', 'timeout' => 10, 'on_stats' => $stats, 'handler' => $handler]);
$passwordClient = new Client(['base_uri' => 'http://localhost:8001/', 'timeout' => 10, 'on_stats' => $stats, 'handler' => $handler]);
$emailClient = new Client(['base_uri' => 'http://localhost:8002/', 'timeout' => 10, 'on_stats' => $stats, 'handler' => $handler]);
$start = microtime(true);
$username1 = $usernameClient->post('generate-username');
$username2 = $usernameClient->post('generate-username');
$password = $passwordClient->post('generate-password');
$email = $emailClient->post('generate-email');
$stop = microtime(true);
$time = $stop - $start;
echo "4 requests in {$time} seconds\n";
$start = microtime(true);
$promises = [];
$promises['username1'] = $usernameClient->postAsync('generate-username');
$promises['username2'] = $usernameClient->postAsync('generate-username');
$promises['password'] = $passwordClient->postAsync('generate-password');
$promises['email'] = $emailClient->postAsync('generate-email');
$results = GuzzleHttp\Promise\unwrap($promises);
$stop = microtime(true);
$time = $stop - $start;
echo "4 requests in {$time} seconds\n";
/**
* @return array
*/
protected function getResponses()
{
/**
* @var Response $response
*/
$responses = \GuzzleHttp\Promise\unwrap($this->promises);
$temp = [];
foreach ($responses as $key => $response) {
$temp[$key] = $this->getDecoder()->decode($response->getBody()->__toString());
}
return $temp;
}
/**
* @param $threads
*/
private function crawlPages($threads)
{
$promises = [];
for ($i = 0; $threads < count($promises) || $i < $threads; $i++) {
if (!($url = array_shift($this->queue))) {
break;
}
if (isset($this->scanned[$url])) {
continue;
}
$this->scanned[$url] = true;
if ($this->getUrlDepth($url) > $this->config->get('maxDepth')) {
continue;
}
$this->log('Start scan page ' . $url, 1);
try {
$promise = $this->loadPage($url, true);
} catch (TooManyRedirectsException $e) {
$this->log("Failed to load resource {$url} with error: " . $e->getMessage() . "\n", 2);
continue;
}
$promise->then(function (ResponseInterface $response) use($url, $threads) {
if ($lastModified = $response->getHeaderLine("Last-Modified")) {
$lastModified = \DateTime::createFromFormat("D, d M Y H:i:s O", $lastModified);
$lastModified = $lastModified ? $lastModified->format('Y-m-d\\TH:i:sP') : null;
} else {
$lastModified = null;
}
if (200 !== (int) $response->getStatusCode()) {
$this->log('Error! Url: ' . $url . ' Status: ' . $response->getStatusCode());
return false;
}
/**
* @var $pageUrl Url
*/
$pageUrl = $this->config->get('url');
$linksAmount = $this->storage->countLinks($pageUrl->getUrl());
if ($linksAmount >= $this->config->get('limits.linksTotal')) {
$this->queue = [];
return false;
}
$html = (string) $response->getBody();
$result = $this->parser->load($html);
$pageInfo = ['canonical' => $result['meta']['canonical'], 'metaRobots' => $result['meta']['robots'], "status" => $response->getStatusCode(), "lastModified" => $lastModified];
if (false === stripos($pageInfo['metaRobots'], 'noindex') && $this->checkContent($html)) {
if ($this->config->get('download.enable')) {
file_put_contents($this->config->get('download.directory') . '/' . urlencode($url) . '.tmp.html', $html);
}
$this->storage->addLink($pageUrl->getUrl(), $url, $pageInfo);
$this->log('Link ' . $url . ' added', 2);
}
if (false !== stripos($pageInfo['metaRobots'], 'nofollow')) {
$this->log("Page has meta tag nofollow\n", 2);
return false;
}
$links = $this->getCorrectLinks($result['links']);
foreach ($links as $link) {
$preparedLink = $this->prepare($link);
if (isset($this->scanned[$preparedLink])) {
continue;
} elseif (in_array($preparedLink, $this->queue)) {
continue;
} elseif ($this->getUrlDepth($preparedLink) > $this->config->get('maxDepth')) {
continue;
}
$this->queue[] = $preparedLink;
}
return true;
}, function (\Exception $e) use($url, $threads) {
if (0 === $e->getCode()) {
$this->queue[] = $url;
} elseif (in_array((int) $e->getCode(), [429, 503])) {
$this->queue[] = $url;
}
$this->log("Failed to load resource {$url} with error: " . $e->getMessage() . "\n", 2);
});
$promises[] = $promise;
}
try {
\GuzzleHttp\Promise\unwrap($promises);
} catch (TooManyRedirectsException $e) {
$this->log("Client error while resolving promise: " . $e->getMessage());
} catch (ClientException $e) {
$this->log("Client error while resolving promise: " . $e->getMessage());
} catch (ServerException $e) {
$this->log("Server error while resolving promise: " . $e->getMessage());
if (in_array((int) $e->getCode(), [429, 503])) {
$threads = $this->config->get('limits.threads');
if ($threads > 1) {
$this->config->set('limits.threads', $threads - 1);
}
$sleepTimeout = $this->config->get('timeout.sleep');
$this->config->set('limits.sleep', $sleepTimeout + 0.5);
$this->log(sprintf('Sleep for 5 minutes. Threads limit is decremented. Now is %d', $threads));
sleep(300);
}
} catch (ConnectException $e) {
$this->log("Connection error while resolving promise: " . $e->getMessage());
}
$this->executeCallback(true);
}
