function parseDetails($urls)
{
$requests = function ($arr) {
for ($i = 0; $i < count($arr); $i++) {
(yield new Request('GET', $arr[$i]));
}
};
$pool = new Pool($this->http, $requests($urls), ['concurrency' => 4, 'fulfilled' => function ($response, $index) use($urls) {
echo "{$index} loaded\n";
$this->html->load((string) $response->getBody());
$detail = $this->parseDetails_($this->html);
echo "Details parsed\n";
$id = $this->parseId($urls[$index]);
$this->details[$id] = $detail;
$this->details[$id]['url'] = $urls[$index];
$this->details[$id]['status'] = 'ok';
//var_dump($this->details);
}, 'rejected' => function ($reason, $index) {
// this is delivered each failed request
echo "{$index} rejected\n";
$id = $this->parseId($urls[$index]);
$this->details[$id]['url'] = $urls[$index];
$this->details[$id]['status'] = $reason;
}]);
// Initiate the transfers and create a promise
$promise = $pool->promise();
// Force the pool of requests to complete.
$promise->wait();
return $this->details;
}
/**
* Make all the requests and push responses (as objects if success) to self::responses["public" or "oauth"]
*/
public static function run()
{
// Create Public and Oauth Clients
self::$clientPublic = new Client(Endpoint::headers());
self::$clientOauth = new Client(Endpoint::headers(TRUE));
// Public
$poolPublic = new Pool(self::$clientPublic, self::$requests["public"], ["concurrency" => 20, "fulfilled" => function (Response $response, $index) {
// On success we json_decode response
$data = json_decode($response->getBody()->getContents(), TRUE);
// Create object and push it to responses success
$factory = OpenCrest::getFactory();
$object = $factory->create(self::$objects["public"][$index], $data, $response);
array_push(self::$responses["public"]["success"], $object);
}, "rejected" => function ($reason, $index) {
// On failure we push it to responses rejected
array_push(self::$responses["public"]["rejected"], ["reason" => $reason, "index" => $index]);
}]);
self::$promise["public"] = $poolPublic->promise();
self::$promise["public"]->wait();
// Oauth
$poolOauth = new Pool(self::$clientOauth, self::$requests["oauth"], ["concurrency" => 20, "fulfilled" => function (Response $response, $index) {
// On success we json_decode response
$data = json_decode($response->getBody()->getContents(), TRUE);
// Create object and push it to responses success
$factory = OpenCrest::getFactory();
$object = $factory->create(self::$objects["oauth"][$index], $data, $response);
array_push(self::$responses["oauth"]["success"], $object);
}, "rejected" => function ($reason, $index) {
// On failure we push it to responses rejected
array_push(self::$responses["oauth"]["rejected"], ["reason" => $reason, "index" => $index]);
}]);
self::$promise["oauth"] = $poolOauth->promise();
self::$promise["oauth"]->wait();
}
public function handle()
{
$this->totalPageCount = count($this->users);
$client = new Client();
$requests = function ($total) use($client) {
foreach ($this->users as $key => $user) {
$uri = 'https://api.github.com/users/' . $user;
(yield function () use($client, $uri) {
return $client->getAsync($uri);
});
}
};
$pool = new Pool($client, $requests($this->totalPageCount), ['concurrency' => $this->concurrency, 'fulfilled' => function ($response, $index) {
$res = json_decode($response->getBody()->getContents());
$this->info("请求第 {$index} 个请求，用户 " . $this->users[$index] . " 的 Github ID 为：" . $res->id);
$this->countedAndCheckEnded();
}, 'rejected' => function ($reason, $index) {
$this->error("rejected");
$this->error("rejected reason: " . $reason);
$this->countedAndCheckEnded();
}]);
// 开始发送请求
$promise = $pool->promise();
$promise->wait();
}
/**
* @param $pathToImportFile
* @return array
* @throws Exception
*/
public function import($pathToImportFile)
{
$masterAccountsToImport = [];
$subAccountsToImport = [];
if (($handle = fopen($pathToImportFile, "r")) !== FALSE) {
$this->validateImportFile($pathToImportFile);
$failureLogName = tempnam(getcwd() . "/log_output", "account_import_failures");
$failureLog = fopen($failureLogName, "w");
$successLogName = tempnam(getcwd() . "/log_output", "account_import_successes");
$successLog = fopen($successLogName, "w");
$returnData = ['successes' => 0, 'failures' => 0, 'failure_log_name' => $failureLogName, 'success_log_name' => $successLogName];
while (($data = fgetcsv($handle, 8096, ",")) !== FALSE) {
try {
$payload = $this->buildPayload($data);
} catch (Exception $e) {
throw new Exception("When building a payload for the following account, the payload generation failed with {$e->getMessage()}: " . implode(",", $data));
}
if (array_key_exists("sub_accounts", $payload)) {
array_push($subAccountsToImport, $data);
continue;
}
array_push($masterAccountsToImport, $data);
}
$allAccounts = array_merge($masterAccountsToImport, $subAccountsToImport);
$requests = function () use($allAccounts) {
foreach ($allAccounts as $account) {
(yield new Request("POST", $this->uri . "/api/v1/accounts", ['Content-Type' => 'application/json; charset=UTF8', 'timeout' => 30, 'Authorization' => 'Basic ' . base64_encode($this->username . ':' . $this->password)], json_encode($this->buildPayload($account))));
}
};
$pool = new Pool($this->client, $requests(), ['concurrency' => 10, 'fulfilled' => function ($response, $index) use(&$returnData, $successLog, $failureLog, $allAccounts) {
$statusCode = $response->getStatusCode();
if ($statusCode > 201) {
$body = json_decode($response->getBody()->getContents());
$line = $allAccounts[$index];
array_push($line, $body);
fputcsv($failureLog, $line);
$returnData['failures'] += 1;
} else {
$returnData['successes'] += 1;
fwrite($successLog, "Import succeeded for account ID {$allAccounts[$index][0]}" . "\n");
}
}, 'rejected' => function ($reason, $index) use(&$returnData, $failureLog, $allAccounts) {
$response = $reason->getResponse();
$body = json_decode($response->getBody()->getContents());
$returnMessage = implode(", ", (array) $body->error->message);
$line = $allAccounts[$index];
array_push($line, $returnMessage);
fputcsv($failureLog, $line);
$returnData['failures'] += 1;
}]);
$promise = $pool->promise();
$promise->wait();
} else {
throw new InvalidArgumentException("File could not be opened.");
}
fclose($failureLog);
fclose($successLog);
return $returnData;
}
/**
* @param $pathToImportFile
* @param bool $validateAddress
* @return array
*/
public function import($pathToImportFile, $validateAddress = false)
{
if (($handle = fopen($pathToImportFile, "r")) !== FALSE) {
$this->validateImportFile($pathToImportFile);
$failureLogName = tempnam(getcwd() . "/log_output", "account_secondary_address_import_failures");
$failureLog = fopen($failureLogName, "w");
$successLogName = tempnam(getcwd() . "/log_output", "account_secondary_address_import_successes");
$successLog = fopen($successLogName, "w");
$returnData = ['successes' => 0, 'failures' => 0, 'failure_log_name' => $failureLogName, 'success_log_name' => $successLogName];
$validData = [];
while (($data = fgetcsv($handle, 8096, ",")) !== FALSE) {
try {
$this->buildPayload($data, (bool) $validateAddress);
} catch (Exception $e) {
echo "Got an exception - {$e->getMessage()}\n";
continue;
}
array_push($validData, $data);
}
$requests = function () use($validData, $validateAddress) {
foreach ($validData as $validDatum) {
if (count($validDatum[0]) > 0) {
(yield new Request("POST", $this->uri . "/api/v1/accounts/" . (int) trim($validDatum[0]) . "/addresses", ['Content-Type' => 'application/json; charset=UTF8', 'timeout' => 30, 'Authorization' => 'Basic ' . base64_encode($this->username . ':' . $this->password)], json_encode($this->buildPayload($validDatum, (bool) $validateAddress))));
}
}
};
$pool = new Pool($this->client, $requests(), ['concurrency' => 10, 'fulfilled' => function ($response, $index) use(&$returnData, $successLog, $failureLog, $validData) {
$statusCode = $response->getStatusCode();
if ($statusCode > 201) {
$body = json_decode($response->getBody()->getContents());
$line = $validData[$index];
array_push($line, $body);
fputcsv($failureLog, $line);
$returnData['failures'] += 1;
} else {
$returnData['successes'] += 1;
fwrite($successLog, "Secondary address import succeeded for account ID {$validData[$index][0]}" . "\n");
}
}, 'rejected' => function ($reason, $index) use(&$returnData, $failureLog, $validData) {
$response = $reason->getResponse();
$body = json_decode($response->getBody()->getContents());
$returnMessage = implode(", ", (array) $body->error->message);
$line = $validData[$index];
array_push($line, $returnMessage);
fputcsv($failureLog, $line);
$returnData['failures'] += 1;
}]);
$promise = $pool->promise();
$promise->wait();
} else {
throw new InvalidArgumentException("File could not be opened.");
}
fclose($failureLog);
fclose($successLog);
return $returnData;
}
/**
* Fetch and process a paged response from JoindIn.
*
* @param $initial
* The initial URL to fetch. That URL may result in others getting
* fetched as well.
* @param callable $processor
* A callable that will be used to process each page of results. Its signature
* must be: (\SplQueue $pages, ResponseInterface $response, $index)
* @param int $concurrency
* The number of concurrent requests to send. In practice this kinda has to
* be 1, or else Guzzle will conclude itself before getting to later-added
* entries.
*/
function fetchPages($initial, callable $processor, $concurrency = 1)
{
$client = getClient();
$pages = new \SplQueue();
$pages->enqueue($initial);
$pageProcessor = partial($processor, $pages);
$pageRequestGenerator = function (\SplQueue $pages) {
foreach ($pages as $page) {
(yield new Request('GET', $page));
}
};
$pool = new Pool($client, $pageRequestGenerator($pages), ['concurrency' => $concurrency, 'fulfilled' => $pageProcessor]);
// Initiate the transfers and create a promise
$promise = $pool->promise();
// Force the pool of requests to complete.
$promise->wait();
}
/**
* {@inheritdoc}
*/
protected function sendInternalRequests(array $internalRequests, $success, $error)
{
$requests = array();
foreach ($internalRequests as $key => $internalRequest) {
$requests[$key] = $this->createRequest($internalRequest);
}
$httpAdapter = $this;
$pool = new Pool($this->client, $requests, array_merge($this->createOptions(), array('fulfilled' => function ($response, $index) use($success, $internalRequests, $httpAdapter) {
$response = $httpAdapter->getConfiguration()->getMessageFactory()->createResponse((int) $response->getStatusCode(), $response->getProtocolVersion(), $response->getHeaders(), BodyNormalizer::normalize(function () use($response) {
return $response->getBody()->detach();
}, $internalRequests[$index]->getMethod()));
$response = $response->withParameter('request', $internalRequests[$index]);
call_user_func($success, $response);
}, 'rejected' => function ($exception, $index) use($error, $internalRequests, $httpAdapter) {
$exception = HttpAdapterException::cannotFetchUri($exception->getRequest()->getUri(), $httpAdapter->getName(), $exception->getMessage());
$exception->setRequest($internalRequests[$index]);
call_user_func($error, $exception);
})));
$pool->promise()->wait();
}
/**
* @param array $data array($url=>$savePath)
*/
public function downloadFiles(array $data)
{
trigger_error('Not yet implemented', E_USER_WARNING);
return;
$client = new Client();
$requests = array();
foreach ($data as $url => $path) {
$requests[] = new Request('GET', HttpAdapterInterface::BASE_URL . $url, ['save_to' => $path]);
}
$pool = new Pool($client, $requests, ['concurrency' => 5, 'fulfilled' => function ($response, $index) {
// this is delivered each successful response
}, 'rejected' => function ($reason, $index) {
// this is delivered each failed request
//TODO error handling
}]);
// Initiate the transfers and create a promise
$promise = $pool->promise();
// Force the pool of requests to complete.
$promise->wait();
}
public function fetch($responseFunction, $isMultithreaded = false, $threadCount = 1)
{
if ($isMultithreaded) {
$requestList = [];
foreach ($this->uriList as $uri) {
$requestList[] = new Request('GET', $uri);
}
$uriPool = new Pool($this->guzzle, $requestList, ['concurrency' => $threadCount, 'fulfilled' => function (Response $response, $index) use($responseFunction) {
$responseFunction($response->getBody()->getContents(), $index);
}, 'rejected' => function () {
}]);
$promise = $uriPool->promise();
$promise->wait();
} else {
$parsedPageCounter = 0;
foreach ($this->uriList as $uri) {
$response = $this->buzz->get($uri);
$responseFunction($response->getContent(), ++$parsedPageCounter);
}
}
}
/**
* run.
*
* @return array
*/
public function run()
{
$urls = $this->urlProvider->getUrls();
$client = $this->client;
//This is a bit messie, need a refacto
$resultCollection = new ResultCollection();
$requests = function () use($urls, $client, $resultCollection) {
foreach ($urls as $url) {
(yield function () use($client, $url, $resultCollection) {
return $client->sendAsync(new Request($url->getMethod(), $url->getUrl(), $url->getHeaders()), ['timeout' => $url->getTimeout(), 'connect_timeout' => $url->getTimeout(), 'on_stats' => function (TransferStats $tranferStats) use($url, $resultCollection) {
$handlerError = null;
$validatorError = null;
$validatorResult = null;
if ($tranferStats->hasResponse()) {
$validatorResult = $url->getValidator()->check((string) $tranferStats->getResponse()->getBody());
if (false === $validatorResult) {
$validatorError = $url->getValidator()->getError();
}
$statusCode = $tranferStats->getResponse()->getStatusCode();
$transferTime = $tranferStats->getTransferTime();
} else {
// If we have a connection error
$statusCode = 400;
$transferTime = 0;
$handlerError = curl_strerror($tranferStats->getHandlerErrorData());
}
$resultCollection->offsetSet($url->getName(), new Result($url, $statusCode, $transferTime, $handlerError, $validatorResult, $validatorError));
}]);
});
}
};
$pool = new Pool($this->client, $requests(), ['concurrency' => 5]);
$promise = $pool->promise();
$promise->wait();
return $resultCollection;
}
/**
* Validate the address in an account import
* @param $pathToImportFile
* @return array
*/
public function validate($pathToImportFile)
{
if (($handle = fopen($pathToImportFile, "r")) !== FALSE) {
$tempFile = tempnam(getcwd(), "validatedAddresses");
$tempHandle = fopen($tempFile, "w");
$this->validateImportFile($pathToImportFile);
$addressFormatter = new AddressFormatter();
$failureLogName = tempnam(getcwd() . "/log_output", "address_validator_failures");
$failureLog = fopen($failureLogName, "w");
$successLogName = tempnam(getcwd() . "/log_output", "address_validator_successes");
$successLog = fopen($successLogName, "w");
$returnData = ['validated_file' => $tempFile, 'successes' => 0, 'failures' => 0, 'failure_log_name' => $failureLogName, 'success_log_name' => $successLogName];
$addressesWithoutCounty = [];
$addressesWithCounty = [];
$validData = [];
while (($data = fgetcsv($handle, 8096, ",")) !== FALSE) {
array_push($addressesWithoutCounty, $this->cleanAddress($data, false));
array_push($addressesWithCounty, $this->cleanAddress($data, true));
array_push($validData, $data);
}
$requests = function () use($addressesWithoutCounty) {
foreach ($addressesWithoutCounty as $addressWithoutCounty) {
(yield new Request("POST", $this->uri . "/api/v1/_data/validate_address", ['Content-Type' => 'application/json; charset=UTF8', 'timeout' => 30, 'Authorization' => 'Basic ' . base64_encode($this->username . ':' . $this->password)], json_encode($addressWithoutCounty)));
}
};
$pool = new Pool($this->client, $requests(), ['concurrency' => 10, 'fulfilled' => function ($response, $index) use(&$returnData, $successLog, $failureLog, $validData, $tempHandle, $addressFormatter, $addressesWithCounty) {
$statusCode = $response->getStatusCode();
if ($statusCode > 201) {
//Need to check if the address with county is valid so we can at least use that
try {
$addressAsArray = $addressFormatter->doChecksOnUnvalidatedAddress($addressesWithCounty[$index]);
$returnData['successes'] += 1;
fwrite($successLog, "Validation succeeded for ID {$validData[$index][0]}" . "\n");
fputcsv($tempHandle, $this->mergeRow($addressAsArray, $validData[$index]));
} catch (Exception $e) {
$body = json_decode($response->getBody()->getContents());
$line = $validData[$index];
array_push($line, $body);
fputcsv($failureLog, $line);
$returnData['failures'] += 1;
}
} else {
$returnData['successes'] += 1;
fwrite($successLog, "Validation succeeded for ID {$validData[$index][0]}" . "\n");
$addressObject = json_decode($response->getBody()->getContents());
$addressAsArray = (array) $addressObject->data;
fputcsv($tempHandle, $this->mergeRow($addressAsArray, $validData[$index]));
}
}, 'rejected' => function ($reason, $index) use(&$returnData, $failureLog, $validData, $addressFormatter, $addressesWithCounty, $successLog, $tempHandle) {
//Need to check if the address with county is valid so we can at least use that
try {
$addressAsArray = $addressFormatter->doChecksOnUnvalidatedAddress($addressesWithCounty[$index]);
$returnData['successes'] += 1;
fwrite($successLog, "Validation succeeded for ID {$validData[$index][0]}" . "\n");
fputcsv($tempHandle, $this->mergeRow($addressAsArray, $validData[$index]));
} catch (Exception $e) {
$response = $reason->getResponse();
$body = json_decode($response->getBody()->getContents());
$returnMessage = implode(", ", (array) $body->error->message);
$line = $validData[$index];
array_push($line, $returnMessage);
fputcsv($failureLog, $line);
$returnData['failures'] += 1;
}
}]);
$promise = $pool->promise();
$promise->wait();
} else {
throw new InvalidArgumentException("File could not be opened.");
}
fclose($tempHandle);
fclose($failureLog);
fclose($successLog);
return $returnData;
}
/**
* Uses the Guzzel Pool to process Requests.
*
* Processes with a concurrency of 18, since Eve limits to 20.
*
* Uses a promise and will wait until finished.
*
* @param callback function - used for rejected responses. $reason and $index are injected.
*/
public function fetchGroupsInfromation($rejectedCallbackFunction)
{
$pool = new Pool($this->client, $this->createdRequests, ['concurrency' => 18, 'fulfilled' => function ($response, $index) {
$responseJson = json_decode($response->getBody()->getContents());
$groupPagesIterator = new MarketGroupsPagesIterator($responseJson, $this->client);
$this->acceptedResponses[$index] = iterator_to_array($groupPagesIterator->getAllPages());
}, 'rejected' => function ($reason, $index) use(&$rejectedCallbackFunction) {
call_user_func_array($rejectedCallbackFunction, array($reason, $index));
}]);
$promise = $pool->promise();
$promise->wait();
}
public function testUsesYieldedKeyInFulfilledCallback()
{
$r1 = new Promise(function () use(&$r1) {
$r1->resolve(new Response());
});
$r2 = new Promise(function () use(&$r2) {
$r2->resolve(new Response());
});
$r3 = new Promise(function () use(&$r3) {
$r3->resolve(new Response());
});
$handler = new MockHandler([$r1, $r2, $r3]);
$c = new Client(['handler' => $handler]);
$keys = [];
$requests = ['request_1' => new Request('GET', 'http://example.com'), 'request_2' => new Request('GET', 'http://example.com'), 'request_3' => new Request('GET', 'http://example.com')];
$p = new Pool($c, $requests, ['pool_size' => 2, 'fulfilled' => function ($res, $index) use(&$keys) {
$keys[] = $index;
}]);
$p->promise()->wait();
$this->assertCount(3, $keys);
$this->assertSame($keys, array_keys($requests));
}
/**
* @param OutputInterface $output
* @param ItemId[] $itemIds
* @param bool $force
*/
private function executeForItemIds(OutputInterface $output, array $itemIds, $force)
{
$itemLookup = $this->wikibaseFactory->newItemLookup();
$processedItemIdStrings = $this->getProcessedItemIdStrings();
$loopCounter = 0;
/** @var FormatterHelper $formatter */
$formatter = $this->getHelper('formatter');
foreach ($itemIds as $itemId) {
$loopCounter++;
$itemIdString = $itemId->getSerialization();
$output->writeln('----------------------------------------------------');
if ($loopCounter % 10 != 0) {
$processedItemIdStrings = $this->getProcessedItemIdStrings();
}
if (!$force && in_array($itemId->getSerialization(), $processedItemIdStrings)) {
$output->writeln($formatter->formatSection($itemIdString, 'Already processed'));
continue;
}
try {
$output->writeln($formatter->formatSection($itemIdString, 'Loading Item'));
$item = $itemLookup->getItemForId($itemId);
} catch (ItemLookupException $e) {
$output->writeln($formatter->formatSection($itemIdString, 'Failed to load item (exception)', 'error'));
continue;
}
if ($item === null) {
$output->writeln($formatter->formatSection($itemIdString, 'Failed to load item (null)', 'error'));
continue;
}
// Get the item types..
$types = array();
foreach ($item->getStatements()->getByPropertyId(new PropertyId('P31'))->toArray() as $instanceStatement) {
$mainSnak = $instanceStatement->getMainSnak();
if ($mainSnak instanceof PropertyValueSnak) {
/** @var EntityIdValue $instanceItemIdValue */
$instanceItemIdValue = $mainSnak->getDataValue();
$idSerialization = $instanceItemIdValue->getEntityId()->getSerialization();
if (array_key_exists($idSerialization, $this->instanceMap)) {
$types[] = $this->instanceMap[$idSerialization];
}
}
}
if (empty($types)) {
$output->writeln($formatter->formatSection($itemIdString, 'Didn\\t find any useful instance of statements', 'comment'));
continue;
}
// Note: only load Wikipedias
$siteLinkList = DataModelUtils::getSitelinksWiteSiteIdSuffix($item->getSiteLinkList(), 'wiki');
$output->writeln($formatter->formatSection($itemIdString, $siteLinkList->count() . ' Wikipedia pages to request'));
$parseProgressBar = new ProgressBar($output, $siteLinkList->count());
$parseProgressBar->display();
/** @var PromiseInterface[] $parsePromises */
$parsePromises = array();
foreach ($siteLinkList->getIterator() as $siteLink) {
$siteId = $siteLink->getSiteId();
$pageName = $item->getSiteLinkList()->getBySiteId($siteId)->getPageName();
$sourceMwFactory = $this->wmFactoryFactory->getFactory($siteId);
$sourceParser = $sourceMwFactory->newParser();
$pageIdentifier = new PageIdentifier(new Title($pageName));
$parsePromises[$siteId] = $sourceParser->parsePageAsync($pageIdentifier);
$parseProgressBar->advance();
}
$links = array();
foreach ($parsePromises as $siteId => $promise) {
try {
$parseResult = $promise->wait();
if (array_key_exists('externallinks', $parseResult)) {
foreach ($parseResult['externallinks'] as $externalLink) {
// Ignore archive.org links
if (strstr($externalLink, 'archive.org') === false) {
$links[] = $this->normalizeExternalLink($externalLink);
}
}
}
} catch (Exception $e) {
$parseProgressBar->clear();
$output->writeln($formatter->formatSection($itemIdString, $e->getMessage(), 'error'));
$parseProgressBar->display();
// Ignore failed requests
}
}
$parseProgressBar->finish();
$output->writeln('');
$links = array_unique($links);
shuffle($links);
/** @var Request[] $linkRequests */
$linkRequests = array();
foreach ($links as $link) {
$linkRequests[] = new Request('GET', $link, array('allow_redirects' => array('track_redirects' => true), 'connect_timeout' => 3.14, 'timeout' => 10));
}
$output->writeln($formatter->formatSection($itemIdString, count($linkRequests) . ' External links to (download, action)'));
if (empty($linkRequests)) {
continue;
}
// Make a bunch of requests and act on the responses
$referencesAddedToItem = 0;
$externalLinkProgressBar = new ProgressBar($output, count($linkRequests) * 2);
$externalLinkProgressBar->display();
$pool = new Pool($this->externalLinkClient, $linkRequests, array('fulfilled' => function ($response) use($externalLinkProgressBar, $item, $types, $referencesAddedToItem, $output) {
$externalLinkProgressBar->advance();
// 1st advance point
if ($response instanceof ResponseInterface) {
$link = $response->getHeaderLine('X-GUZZLE-EFFECTIVE-URL');
$html = $response->getBody();
$referencesAddedFromLink = 0;
foreach ($this->microDataExtractor->extract($html) as $microData) {
foreach ($types as $type) {
if ($microData->hasType($type) && array_key_exists($type, $this->referencerMap)) {
foreach ($this->referencerMap[$type] as $referencer) {
/** @var Referencer $referencer */
$addedReferences = $referencer->addReferences($microData, $item, $link);
$referencesAddedToItem += $addedReferences;
$referencesAddedFromLink += $addedReferences;
}
}
}
}
if ($referencesAddedFromLink > 0) {
$externalLinkProgressBar->clear();
$output->write("\r");
$output->writeln($referencesAddedFromLink . ' reference(s) added from ' . urldecode($link));
$externalLinkProgressBar->display();
}
}
$externalLinkProgressBar->advance();
// 2nd advance point
}, 'rejected' => function () use($externalLinkProgressBar) {
// TODO add this to some kind of verbose log?
$externalLinkProgressBar->advance();
// 1st advance point
}));
$pool->promise()->wait();
$externalLinkProgressBar->finish();
$output->writeln('');
$output->writeln($formatter->formatSection($itemIdString, $referencesAddedToItem . ' References added'));
$this->markIdAsProcessed($itemId);
}
}
/**
* @param Request[] $requests
* @param array $options
* @return PromiseInterface
*/
public function getPool(array $requests, $options)
{
$pool = new Pool($this->client, $requests, $options);
return $pool->promise();
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$registry = $this->getContainer()->get('doctrine');
$log = $this->getContainer()->get('logger');
$em = $registry->getManager();
$eveRegistry = $this->getContainer()->get('evedata.registry');
$regionRepo = $eveRegistry->get('EveBundle:Region');
$items = $eveRegistry->get('EveBundle:ItemType')->findAllMarketItems();
$chunked_items = array_chunk($items, count($items) / 30);
// regions we actually need
$configs = $registry->getManager()->getRepository('AppBundle:BuybackConfiguration')->findBy(['type' => BuybackConfiguration::TYPE_REGION]);
$neededRegions = array_reduce($configs, function ($carry, $value) {
if ($carry === null) {
return $value->getRegions();
}
return array_merge($carry, $value->getRegions());
});
if (!$neededRegions) {
$log->addInfo('No regions to update');
return;
}
$log->addDebug('Beginning Import');
$client = new Client();
$requests = function ($region, $items) {
foreach ($items as $i) {
(yield new Request('GET', $this->getCrestUrl($region, $i['typeID'])));
}
};
foreach ($neededRegions as $region) {
$real_region = $regionRepo->getRegionById($region);
foreach ($chunked_items as $items) {
$progress = new ProgressBar($output, count($items));
$progress->setFormat('<comment> %current%/%max% </comment>[%bar%] %percent:3s%% %elapsed:6s%/%estimated:-6s% <question>%memory:6s%</question> <info> Updating %message% </info>');
list($processableData, $index) = $this->buildIndex($items);
$progress->setMessage($real_region['regionName']);
$pool = new Pool($client, $requests($region, $items), ['fulfilled' => function ($response, $i) use(&$processableData, $progress, $log) {
$obj = json_decode($response->getBody()->getContents(), true);
$processableData[$i] = array_pop($obj['items']);
$progress->advance();
}, 'rejected' => function ($reason, $i) use($log, $progress, $real_region) {
$errors[$i] = $reason;
$progress->setMessage(sprintf("{$real_region['regionName']} with %s errors", count($errors)));
$progress->advance();
}]);
$promise = $pool->promise();
$promise->wait();
$progress->finish();
$progress = new ProgressBar($output, count($processableData));
$progress->setFormat('<comment> %current%/%max% </comment>[%bar%] %percent:3s%% <question>%memory:6s%</question> <info> Updating Database </info>');
$count = 0;
foreach ($processableData as $i => $processableItem) {
if (is_array($processableItem) && isset($index[$i])) {
$exists = $em->getRepository('AppBundle:ItemPrice')->hasItem($real_region['regionID'], $index[$i]['typeID']);
if ($exists instanceof ItemPrice) {
$p = $this->updatePriceData($processableItem, $exists);
$log->addDebug("Updating item {$p->getTypeName()} in {$p->getRegionName()}");
} else {
$p = $this->makePriceData($processableItem, $real_region, $index[$i]);
$log->addDebug("Adding item {$p->getTypeName()} in {$p->getRegionName()}");
}
$progress->advance();
$em->persist($p);
++$count;
if ($count % (count($processableData) / 20) === 0) {
$log->addDebug('Flushing Set');
$em->flush();
$em->clear();
}
}
}
$progress->finish();
$em->flush();
}
}
}
$html->load($text);
$table = $html->find('body>center>table>tr', 2);
echo $table->outertext;
$table1 = $table->find('table td', 4);
//echo $table1->outertext;
//$table2=$table1->find('table td',1);
if ($table1 == null) {
echo "Warning: format error!\n";
return;
}
$res = $table1->find('td a img');
foreach ($res as $img) {
$a = $img->parent();
if (isset($a->href)) {
$productsLinks[] = $a->href;
} else {
echo "Warning: no href!\n";
}
}
print_r($productsLinks);
die;
}
$productsLinks = [];
// Initiate the transfers and create a promise
$promise = $pool->promise();
// Force the pool of requests to complete.
$promise->wait();
print_r($productsLinks);
?>
</body>
</html>
/**
* Do one or more requests simultaniously.
*
* @param string|array $paths
* @param Closure|array $callback
*/
protected function request($paths, $callback)
{
/** @var Request[] $requests */
$requests = [];
$callbacks = [];
// We need to keep the same indeces for both the request and
// the callback. We lose the reference to the request in the
// asynchronous request pool, so the index is very important
// for matching the right callback.
foreach ((array) $paths as $path) {
$requests[] = $this->createRequest($path);
$callbacks[] = $callback;
}
// Handle each request simultanious.
$pool = new Pool($this->client, $requests, ['concurrency' => $this->concurrency, 'fulfilled' => function (Response $response, $index) use($callbacks, $requests) {
// Setup a crawler instance
$crawler = new Crawler();
$crawler->addContent($response->getBody(), $response->getHeaderLine('Content-Type'));
$request = $requests[$index];
try {
// Perform the callback, fetching data from the DOM.
call_user_func_array($callbacks[$index], [$crawler, $request]);
$this->log($request, 'success', 'Callback complete');
} catch (\Exception $e) {
$this->logException($e, $request, compact('crawler'));
}
}, 'rejected' => function (\Exception $e, $index) use($requests) {
$request = $requests[$index];
$this->logException($e, $request);
}]);
// Initiate the transfers and create a promise
$promise = $pool->promise();
// Force the pool of requests to complete.
$promise->wait();
}
/**
* Concurrently delete the files.
*
* @param array $files
* Array of file ID's.
* @param OutputInterface $output
* Console output interface.
*/
protected function deleteFiles(array $files, OutputInterface $output)
{
$requests = function ($files) {
$uri = "https://slack.com/api/files.delete";
foreach ($files as $file_id) {
$req = new Request('POST', $uri);
$modify['body'] = Psr7\stream_for(http_build_query(['file' => $file_id, 'token' => getenv('SLACK_AUTH_TOKEN')]));
$modify['set_headers']['Content-Type'] = 'application/x-www-form-urlencoded';
$modify['set_headers']['Cache-Control'] = 'no-cache';
$request = Psr7\modify_request($req, $modify);
(yield $request);
}
};
$pool = new Pool($this->client, $requests($files), ['concurrency' => self::CONCURRENT_REQUESTS, 'fulfilled' => function ($response, $index) use($output) {
$output->writeln("<info>File deleted</info>");
}, 'rejected' => function ($reason, $index) use($output) {
$output->writeln("<error>Error deleting file {$reason}</error>");
}]);
$promise = $pool->promise();
$promise->wait();
}
/**
* Get the item history for the a region.
*
* You can specificy the response json items to come back. This means
* that you can say something like -20, this will give you the last 20
* items of the array of response->items.
*
* The second argument is a callback that takes the following paramters:
*
* - array regionItemPairs, example: [0, 1] where 0 is the region id and 1 is the item id.
* - json object responseJson which is the response json from the promise.
*
* This call back function can then be used to do what ever you wish with the data backing back.
* the region and item pairs map to the response json in question.
*
* @param $howManyItemsBack How many items should we get back?
* @param $successCallbackFunction the function which takes an array of region id and item id and a responseJson object.
* @param $rejectedCallbackFunction the function to handle rejected response. $reason and $index are injected into the function.
*/
public function getItemHistoryForRegion($howManyItemsBack, $successCallbackFunction, $rejectedCallbackFunction)
{
$pool = new Pool($this->client, $this->createdRequests, $this->getOptions($howManyItemsBack, $successCallbackFunction, $rejectedCallbackFunction));
$promise = $pool->promise();
$promise->wait();
}
/**
* @param $pathToImportFile
* @return array
*/
public function import($pathToImportFile)
{
if (($handle = fopen($pathToImportFile, "r")) !== FALSE) {
$this->validateImportFile($pathToImportFile);
$this->getExistingSubnets();
$failureLogName = tempnam(getcwd() . "/log_output", "ip_pool_import_failures");
$failureLog = fopen($failureLogName, "w");
$successLogName = tempnam(getcwd() . "/log_output", "ip_pool_import_successes");
$successLog = fopen($successLogName, "w");
$returnData = ['successes' => 0, 'failures' => 0, 'failure_log_name' => $failureLogName, 'success_log_name' => $successLogName];
$validData = [];
while (($data = fgetcsv($handle, 8096, ",")) !== FALSE) {
$lethStart = Address::factory(trim($data[1]));
$lethEnd = Address::factory(trim($data[2]));
$foundAFit = false;
foreach ($this->subnets as $subnet) {
try {
if ($subnet['object']->encloses_address($lethStart) && $subnet['object']->encloses_address($lethEnd)) {
array_push($data, $subnet['id']);
array_push($data, $subnet['supernet_id']);
array_push($validData, $data);
$foundAFit = true;
break;
}
} catch (Exception $e) {
//
}
}
if ($foundAFit === false) {
array_push($data, "IP pool did not fit into any defined subnets.");
fputcsv($failureLog, $data);
$returnData['failures'] += 1;
}
}
$requests = function () use($validData) {
foreach ($validData as $validDatum) {
$supernetID = array_pop($validDatum);
$subnetID = array_pop($validDatum);
(yield new Request("POST", $this->uri . "/api/v1/network/ipam/supernets/{$supernetID}/subnets/{$subnetID}/ip_pools", ['Content-Type' => 'application/json; charset=UTF8', 'timeout' => 30, 'Authorization' => 'Basic ' . base64_encode($this->username . ':' . $this->password)], json_encode($this->buildPayload($validDatum))));
}
};
$pool = new Pool($this->client, $requests(), ['concurrency' => 10, 'fulfilled' => function ($response, $index) use(&$returnData, $successLog, $failureLog, $validData) {
$statusCode = $response->getStatusCode();
if ($statusCode > 201) {
$body = json_decode($response->getBody()->getContents());
$line = $validData[$index];
array_push($line, $body);
fputcsv($failureLog, $line);
$returnData['failures'] += 1;
} else {
$returnData['successes'] += 1;
fwrite($successLog, "Import succeeded for account ID {$validData[$index][0]}" . "\n");
}
}, 'rejected' => function ($reason, $index) use(&$returnData, $failureLog, $validData) {
$response = $reason->getResponse();
$body = json_decode($response->getBody()->getContents());
$returnMessage = implode(", ", (array) $body->error->message);
$line = $validData[$index];
array_push($line, $returnMessage);
fputcsv($failureLog, $line);
$returnData['failures'] += 1;
}]);
$promise = $pool->promise();
$promise->wait();
} else {
throw new InvalidArgumentException("File could not be opened.");
}
fclose($failureLog);
fclose($successLog);
return $returnData;
}
/**
* @param int $buffer
* @throws \Exception
*/
public function flush($buffer = 10)
{
$psr7Requests = [];
$flushedPromises = [];
foreach ($this->promises as $requestId => $promise) {
$event = $this->getEventDispatcher()->dispatch(sprintf('%s.BEFORE', $requestId), new RequestEvent($promise));
if (!$event->isPropagationStopped() && $event->getResponse()) {
$this->getEventDispatcher()->dispatch(sprintf('%s.SUCCESS', $requestId), new RequestEvent($promise, $event->getResponse()));
$this->removePromise($requestId);
}
}
# Transform our proprietary promises to PSR7-requests
foreach ($this->promises as $requestId => $promise) {
$psr7Requests[$requestId] = $this->fromHttpFoundationToPSR7($promise->getRequest());
}
if ($this->promises) {
# Create a pool of promises
$pool = new Pool($this->client, $psr7Requests, ['concurrency' => $buffer, 'fulfilled' => function (PSR7Response $psr7Response, $index) use(&$flushedPromises) {
$promise = $this->getPromiseByIndex($index);
$requestId = $promise->getRequestId();
$response = $this->getHttpFoundationFactory()->createResponse($psr7Response);
$this->getEventDispatcher()->dispatch(sprintf('%s.SUCCESS', $requestId), new RequestEvent($promise, $response));
$flushedPromises[] = $requestId;
}, 'rejected' => function (BadResponseException $exception, $index) use(&$flushedPromises) {
$promise = $this->getPromiseByIndex($index);
$requestId = $promise->getRequestId();
$psr7Response = $exception->getResponse();
$response = $this->getHttpFoundationFactory()->createResponse($psr7Response);
$this->eventDispatcher->dispatch(sprintf('%s.ERROR', $requestId), new RequestEvent($promise, $response, $exception));
$flushedPromises[] = $requestId;
}]);
# Initiate the transfers and create a promise
$promise = $pool->promise();
# Force the pool of requests to complete.
$promise->wait();
}
# Remove promises from the pool
foreach ($flushedPromises as $requestId) {
$this->removePromise($requestId);
}
# If new promises have been set during callbacks, flush again
if ($this->promises) {
$this->flush($buffer);
}
}
public function processSubmissionsRequest(SubmissionsRequest $request)
{
$asynch = false;
//this variable will determine whether we'll send the request asynchronously or not.
if (!isset($_SESSION)) {
session_start();
}
$domain = $_SESSION['domain'];
$token = \Crypt::decrypt($_SESSION['userToken']);
$courseId = $_SESSION['courseID'];
$userId = $_SESSION['userID'];
$urlPieces = array();
$urlArgs = array();
$urlPieces[] = "https://{$domain}/api/v1/courses/{$courseId}";
//MULTIPLE ASSIGNMENTS AND POTENTIALLY MULTIPLE USERS
if ($request->getMultipleAssignments()) {
//GET /api/v1/courses/:course_id/students/submissions
$urlPieces[] = "students/submissions";
//STUDENT IDS
//student_ids can be "All", or a list of actual studentIds
if ($request->getMultipleStudents() && $request->getAllStudents()) {
//if all students were requested, we will retrieve them by chunks so as to speed up the request
$allUsers = $this->dbHelper->getUsersInCourseWithRole($courseId, 'Learner');
$count = count($allUsers);
$asynch = !$count <= 20;
$masterArr = array();
//this var will store al the requests we will make
$requests = array();
for ($i = 0; $i <= $count - 1; $i++) {
$urlArgs[] = "student_ids[]={$allUsers[$i]->user_id}";
$result = fmod($i, 10);
if ($result == 0 && $asynch && $i > 0) {
//we'll send the request every 10 students
if ($request->getGrouped()) {
$urlArgs[] = "grouped=true";
}
$urlArgs[] = "access_token={$token}&per_page=100";
$url = GuzzleHelper::constructUrl($urlPieces, $urlArgs);
$req = new \GuzzleHttp\Psr7\Request('GET', $url);
$requests[] = $req;
$urlArgs = array();
}
}
//for
//at the end we'll have requests left that need to be sent
if ($request->getGrouped()) {
$urlArgs[] = "grouped=true";
}
$urlArgs[] = "access_token={$token}&per_page=100";
$url = GuzzleHelper::constructUrl($urlPieces, $urlArgs);
$req = new \GuzzleHttp\Psr7\Request('GET', $url);
$requests[] = $req;
$client = new Client();
$returnData = array();
$pool = new Pool($client, $requests, ['concurrency' => count($requests), 'fulfilled' => function ($response, $index) use(&$returnData, $request) {
$newData = json_decode($response->getBody());
$processedData = $this->processCanvasSubmissionData($newData, $request->getIncludeTags(), $request->getGrouped());
$returnData = array_merge($returnData, $processedData);
}, 'rejected' => function ($reason, $index) {
//TODO: figure out what to do here
}]);
// Initiate the transfers and create a promise
$promise = $pool->promise();
// Force the pool of requests to complete.
$promise->wait();
return $returnData;
} else {
if ($request->getMultipleStudents() && count($request->getStudentIds() > 1)) {
$ids = json_encode($request->getStudentIds());
$urlArgs[] = "student_ids[]={$ids}";
} else {
$urlArgs[] = "student_ids[]={$request->getStudentIds()[0]}";
}
}
//ASSIGNMENT IDS
//assignment_ids can be a list of assignmentIds, or if empty, all assignments will be returned
$assignmentIds = implode(',', $request->getAssignmentIds());
if (count($request->getAssignmentIds()) > 0) {
$urlArgs[] = "assignment_ids[]={$assignmentIds}";
}
} else {
if ($request->getMultipleStudents()) {
// GET /api/v1/courses/:course_id/assignments/:assignment_id/submissions
$urlPieces[] = "assignments";
//grab the first assignment id. Shouldn't have more than one (all this has been validated in the SubmissionsRequest constructor)
$urlPieces[] = $request->getAssignmentIds()[0];
$urlPieces[] = "submissions";
} else {
//GET /api/v1/courses/:course_id/assignments/:assignment_id/submissions
if ($request->getAssignmentIds()) {
$urlPieces[] = "assignments";
//input1
$urlPieces[] = $request->getAssignmentIds()[0];
// get the first assignment id from the array (there shouldn't be more than one anyway)
$urlPieces[] = "submissions";
$urlPieces[] = $request->getStudentIds()[0];
}
}
}
if (!$asynch) {
if ($request->getGrouped()) {
$urlArgs[] = "grouped=true";
}
$urlArgs[] = "access_token={$token}&per_page=100";
$url = GuzzleHelper::constructUrl($urlPieces, $urlArgs);
$response = GuzzleHelper::recursiveGet($url);
return $this->processCanvasSubmissionData($response, $request->getIncludeTags(), $request->getGrouped());
}
}
protected function startCrawlingQueue()
{
while ($this->crawlQueue->hasPendingUrls()) {
$pool = new Pool($this->client, $this->getCrawlRequests(), ['concurrency' => $this->concurrency, 'options' => [RequestOptions::CONNECT_TIMEOUT => 10, RequestOptions::TIMEOUT => 10, RequestOptions::ALLOW_REDIRECTS => false], 'fulfilled' => function (ResponseInterface $response, int $index) {
$this->handleResponse($response, $index);
$crawlUrl = $this->crawlQueue->getPendingUrlAtIndex($index);
if ($crawlUrl->url->host !== $this->baseUrl->host) {
return;
}
$this->addAllLinksToCrawlQueue((string) $response->getBody(), $crawlUrl = $this->crawlQueue->getPendingUrlAtIndex($index)->url);
}, 'rejected' => function (RequestException $exception, int $index) {
$this->handleResponse($exception->getResponse(), $index);
}]);
$promise = $pool->promise();
$promise->wait();
$this->crawlQueue->removeProcessedUrlsFromPending();
}
}
