/**
* @inheritDoc ITask::runStep()
*
* @param int $step
*
* @return bool
*/
public function runStep($step)
{
// NOTE: Perhaps much of this should be moved into a service
$batch = \Guzzle\Batch\BatchBuilder::factory()->transferRequests(20)->bufferExceptions()->build();
// Make the client
$client = new \Guzzle\Http\Client();
// Set the Accept header
$client->setDefaultOption('headers/Accept', '*/*');
// Loop the paths in this step
foreach ($this->_paths[$step] as $path) {
// Make the url, stripping 'site:' from the path if it exists
$newPath = preg_replace('/site:/', '', $path, 1);
$url = UrlHelper::getSiteUrl($newPath);
// Create the GET request
$request = $client->get($url);
// Add it to the batch
$batch->add($request);
}
// Flush the queue and retrieve the flushed items
$requests = $batch->flush();
// Log any exceptions
foreach ($batch->getExceptions() as $e) {
Craft::log('CacheMonster: an exception occurred: ' . $e->getMessage(), LogLevel::Error);
}
// Clear any exceptions
$batch->clearExceptions();
return true;
}
public function apply($perBatch = 50)
{
$this->iterated = $this->batches = $batches = 0;
$that = $this;
$it = $this->iterator;
$callback = $this->callback;
$batch = BatchBuilder::factory()
->createBatchesWith(new BatchSizeDivisor($perBatch))
->transferWith(new BatchClosureTransfer(function (array $batch) use ($that, $callback, &$batches, $it) {
$batches++;
$that->dispatch('iterator_batch.before_batch', array('iterator' => $it, 'batch' => $batch));
call_user_func_array($callback, array($it, $batch));
$that->dispatch('iterator_batch.after_batch', array('iterator' => $it, 'batch' => $batch));
}))
->autoFlushAt($perBatch)
->build();
$this->dispatch('iterator_batch.created_batch', array('batch' => $batch));
foreach ($this->iterator as $resource) {
$this->iterated++;
$batch->add($resource);
}
$batch->flush();
$this->batches = $batches;
return $this->iterated;
}
public function runStep($step)
{
VarnishpurgePlugin::log('Varnish purge task run step: ' . $step, LogLevel::Info, craft()->varnishpurge->getSetting('varnishLogAll'));
$servers = craft()->varnishpurge->getSetting('varnishUrl');
if (!is_array($servers)) {
$servers = array($servers);
}
foreach ($servers as $server) {
$batch = \Guzzle\Batch\BatchBuilder::factory()->transferRequests(20)->bufferExceptions()->build();
$client = new \Guzzle\Http\Client();
$client->setDefaultOption('headers/Accept', '*/*');
foreach ($this->_urls[$step] as $url) {
$urlComponents = parse_url($url);
$targetUrl = preg_replace('{/$}', '', $server) . $urlComponents['path'];
VarnishpurgePlugin::log('Adding url to purge: ' . $targetUrl . ' with Host: ' . $urlComponents['host'], LogLevel::Info, craft()->varnishpurge->getSetting('varnishLogAll'));
$request = $client->createRequest('PURGE', $targetUrl);
$request->addHeader('Host', $urlComponents['host']);
$batch->add($request);
}
$requests = $batch->flush();
foreach ($batch->getExceptions() as $e) {
VarnishpurgePlugin::log('An exception occurred: ' . $e->getMessage(), LogLevel::Error);
}
$batch->clearExceptions();
}
return true;
}
/**
* Factory for creating a DynamoDB BatchWriteItemQueue
*
* @param AwsClientInterface $client Client used to transfer requests
* @param int $batchSize Size of each batch. The WriteRequestBatch works most efficiently with a
* batch size that is a multiple of 25
* @param mixed $notify Callback to be run after each flush
*
* @return WriteRequestBatch
*/
public static function factory(AwsClientInterface $client, $batchSize = WriteRequestBatchTransfer::BATCH_WRITE_MAX_SIZE, $notify = null)
{
$builder = BatchBuilder::factory()->createBatchesWith(new BatchSizeDivisor($batchSize))->transferWith(new WriteRequestBatchTransfer($client));
if ($notify) {
$builder->notify($notify);
}
$batch = new self($builder->build());
$batch = new FlushingBatch($batch, $batchSize);
return $batch;
}
public function runStep($step)
{
VarnishpurgePlugin::log('Varnish purge task run step: ' . $step, LogLevel::Info, craft()->varnishpurge->getSetting('varnishLogAll'));
$batch = \Guzzle\Batch\BatchBuilder::factory()->transferRequests(20)->bufferExceptions()->build();
$client = new \Guzzle\Http\Client();
$client->setDefaultOption('headers/Accept', '*/*');
foreach ($this->_urls[$step] as $url) {
VarnishpurgePlugin::log('Adding url to purge: ' . $url, LogLevel::Info, craft()->varnishpurge->getSetting('varnishLogAll'));
$request = $client->createRequest('PURGE', $url);
$batch->add($request);
}
$requests = $batch->flush();
foreach ($batch->getExceptions() as $e) {
VarnishpurgePlugin::log('An exception occurred: ' . $e->getMessage(), LogLevel::Error);
}
$batch->clearExceptions();
return true;
}
/**
* @depends testIteratesOverScan
*/
public function testImplementsCustomExponentialBackoffStrategy()
{
self::log('Getting an item a bunch of times in parallel');
$batch = BatchBuilder::factory()->transferCommands(100)->build();
$s = microtime(true);
$total = 300;
for ($i = 0; $i < $total; $i++) {
$command = $this->client->getCommand('GetItem', array('TableName' => $this->table, 'Key' => $this->client->formatAttributes(array('HashKeyElement' => 'Test', 'RangeKeyElement' => 10))));
$batch->add($command);
}
$retries = 0;
foreach ($batch->flush() as $command) {
$retries += $command->getRequest()->getParams()->get(BackoffPlugin::RETRY_PARAM);
}
$elapsed = microtime(true) - $s;
self::log("Got the item {$total} times with {$retries} retries in {$elapsed} seconds");
}
private function findImagesInHtml(DOMDocument $doc)
{
$fetched = array();
$self = get_class();
$batch = BatchBuilder::factory()->transferRequests(1)->autoFlushAt(10)->notify(function ($transferred) use(&$fetched, $self) {
/** @var Request[] $transferred */
foreach ($transferred as $request) {
$length = $request->getResponse()->getContentLength();
if ($request->getResponse()->hasHeader('content-length') && $length < $self::MIN_FILESIZE) {
continue;
}
$img_data = @getimagesize($request->getResponse()->getBody()->getUri());
if (!is_array($img_data)) {
continue;
}
$image = array('url' => $request->getResponse()->getInfo('url'), 'length' => $length, 'width' => $img_data[0], 'height' => $img_data[1], 'ratio' => $img_data[0] / $img_data[1], 'area' => $img_data[0] * $img_data[1]);
$image['aspect'] = $image['ratio'] > $self::MIN_RATIO && $image['ratio'] < $self::MAX_RATIO;
$fetched[] = $image;
}
})->bufferExceptions()->build();
foreach ($doc->getElementsByTagName('img') as $img) {
$src = self::resolveUrl($this->url, $img->getAttribute('src'));
$key = md5($src);
if (!isset($images[$key])) {
$images[$key] = true;
$batch->add($this->client->get($src, null, tmpfile()));
}
}
unset($doc);
unset($images);
$batch->flush();
usort($fetched, function ($a, $b) {
// if either is within the bounds but not both, the one that is wins
if ($a['aspect'] ^ $b['aspect']) {
return $a['aspect'] ? -1 : 1;
} else {
return $b['area'] - $a['area'];
}
});
$filtered = array();
$count = 0;
foreach ($fetched as $img) {
foreach (self::$PREFERRED_TIERS as $area => $limit) {
if ($img['area'] >= $area) {
$count++;
$filtered[] = $img['url'];
if ($count >= $limit) {
break 2;
}
break;
}
}
}
return $filtered;
}
/**
* Set the write queue as a {@see \Guzzle\Batch\Batch} queue using the {@see \Guzzle\Batch\BatchBuilder}
*/
public function setupWriteQueue()
{
$this->writeQueue = BatchBuilder::factory()->transferRequests($this->options->get('write.batchLimit'))->build();
}
/**
* Factory for creating a DeleteObjectsBatch
*
* @param AwsClientInterface $client Client used to transfer requests
* @param string $bucket Bucket that contains the objects to delete
* @param string $mfa MFA token to use with the request
*
* @return self
*/
public static function factory(AwsClientInterface $client, $bucket, $mfa = null)
{
$batch = BatchBuilder::factory()->createBatchesWith(new BatchSizeDivisor(1000))->transferWith(new DeleteObjectsTransfer($client, $bucket, $mfa))->build();
return new self($batch);
}
private function deliverEvent(EventProcessorFactory $eventFactory, Application $app, array $thirdPartyApplications, WebhookEvent $event)
{
if (count($thirdPartyApplications) === 0) {
$this->log('info', sprintf('No applications defined to listen for webhook events'));
return;
}
// format event data
$eventProcessor = $eventFactory->get($event);
$data = $eventProcessor->process($event);
// batch requests
$batch = BatchBuilder::factory()->transferRequests(10)->build();
foreach ($thirdPartyApplications as $thirdPartyApplication) {
$delivery = $app['manipulator.webhook-delivery']->create($thirdPartyApplication, $event);
// append delivery id as url anchor
$uniqueUrl = $this->getUrl($thirdPartyApplication, $delivery);
// create http request with data as request body
$batch->add($this->httpClient->createRequest('POST', $uniqueUrl, ['Content-Type' => 'application/vnd.phraseanet.event+json'], json_encode($data)));
}
$batch->flush();
}
public function testTransfersCommands()
{
$batch = BatchBuilder::factory()->transferCommands(10)->build();
$this->assertInstanceOf('Guzzle\\Batch\\BatchCommandTransfer', $this->readAttribute($batch, 'transferStrategy'));
}
public function getBatchBuilder()
{
return BatchBuilder::factory()->transferRequests(10)->bufferExceptions();
}