コード例 #1
0
 /**
  * @inheritDoc ITask::runStep()
  *
  * @param int $step
  *
  * @return bool
  */
 public function runStep($step)
 {
     // NOTE: Perhaps much of this should be moved into a service
     $batch = \Guzzle\Batch\BatchBuilder::factory()->transferRequests(20)->bufferExceptions()->build();
     // Make the client
     $client = new \Guzzle\Http\Client();
     // Set the Accept header
     $client->setDefaultOption('headers/Accept', '*/*');
     // Loop the paths in this step
     foreach ($this->_paths[$step] as $path) {
         // Make the url, stripping 'site:' from the path if it exists
         $newPath = preg_replace('/site:/', '', $path, 1);
         $url = UrlHelper::getSiteUrl($newPath);
         // Create the GET request
         $request = $client->get($url);
         // Add it to the batch
         $batch->add($request);
     }
     // Flush the queue and retrieve the flushed items
     $requests = $batch->flush();
     // Log any exceptions
     foreach ($batch->getExceptions() as $e) {
         Craft::log('CacheMonster: an exception occurred: ' . $e->getMessage(), LogLevel::Error);
     }
     // Clear any exceptions
     $batch->clearExceptions();
     return true;
 }
コード例 #2
0
public function apply($perBatch = 50)
{
$this->iterated = $this->batches = $batches = 0;
$that = $this;
$it = $this->iterator;
$callback = $this->callback;

$batch = BatchBuilder::factory()
->createBatchesWith(new BatchSizeDivisor($perBatch))
->transferWith(new BatchClosureTransfer(function (array $batch) use ($that, $callback, &$batches, $it) {
$batches++;
$that->dispatch('iterator_batch.before_batch', array('iterator' => $it, 'batch' => $batch));
call_user_func_array($callback, array($it, $batch));
$that->dispatch('iterator_batch.after_batch', array('iterator' => $it, 'batch' => $batch));
}))
->autoFlushAt($perBatch)
->build();

$this->dispatch('iterator_batch.created_batch', array('batch' => $batch));

foreach ($this->iterator as $resource) {
$this->iterated++;
$batch->add($resource);
}

$batch->flush();
$this->batches = $batches;

return $this->iterated;
}
コード例 #3
0
 public function runStep($step)
 {
     VarnishpurgePlugin::log('Varnish purge task run step: ' . $step, LogLevel::Info, craft()->varnishpurge->getSetting('varnishLogAll'));
     $servers = craft()->varnishpurge->getSetting('varnishUrl');
     if (!is_array($servers)) {
         $servers = array($servers);
     }
     foreach ($servers as $server) {
         $batch = \Guzzle\Batch\BatchBuilder::factory()->transferRequests(20)->bufferExceptions()->build();
         $client = new \Guzzle\Http\Client();
         $client->setDefaultOption('headers/Accept', '*/*');
         foreach ($this->_urls[$step] as $url) {
             $urlComponents = parse_url($url);
             $targetUrl = preg_replace('{/$}', '', $server) . $urlComponents['path'];
             VarnishpurgePlugin::log('Adding url to purge: ' . $targetUrl . ' with Host: ' . $urlComponents['host'], LogLevel::Info, craft()->varnishpurge->getSetting('varnishLogAll'));
             $request = $client->createRequest('PURGE', $targetUrl);
             $request->addHeader('Host', $urlComponents['host']);
             $batch->add($request);
         }
         $requests = $batch->flush();
         foreach ($batch->getExceptions() as $e) {
             VarnishpurgePlugin::log('An exception occurred: ' . $e->getMessage(), LogLevel::Error);
         }
         $batch->clearExceptions();
     }
     return true;
 }
コード例 #4
0
 /**
  * Factory for creating a DynamoDB BatchWriteItemQueue
  *
  * @param AwsClientInterface $client    Client used to transfer requests
  * @param int                $batchSize Size of each batch. The WriteRequestBatch works most efficiently with a
  *                                      batch size that is a multiple of 25
  * @param mixed $notify Callback to be run after each flush
  *
  * @return WriteRequestBatch
  */
 public static function factory(AwsClientInterface $client, $batchSize = WriteRequestBatchTransfer::BATCH_WRITE_MAX_SIZE, $notify = null)
 {
     $builder = BatchBuilder::factory()->createBatchesWith(new BatchSizeDivisor($batchSize))->transferWith(new WriteRequestBatchTransfer($client));
     if ($notify) {
         $builder->notify($notify);
     }
     $batch = new self($builder->build());
     $batch = new FlushingBatch($batch, $batchSize);
     return $batch;
 }
コード例 #5
0
 public function runStep($step)
 {
     VarnishpurgePlugin::log('Varnish purge task run step: ' . $step, LogLevel::Info, craft()->varnishpurge->getSetting('varnishLogAll'));
     $batch = \Guzzle\Batch\BatchBuilder::factory()->transferRequests(20)->bufferExceptions()->build();
     $client = new \Guzzle\Http\Client();
     $client->setDefaultOption('headers/Accept', '*/*');
     foreach ($this->_urls[$step] as $url) {
         VarnishpurgePlugin::log('Adding url to purge: ' . $url, LogLevel::Info, craft()->varnishpurge->getSetting('varnishLogAll'));
         $request = $client->createRequest('PURGE', $url);
         $batch->add($request);
     }
     $requests = $batch->flush();
     foreach ($batch->getExceptions() as $e) {
         VarnishpurgePlugin::log('An exception occurred: ' . $e->getMessage(), LogLevel::Error);
     }
     $batch->clearExceptions();
     return true;
 }
コード例 #6
0
 /**
  * @depends testIteratesOverScan
  */
 public function testImplementsCustomExponentialBackoffStrategy()
 {
     self::log('Getting an item a bunch of times in parallel');
     $batch = BatchBuilder::factory()->transferCommands(100)->build();
     $s = microtime(true);
     $total = 300;
     for ($i = 0; $i < $total; $i++) {
         $command = $this->client->getCommand('GetItem', array('TableName' => $this->table, 'Key' => $this->client->formatAttributes(array('HashKeyElement' => 'Test', 'RangeKeyElement' => 10))));
         $batch->add($command);
     }
     $retries = 0;
     foreach ($batch->flush() as $command) {
         $retries += $command->getRequest()->getParams()->get(BackoffPlugin::RETRY_PARAM);
     }
     $elapsed = microtime(true) - $s;
     self::log("Got the item {$total} times with {$retries} retries in {$elapsed} seconds");
 }
コード例 #7
0
ファイル: Summon.php プロジェクト: phindmarsh/summon
 private function findImagesInHtml(DOMDocument $doc)
 {
     $fetched = array();
     $self = get_class();
     $batch = BatchBuilder::factory()->transferRequests(1)->autoFlushAt(10)->notify(function ($transferred) use(&$fetched, $self) {
         /** @var Request[] $transferred */
         foreach ($transferred as $request) {
             $length = $request->getResponse()->getContentLength();
             if ($request->getResponse()->hasHeader('content-length') && $length < $self::MIN_FILESIZE) {
                 continue;
             }
             $img_data = @getimagesize($request->getResponse()->getBody()->getUri());
             if (!is_array($img_data)) {
                 continue;
             }
             $image = array('url' => $request->getResponse()->getInfo('url'), 'length' => $length, 'width' => $img_data[0], 'height' => $img_data[1], 'ratio' => $img_data[0] / $img_data[1], 'area' => $img_data[0] * $img_data[1]);
             $image['aspect'] = $image['ratio'] > $self::MIN_RATIO && $image['ratio'] < $self::MAX_RATIO;
             $fetched[] = $image;
         }
     })->bufferExceptions()->build();
     foreach ($doc->getElementsByTagName('img') as $img) {
         $src = self::resolveUrl($this->url, $img->getAttribute('src'));
         $key = md5($src);
         if (!isset($images[$key])) {
             $images[$key] = true;
             $batch->add($this->client->get($src, null, tmpfile()));
         }
     }
     unset($doc);
     unset($images);
     $batch->flush();
     usort($fetched, function ($a, $b) {
         // if either is within the bounds but not both, the one that is wins
         if ($a['aspect'] ^ $b['aspect']) {
             return $a['aspect'] ? -1 : 1;
         } else {
             return $b['area'] - $a['area'];
         }
     });
     $filtered = array();
     $count = 0;
     foreach ($fetched as $img) {
         foreach (self::$PREFERRED_TIERS as $area => $limit) {
             if ($img['area'] >= $area) {
                 $count++;
                 $filtered[] = $img['url'];
                 if ($count >= $limit) {
                     break 2;
                 }
                 break;
             }
         }
     }
     return $filtered;
 }
コード例 #8
0
 /**
  * Set the write queue as a {@see \Guzzle\Batch\Batch} queue using the {@see \Guzzle\Batch\BatchBuilder}
  */
 public function setupWriteQueue()
 {
     $this->writeQueue = BatchBuilder::factory()->transferRequests($this->options->get('write.batchLimit'))->build();
 }
コード例 #9
0
 /**
  * Factory for creating a DeleteObjectsBatch
  *
  * @param AwsClientInterface $client Client used to transfer requests
  * @param string             $bucket Bucket that contains the objects to delete
  * @param string             $mfa    MFA token to use with the request
  *
  * @return self
  */
 public static function factory(AwsClientInterface $client, $bucket, $mfa = null)
 {
     $batch = BatchBuilder::factory()->createBatchesWith(new BatchSizeDivisor(1000))->transferWith(new DeleteObjectsTransfer($client, $bucket, $mfa))->build();
     return new self($batch);
 }
コード例 #10
0
ファイル: WebhookJob.php プロジェクト: luisbrito/Phraseanet
 private function deliverEvent(EventProcessorFactory $eventFactory, Application $app, array $thirdPartyApplications, WebhookEvent $event)
 {
     if (count($thirdPartyApplications) === 0) {
         $this->log('info', sprintf('No applications defined to listen for webhook events'));
         return;
     }
     // format event data
     $eventProcessor = $eventFactory->get($event);
     $data = $eventProcessor->process($event);
     // batch requests
     $batch = BatchBuilder::factory()->transferRequests(10)->build();
     foreach ($thirdPartyApplications as $thirdPartyApplication) {
         $delivery = $app['manipulator.webhook-delivery']->create($thirdPartyApplication, $event);
         // append delivery id as url anchor
         $uniqueUrl = $this->getUrl($thirdPartyApplication, $delivery);
         // create http request with data as request body
         $batch->add($this->httpClient->createRequest('POST', $uniqueUrl, ['Content-Type' => 'application/vnd.phraseanet.event+json'], json_encode($data)));
     }
     $batch->flush();
 }
コード例 #11
0
 public function testTransfersCommands()
 {
     $batch = BatchBuilder::factory()->transferCommands(10)->build();
     $this->assertInstanceOf('Guzzle\\Batch\\BatchCommandTransfer', $this->readAttribute($batch, 'transferStrategy'));
 }
 public function getBatchBuilder()
 {
     return BatchBuilder::factory()->transferRequests(10)->bufferExceptions();
 }