Beispiel #1
0
 /**
  * Returns a recursive directory iterator that yields absolute filenames.
  *
  * This iterator is not broken like PHP's built-in DirectoryIterator (which
  * will read the first file from a stream wrapper, then rewind, then read
  * it again).
  *
  * @param string   $path    Path to traverse (e.g., s3://bucket/key, /tmp)
  * @param resource $context Stream context options.
  *
  * @return \Generator Yields absolute filenames.
  */
 public static function recursiveDirIterator($path, $context = null)
 {
     $invalid = ['.' => true, '..' => true];
     $pathLen = strlen($path) + 1;
     $iterator = self::dirIterator($path, $context);
     $queue = [];
     do {
         while ($iterator->valid()) {
             $file = $iterator->current();
             $iterator->next();
             if (isset($invalid[basename($file)])) {
                 continue;
             }
             $fullPath = "{$path}/{$file}";
             (yield $fullPath);
             if (is_dir($fullPath)) {
                 $queue[] = $iterator;
                 $iterator = t\to_iter(self::dirIterator($fullPath, $context), t\map(function ($file) use($fullPath, $pathLen) {
                     return substr("{$fullPath}/{$file}", $pathLen);
                 }));
                 continue;
             }
         }
         $iterator = array_pop($queue);
     } while ($iterator);
 }
 /**
  * Returns an iterator that iterates over the values of applying a JMESPath
  * search to each result yielded by the iterator as a flat sequence.
  *
  * @param string   $expression JMESPath expression to apply to each result.
  * @param int|null $limit      Total number of items that should be returned
  *                             or null for no limit.
  *
  * @return \Iterator
  */
 public function search($expression, $limit = null)
 {
     // Apply JMESPath expression on each result, but as a flat sequence.
     $xf = t\mapcat(function (Result $result) use($expression) {
         return (array) $result->search($expression);
     });
     // Apply a limit to the total items returned.
     if ($limit) {
         $xf = t\comp($xf, t\take($limit));
     }
     // Return items as an iterator.
     return t\to_iter($this, $xf);
 }
Beispiel #3
0
 /**
  * Upload the source to S3 using multipart upload operations.
  *
  * @param int $concurrency Number of parts that the Uploader will upload
  *     concurrently (in parallel). This defaults to 1. You may need to do
  *     some experimenting to find the most optimum concurrency value
  *     for your system, but using 20-25 usually yields decent results.
  * @param callable|null $before Callback to execute before each upload.
  *     This callback will receive a PreparedEvent object as its argument.
  *
  * @return Result The result of the CompleteMultipartUpload operation.
  * @throws \LogicException if the upload is already complete or aborted.
  * @throws MultipartUploadException if an upload operation fails.
  */
 public function upload($concurrency = 1, callable $before = null)
 {
     // Ensure the upload is in a valid state for uploading parts.
     if (!$this->state->isInitiated()) {
         $this->initiate();
     } elseif ($this->state->isCompleted()) {
         throw new \LogicException('The upload has been completed.');
     } elseif ($this->state->isAborted()) {
         throw new \LogicException('This upload has been aborted.');
     }
     // Create iterator that will yield UploadPart commands for each part.
     $commands = t\to_iter($this->parts, t\map(function (array $partData) {
         return $this->createCommand('upload', $partData);
     }));
     // Execute the commands in parallel and process results. This collects
     // unhandled errors along the way and throws an exception at the end
     // that contains the state and a list of the failed parts.
     $errors = [];
     $this->client->executeAll($commands, ['pool_size' => $concurrency, 'prepared' => $before, 'process' => ['fn' => function (ProcessEvent $event) use(&$errors) {
         $command = $event->getCommand();
         $partNumber = $command[$this->config['part']['param']];
         if ($ex = $event->getException()) {
             $errors[$partNumber] = $ex->getMessage();
         } else {
             unset($errors[$partNumber]);
             $this->config['fn']['result']($command, $event->getResult());
         }
     }, 'priority' => 'last']]);
     if ($errors) {
         throw new MultipartUploadException($this->state, $errors);
     }
     return $this->complete();
 }
Beispiel #4
0
 /**
  * Support for opendir().
  *
  * The opendir() method of the Amazon S3 stream wrapper supports a stream
  * context option of "listFilter". listFilter must be a callable that
  * accepts an associative array of object data and returns true if the
  * object should be yielded when iterating the keys in a bucket.
  *
  * @param string $path    The path to the directory
  *                        (e.g. "s3://dir[</prefix>]")
  * @param string $options Unused option variable
  *
  * @return bool true on success
  * @see http://www.php.net/manual/en/function.opendir.php
  */
 public function dir_opendir($path, $options)
 {
     $this->openedPath = $path;
     $params = $this->withPath($path);
     $delimiter = $this->getOption('delimiter');
     /** @var callable $filterFn */
     $filterFn = $this->getOption('listFilter');
     $op = ['Bucket' => $params['Bucket']];
     $this->openedBucket = $params['Bucket'];
     if ($delimiter === null) {
         $delimiter = '/';
     }
     if ($delimiter) {
         $op['Delimiter'] = $delimiter;
     }
     if ($params['Key']) {
         $params['Key'] = rtrim($params['Key'], $delimiter) . $delimiter;
         $op['Prefix'] = $params['Key'];
     }
     $this->openedBucketPrefix = $params['Key'];
     // Filter our "/" keys added by the console as directories, and ensure
     // that if a filter function is provided that it passes the filter.
     $this->objectIterator = t\to_iter($this->getClient()->getPaginator('ListObjects', $op), t\mapcat(function (Result $result) use($filterFn) {
         $contentsAndPrefixes = $result->search('[Contents[], CommonPrefixes[]][]');
         // Filter out dir place holder keys and use the filter fn.
         return array_filter($contentsAndPrefixes, function ($key) use($filterFn) {
             return (!$filterFn || call_user_func($filterFn, $key)) && (!isset($key['Key']) || substr($key['Key'], -1, 1) !== '/');
         });
     }));
     return true;
 }
Beispiel #5
0
 /**
  * Creates an iterator that yields Commands from each filename.
  *
  * @param \Iterator $iter Iterator to wrap.
  *
  * @return \Iterator
  */
 private function wrapIterator(\Iterator $iter)
 {
     $comp = [];
     // Filter out MUP uploads to send separate operations.
     if ($this->destScheme == 's3' && $this->sourceScheme == 'file') {
         $comp[] = t\filter(function ($file) {
             if ($this->sourceScheme == 'file' && filesize($file) >= $this->mup_threshold) {
                 $this->mup($file);
                 return false;
             }
             // Filter out "/" files stored on S3 as buckets.
             return substr($file, -1, 1) != '/';
         });
     }
     $comp[] = t\map($this->getTransferFunction($this->sourceScheme, $this->destScheme));
     return t\to_iter($iter, call_user_func_array('transducers\\comp', $comp));
 }