Exemple #1
0
 /**
  * Returns a recursive directory iterator that yields absolute filenames.
  *
  * This iterator is not broken like PHP's built-in DirectoryIterator (which
  * will read the first file from a stream wrapper, then rewind, then read
  * it again).
  *
  * @param string   $path    Path to traverse (e.g., s3://bucket/key, /tmp)
  * @param resource $context Stream context options.
  *
  * @return \Generator Yields absolute filenames.
  */
 public static function recursiveDirIterator($path, $context = null)
 {
     $invalid = ['.' => true, '..' => true];
     $pathLen = strlen($path) + 1;
     $iterator = self::dirIterator($path, $context);
     $queue = [];
     do {
         while ($iterator->valid()) {
             $file = $iterator->current();
             $iterator->next();
             if (isset($invalid[basename($file)])) {
                 continue;
             }
             $fullPath = "{$path}/{$file}";
             (yield $fullPath);
             if (is_dir($fullPath)) {
                 $queue[] = $iterator;
                 $iterator = t\to_iter(self::dirIterator($fullPath, $context), t\map(function ($file) use($fullPath, $pathLen) {
                     return substr("{$fullPath}/{$file}", $pathLen);
                 }));
                 continue;
             }
         }
         $iterator = array_pop($queue);
     } while ($iterator);
 }
Exemple #2
0
 /**
  * Upload the source to S3 using multipart upload operations.
  *
  * @param int $concurrency Number of parts that the Uploader will upload
  *     concurrently (in parallel). This defaults to 1. You may need to do
  *     some experimenting to find the most optimum concurrency value
  *     for your system, but using 20-25 usually yields decent results.
  * @param callable|null $before Callback to execute before each upload.
  *     This callback will receive a PreparedEvent object as its argument.
  *
  * @return Result The result of the CompleteMultipartUpload operation.
  * @throws \LogicException if the upload is already complete or aborted.
  * @throws MultipartUploadException if an upload operation fails.
  */
 public function upload($concurrency = 1, callable $before = null)
 {
     // Ensure the upload is in a valid state for uploading parts.
     if (!$this->state->isInitiated()) {
         $this->initiate();
     } elseif ($this->state->isCompleted()) {
         throw new \LogicException('The upload has been completed.');
     } elseif ($this->state->isAborted()) {
         throw new \LogicException('This upload has been aborted.');
     }
     // Create iterator that will yield UploadPart commands for each part.
     $commands = t\to_iter($this->parts, t\map(function (array $partData) {
         return $this->createCommand('upload', $partData);
     }));
     // Execute the commands in parallel and process results. This collects
     // unhandled errors along the way and throws an exception at the end
     // that contains the state and a list of the failed parts.
     $errors = [];
     $this->client->executeAll($commands, ['pool_size' => $concurrency, 'prepared' => $before, 'process' => ['fn' => function (ProcessEvent $event) use(&$errors) {
         $command = $event->getCommand();
         $partNumber = $command[$this->config['part']['param']];
         if ($ex = $event->getException()) {
             $errors[$partNumber] = $ex->getMessage();
         } else {
             unset($errors[$partNumber]);
             $this->config['fn']['result']($command, $event->getResult());
         }
     }, 'priority' => 'last']]);
     if ($errors) {
         throw new MultipartUploadException($this->state, $errors);
     }
     return $this->complete();
 }
Exemple #3
0
 /**
  * Creates an iterator that yields Commands from each filename.
  *
  * @param \Iterator $iter Iterator to wrap.
  *
  * @return \Iterator
  */
 private function wrapIterator(\Iterator $iter)
 {
     $comp = [];
     // Filter out MUP uploads to send separate operations.
     if ($this->destScheme == 's3' && $this->sourceScheme == 'file') {
         $comp[] = t\filter(function ($file) {
             if ($this->sourceScheme == 'file' && filesize($file) >= $this->mup_threshold) {
                 $this->mup($file);
                 return false;
             }
             // Filter out "/" files stored on S3 as buckets.
             return substr($file, -1, 1) != '/';
         });
     }
     $comp[] = t\map($this->getTransferFunction($this->sourceScheme, $this->destScheme));
     return t\to_iter($iter, call_user_func_array('transducers\\comp', $comp));
 }