/**
* Returns a recursive directory iterator that yields absolute filenames.
*
* This iterator is not broken like PHP's built-in DirectoryIterator (which
* will read the first file from a stream wrapper, then rewind, then read
* it again).
*
* @param string $path Path to traverse (e.g., s3://bucket/key, /tmp)
* @param resource $context Stream context options.
*
* @return \Generator Yields absolute filenames.
*/
public static function recursiveDirIterator($path, $context = null)
{
$invalid = ['.' => true, '..' => true];
$pathLen = strlen($path) + 1;
$iterator = self::dirIterator($path, $context);
$queue = [];
do {
while ($iterator->valid()) {
$file = $iterator->current();
$iterator->next();
if (isset($invalid[basename($file)])) {
continue;
}
$fullPath = "{$path}/{$file}";
(yield $fullPath);
if (is_dir($fullPath)) {
$queue[] = $iterator;
$iterator = t\to_iter(self::dirIterator($fullPath, $context), t\map(function ($file) use($fullPath, $pathLen) {
return substr("{$fullPath}/{$file}", $pathLen);
}));
continue;
}
}
$iterator = array_pop($queue);
} while ($iterator);
}
/**
* Returns an iterator that iterates over the values of applying a JMESPath
* search to each result yielded by the iterator as a flat sequence.
*
* @param string $expression JMESPath expression to apply to each result.
* @param int|null $limit Total number of items that should be returned
* or null for no limit.
*
* @return \Iterator
*/
public function search($expression, $limit = null)
{
// Apply JMESPath expression on each result, but as a flat sequence.
$xf = t\mapcat(function (Result $result) use($expression) {
return (array) $result->search($expression);
});
// Apply a limit to the total items returned.
if ($limit) {
$xf = t\comp($xf, t\take($limit));
}
// Return items as an iterator.
return t\to_iter($this, $xf);
}
/**
* Upload the source to S3 using multipart upload operations.
*
* @param int $concurrency Number of parts that the Uploader will upload
* concurrently (in parallel). This defaults to 1. You may need to do
* some experimenting to find the most optimum concurrency value
* for your system, but using 20-25 usually yields decent results.
* @param callable|null $before Callback to execute before each upload.
* This callback will receive a PreparedEvent object as its argument.
*
* @return Result The result of the CompleteMultipartUpload operation.
* @throws \LogicException if the upload is already complete or aborted.
* @throws MultipartUploadException if an upload operation fails.
*/
public function upload($concurrency = 1, callable $before = null)
{
// Ensure the upload is in a valid state for uploading parts.
if (!$this->state->isInitiated()) {
$this->initiate();
} elseif ($this->state->isCompleted()) {
throw new \LogicException('The upload has been completed.');
} elseif ($this->state->isAborted()) {
throw new \LogicException('This upload has been aborted.');
}
// Create iterator that will yield UploadPart commands for each part.
$commands = t\to_iter($this->parts, t\map(function (array $partData) {
return $this->createCommand('upload', $partData);
}));
// Execute the commands in parallel and process results. This collects
// unhandled errors along the way and throws an exception at the end
// that contains the state and a list of the failed parts.
$errors = [];
$this->client->executeAll($commands, ['pool_size' => $concurrency, 'prepared' => $before, 'process' => ['fn' => function (ProcessEvent $event) use(&$errors) {
$command = $event->getCommand();
$partNumber = $command[$this->config['part']['param']];
if ($ex = $event->getException()) {
$errors[$partNumber] = $ex->getMessage();
} else {
unset($errors[$partNumber]);
$this->config['fn']['result']($command, $event->getResult());
}
}, 'priority' => 'last']]);
if ($errors) {
throw new MultipartUploadException($this->state, $errors);
}
return $this->complete();
}
/**
* Support for opendir().
*
* The opendir() method of the Amazon S3 stream wrapper supports a stream
* context option of "listFilter". listFilter must be a callable that
* accepts an associative array of object data and returns true if the
* object should be yielded when iterating the keys in a bucket.
*
* @param string $path The path to the directory
* (e.g. "s3://dir[</prefix>]")
* @param string $options Unused option variable
*
* @return bool true on success
* @see http://www.php.net/manual/en/function.opendir.php
*/
public function dir_opendir($path, $options)
{
$this->openedPath = $path;
$params = $this->withPath($path);
$delimiter = $this->getOption('delimiter');
/** @var callable $filterFn */
$filterFn = $this->getOption('listFilter');
$op = ['Bucket' => $params['Bucket']];
$this->openedBucket = $params['Bucket'];
if ($delimiter === null) {
$delimiter = '/';
}
if ($delimiter) {
$op['Delimiter'] = $delimiter;
}
if ($params['Key']) {
$params['Key'] = rtrim($params['Key'], $delimiter) . $delimiter;
$op['Prefix'] = $params['Key'];
}
$this->openedBucketPrefix = $params['Key'];
// Filter our "/" keys added by the console as directories, and ensure
// that if a filter function is provided that it passes the filter.
$this->objectIterator = t\to_iter($this->getClient()->getPaginator('ListObjects', $op), t\mapcat(function (Result $result) use($filterFn) {
$contentsAndPrefixes = $result->search('[Contents[], CommonPrefixes[]][]');
// Filter out dir place holder keys and use the filter fn.
return array_filter($contentsAndPrefixes, function ($key) use($filterFn) {
return (!$filterFn || call_user_func($filterFn, $key)) && (!isset($key['Key']) || substr($key['Key'], -1, 1) !== '/');
});
}));
return true;
}
/**
* Creates an iterator that yields Commands from each filename.
*
* @param \Iterator $iter Iterator to wrap.
*
* @return \Iterator
*/
private function wrapIterator(\Iterator $iter)
{
$comp = [];
// Filter out MUP uploads to send separate operations.
if ($this->destScheme == 's3' && $this->sourceScheme == 'file') {
$comp[] = t\filter(function ($file) {
if ($this->sourceScheme == 'file' && filesize($file) >= $this->mup_threshold) {
$this->mup($file);
return false;
}
// Filter out "/" files stored on S3 as buckets.
return substr($file, -1, 1) != '/';
});
}
$comp[] = t\map($this->getTransferFunction($this->sourceScheme, $this->destScheme));
return t\to_iter($iter, call_user_func_array('transducers\\comp', $comp));
}
