/**
* @param string $bucket
* @return string
*/
public function listAction($bucket)
{
$errors = [];
$buckets = [];
try {
$result = $this->s3Client->listBuckets();
$buckets = $result->get('Buckets');
} catch (S3Exception $e) {
$errors[] = sprintf('Cannot retrieve buckets: %s', $e->getMessage());
}
$objects = [];
if (!empty($bucket)) {
try {
$maxIteration = 10;
$iteration = 0;
$marker = '';
do {
$result = $this->s3Client->listObjects(['Bucket' => $bucket, 'Marker' => $marker]);
if ($result->get('Contents')) {
$objects = array_merge($objects, $result->get('Contents'));
}
if (count($objects)) {
$marker = $objects[count($objects) - 1]['Key'];
}
} while ($result->get('IsTruncated') && ++$iteration < $maxIteration);
if ($result->get('IsTruncated')) {
$errors[] = sprintf('The number of keys greater than %u, the first part is shown', count($objects));
}
} catch (S3Exception $e) {
$errors[] = sprintf('Cannot retrieve objects: %s', $e->getMessage());
}
}
return $this->twig->render('list.html.twig', ['selected_bucket' => $bucket, 'buckets' => $buckets, 'objects' => $objects, 'errors' => $errors]);
}
/**
* List contents of a directory
*
* @param string $dirname
* @param bool $recursive
* @return array directory contents
*/
public function listContents($dirname = '', $recursive = false)
{
$result = $this->client->listObjects(array('Bucket' => $this->bucket))->getAll(array('Contents'));
$contents = isset($result['Contents']) ? $result['Contents'] : array();
$result = array_map(array($this, 'normalizeObject'), $contents);
return Util::emulateDirectories($result);
}
/**
* Return files list in directory
*
* @param string $path dir path
* @return array
* @author Dmitry (dio) Levashov,
* @author Alexey Sukhotin
**/
protected function _scandir($path)
{
$s3path = preg_replace("/\\/\$/", "", $path);
$s3path = preg_replace("/^\\//", "", $s3path);
$files = (array) $this->s3->listObjects(array('Bucket' => $this->options['bucket'], 'delimiter' => '/', 'Prefix' => $s3path))->get('Contents');
$finalfiles = array();
$folders = array();
foreach ($files as $file) {
if (preg_match("|^" . preg_replace("/^\\//", "", $s3path) . '/' . "[^/]*/?\$|", $file['Key'])) {
$fname = $file['Key'];
if (!$fname || $fname == preg_replace("/\\/\$/", "", $s3path) || $fname == preg_replace("/\$/", "/", $s3path)) {
continue;
}
$finalfiles[] = preg_replace("/\\/\$/", "", $fname);
} else {
$matches = array();
if ($res = preg_match("|^" . preg_replace("/^\\//", "", $s3path) . '/' . "(.*?)\\/|", $file['Key'], $matches)) {
$folders[$matches[1]] = true;
}
}
}
// Folders retrieved differently, as it's not a real object on S3
foreach ($folders as $forlderName => $tmp) {
if (!in_array(preg_replace("/^\\//", "", $s3path) . "/" . $forlderName, $finalfiles)) {
$finalfiles[] = preg_replace("/^\\//", "", $s3path) . "/" . $forlderName;
}
}
sort($finalfiles);
return $finalfiles;
}
/**
* Scan directory
*/
protected function cacheDir($path)
{
$rpath = rtrim($this->pathToKey($path), '/') . "/";
$this->dirsCache[$path] = array();
if ($rpath == '/') {
$scan = $this->s3->listObjects(array("Bucket" => $this->bucket, "Key" => '/', "Delimiter" => "/"));
} else {
$scan = $this->s3->listObjects(array("Bucket" => $this->bucket, "Prefix" => $rpath, "Delimiter" => "/"));
}
if (isset($scan['CommonPrefixes'])) {
foreach ($scan['CommonPrefixes'] as $prefix) {
if ($prefix['Prefix'] == $rpath) {
continue;
}
//preg_match('~/([^/]+)/$~', $prefix['Prefix'], $m);
if ($rpath !== '/') {
$name = substr($prefix['Prefix'], strlen($rpath));
} else {
$name = $prefix['Prefix'];
}
$name = trim($name, '/');
$data = array("name" => $name, "size" => 0, "mime" => "directory", "read" => true, "write" => true, "parent_id" => $path);
$fullpath = $this->_joinPath($path, $data['name']);
//print $fullpath."\n";
if ($stat = $this->updateCache($fullpath, $data)) {
$this->dirsCache[$path][] = $fullpath;
}
}
}
if (isset($scan['Contents'])) {
foreach ($scan['Contents'] as $file) {
if (preg_match('~/$~', $file['Key'])) {
continue;
}
$data = $this->format_stat($this->getFile($file['Key']));
$fullpath = $this->_joinPath($path, $data['name']);
if ($stat = $this->updateCache($fullpath, $data)) {
$this->dirsCache[$path][] = $fullpath;
}
}
}
return $this->dirsCache[$path];
}
/**
* List blobs
*
* @param string $container Container name
* @param string $prefix Optional. Filters the results to return only blobs whose name begins with the
* specified prefix.
* @param string $delimiter Optional. Delimiter, i.e. '/', for specifying folder hierarchy
*
* @return array
* @throws DfException
*/
public function listBlobs($container = '', $prefix = '', $delimiter = '')
{
$options = ['Bucket' => $container, 'Prefix' => $prefix];
if (!empty($delimiter)) {
$options['Delimiter'] = $delimiter;
}
// No max-keys specified. Get everything.
$keys = [];
do {
/** @var \Aws\Result $list */
$list = $this->blobConn->listObjects($options);
$objects = $list->get('Contents');
if (!empty($objects)) {
foreach ($objects as $object) {
if (0 != strcasecmp($prefix, $object['Key'])) {
$keys[$object['Key']] = true;
}
}
}
$objects = $list->get('CommonPrefixes');
if (!empty($objects)) {
foreach ($objects as $object) {
if (0 != strcasecmp($prefix, $object['Prefix'])) {
if (!isset($keys[$object['Prefix']])) {
$keys[$object['Prefix']] = false;
}
}
}
}
$options['Marker'] = $list->get('Marker');
} while ($list->get('IsTruncated'));
$options = ['Bucket' => $container, 'Key' => ''];
$out = [];
foreach ($keys as $key => $isObject) {
$options['Key'] = $key;
if ($isObject) {
/** @var \Aws\Result $meta */
$meta = $this->blobConn->headObject($options);
$out[] = ['name' => $key, 'content_type' => $meta->get('ContentType'), 'content_length' => intval($meta->get('ContentLength')), 'last_modified' => $meta->get('LastModified')];
} else {
$out[] = ['name' => $key, 'content_type' => null, 'content_length' => 0, 'last_modified' => null];
}
}
return $out;
}
public function rmdir($path)
{
$path = $this->normalizePath($path);
if (!$this->file_exists($path)) {
return false;
}
// Since there are no real directories on S3, we need
// to delete all objects prefixed with the path.
$objects = $this->connection->listObjects(array('Bucket' => $this->bucket, 'Prefix' => $path . '/'));
try {
$result = $this->connection->deleteObjects(array('Bucket' => $this->bucket, 'Objects' => $objects['Contents']));
$this->testTimeout();
} catch (S3Exception $e) {
\OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR);
return false;
}
return true;
}
/**
* Publishes the whole collection to this target
*
* @param \TYPO3\Flow\Resource\Collection $collection The collection to publish
* @return void
* @throws Exception
*/
public function publishCollection(Collection $collection)
{
if (!isset($this->existingObjectsInfo)) {
$this->existingObjectsInfo = array();
$requestArguments = array('Bucket' => $this->bucketName, 'Delimiter' => '/', 'Prefix' => $this->keyPrefix);
do {
$result = $this->s3Client->listObjects($requestArguments);
$this->existingObjectsInfo[] = $result->get('Contents');
if ($result->get('IsTruncated')) {
$requestArguments['marker'] = $result->get('NextMarker');
}
} while ($result->get('IsTruncated'));
}
$obsoleteObjects = array_fill_keys(array_keys($this->existingObjectsInfo), true);
$storage = $collection->getStorage();
if ($storage instanceof S3Storage) {
$storageBucketName = $storage->getBucketName();
if ($storageBucketName === $this->bucketName && $storage->getKeyPrefix() === $this->keyPrefix) {
throw new Exception(sprintf('Could not publish collection %s because the source and target S3 bucket is the same, with identical key prefixes. Either choose a different bucket or at least key prefix for the target.', $collection->getName()), 1428929137);
}
foreach ($collection->getObjects() as $object) {
/** @var \TYPO3\Flow\Resource\Storage\Object $object */
$objectName = $this->keyPrefix . $this->getRelativePublicationPathAndFilename($object);
$options = array('ACL' => 'public-read', 'Bucket' => $this->bucketName, 'CopySource' => urlencode($storageBucketName . '/' . $storage->getKeyPrefix() . $object->getSha1()), 'ContentType' => $object->getMediaType(), 'MetadataDirective' => 'REPLACE', 'Key' => $objectName);
try {
$this->s3Client->copyObject($options);
} catch (S3Exception $e) {
throw new Exception(sprintf('Could not copy resource with SHA1 hash %s of collection %s from bucket %s to %s: %s', $object->getSha1(), $collection->getName(), $storageBucketName, $this->bucketName, $e->getMessage()), 1431009234);
}
$this->systemLogger->log(sprintf('Successfully copied resource as object "%s" (MD5: %s) from bucket "%s" to bucket "%s"', $objectName, $object->getMd5() ?: 'unknown', $storageBucketName, $this->bucketName), LOG_DEBUG);
unset($obsoleteObjects[$this->getRelativePublicationPathAndFilename($object)]);
}
} else {
foreach ($collection->getObjects() as $object) {
/** @var \TYPO3\Flow\Resource\Storage\Object $object */
$this->publishFile($object->getStream(), $this->getRelativePublicationPathAndFilename($object), $object);
unset($obsoleteObjects[$this->getRelativePublicationPathAndFilename($object)]);
}
}
foreach (array_keys($obsoleteObjects) as $relativePathAndFilename) {
$this->s3Client->deleteObject(array('Bucket' => $this->bucketName, 'Key' => $this->keyPrefix . $relativePathAndFilename));
}
}
/**
* Ensure that an uploaded file is unique
*
* @param string $prefix
* @param string $fileName
*/
protected function uniqueFile($prefix, $fileName)
{
// request a list of objects filtered by prefix
$objects = $this->s3->listObjects(['Bucket' => $this->bucket, 'Prefix' => $prefix]);
$list = [];
foreach ($objects['Contents'] as $object) {
$list[] = $object['Key'];
}
// $this->s3->get_object_list($this->bucket, compact('prefix'));
$path = join('/', array($prefix, $fileName));
$i = 0;
while (in_array($path, $list)) {
$i++;
$parts = explode('.', $fileName);
$ext = array_pop($parts);
$parts = array_diff($parts, array("copy{$i}", "copy" . ($i - 1)));
$parts[] = "copy{$i}";
$parts[] = $ext;
$path = join('/', array($prefix, implode('.', $parts)));
}
if (isset($parts)) {
$_FILES['newfile']['name'] = implode('.', $parts);
}
}
/**
* Returns some or all (up to 1000) of the objects in a bucket.
* You can use the request parameters as selection criteria to return a subset of the objects in a bucket.
*
* @param string $bucket
*
* @param array $params
*
* @return mixed
*/
public function listObjects($bucket, array $params = [])
{
$params['Bucket'] = $bucket;
return $this->instance->listObjects($params);
}
/**
* listObjects
*
* @param array $params
*
* @return \Guzzle\Service\Resource\Model
*/
public function listObjects(array $params = array())
{
$params['Bucket'] = $this->name;
return $this->client->listObjects($params);
}
