public function tearDown()
{
// Delete local files
$finder = new Finder();
$fs = new Filesystem();
$fs->remove($finder->files()->in($this->tmpDir . "/download"));
$fs->remove($finder->files()->in($this->tmpDir));
$fs->remove($this->tmpDir . "/download");
$fs->remove($this->tmpDir);
// Delete file uploads
$options = new ListFilesOptions();
$options->setTags(["docker-bundle-test"]);
$files = $this->client->listFiles($options);
foreach ($files as $file) {
$this->client->deleteFile($file["id"]);
}
if ($this->client->bucketExists("in.c-docker-test")) {
// Delete tables
foreach ($this->client->listTables("in.c-docker-test") as $table) {
$this->client->dropTable($table["id"]);
}
// Delete bucket
$this->client->dropBucket("in.c-docker-test");
}
if ($this->client->bucketExists("in.c-docker-test-redshift")) {
// Delete tables
foreach ($this->client->listTables("in.c-docker-test-redshift") as $table) {
$this->client->dropTable($table["id"]);
}
// Delete bucket
$this->client->dropBucket("in.c-docker-test-redshift");
}
}
/**
* @param $fileConfiguration
* @return array
*/
public function getFiles($fileConfiguration)
{
$options = new ListFilesOptions();
if (empty($fileConfiguration['tags']) && empty($fileConfiguration['query'])) {
throw new InvalidInputException("Invalid file mapping, both 'tags' and 'query' are empty.");
}
if (!empty($fileConfiguration['filter_by_run_id'])) {
$options->setRunId($this->getParentRunId());
}
if (isset($fileConfiguration["tags"]) && count($fileConfiguration["tags"])) {
$options->setTags($fileConfiguration["tags"]);
}
if (isset($fileConfiguration["query"])) {
$options->setQuery($fileConfiguration["query"]);
}
$files = $this->getClient()->listFiles($options);
// a little sanity check, otherwise it may easily happen that a wrong ES query would fill up the server
if (empty($fileConfiguration["limit"])) {
$fileConfiguration["limit"] = 10;
}
if (count($files) > $fileConfiguration["limit"]) {
throw new InvalidInputException("File input mapping downloads more than {$fileConfiguration['limit']} files, this seems like a mistake.");
}
return $files;
}
/**
*/
public function testWriteFilesOutputMappingAndManifest()
{
$root = $this->tmp->getTmpFolder();
file_put_contents($root . "/upload/file1", "test");
file_put_contents($root . "/upload/file1.manifest", "tags: [\"tde-exporter-php-test\", \"xxx\"]\nis_public: true");
$configs = array(array("source" => "file1", "tags" => array("tde-exporter-php-test", "yyy"), "is_public" => false));
$writer = new Uploader($this->client);
$writer->uploadFiles($root . "/upload", $configs);
$options = new ListFilesOptions();
$options->setTags(array("tde-exporter-php-test"));
$files = $this->client->listFiles($options);
$this->assertCount(1, $files);
$file1 = null;
foreach ($files as $file) {
if ($file["name"] == 'file1') {
$file1 = $file;
}
}
$this->assertNotNull($file1);
$this->assertEquals(4, $file1["sizeBytes"]);
$this->assertEquals(array("tde-exporter-php-test", "yyy"), $file1["tags"]);
$this->assertFalse($file1["isPublic"]);
}
