public function initDataBucket($accountId)
{
$bucketId = self::IN_PREFIX . $this->componentName . '-' . $accountId;
if (!$this->storageApi->bucketExists(self::IN_PREFIX . $this->componentName . '-' . $accountId)) {
$this->storageApi->createBucket($this->componentName . '-' . $accountId, 'in', 'GoodData Account bucket');
}
return $bucketId;
}
public function initDataBucket($tableId)
{
$tableArr = explode('.', $tableId);
if (strtolower($tableArr[0]) != 'in') {
throw new UserException("Extractor can only import to 'IN' stage bucket");
}
$bucketId = $tableArr[0] . '.' . $tableArr[1];
if (!$this->storageApi->bucketExists($bucketId)) {
$bucketName = str_replace('c-', '', $tableArr[1]);
$this->storageApi->createBucket($bucketName, 'in', 'Google Drive data bucket');
}
}
public function init()
{
try {
$this->storageApi->createBucket($this->componentName, 'sys', 'DB extractor');
} catch (\Exception $e) {
// bucket exists
}
return $this;
}
public function createWriter($name, $description = '')
{
$bucketName = $this->componentName . '-' . $name;
$bucketId = self::SYS_PREFIX . $bucketName;
if (!$this->storageApi->bucketExists($bucketId)) {
$this->storageApi->createBucket($bucketName, StorageApi::STAGE_SYS, $description);
}
$this->storageApi->setBucketAttribute($bucketId, 'writer', 'db');
$this->storageApi->setBucketAttribute($bucketId, 'driver', $this->driver);
$this->storageApi->setBucketAttribute($bucketId, 'writerId', $name);
if (!empty($description)) {
$this->storageApi->setBucketAttribute($bucketId, 'description', $description);
}
return ['id' => $name, 'name' => $name, 'description' => $description];
}
protected function writeTable(CsvFile $csv, $outputTable, $incremental, $primaryKey)
{
try {
$tableNameArr = explode('.', $outputTable);
$bucketId = $tableNameArr[0] . "." . $tableNameArr[1];
$tableName = $tableNameArr[2];
} catch (ContextErrorException $e) {
throw new UserException("Wrong output table name.", $e);
}
if (!count($csv->getHeader())) {
throw new ApplicationException("Trying to upload an empty file");
}
try {
if (!$this->storageApi->bucketExists($bucketId)) {
$bucketArr = explode('.', $bucketId);
$this->storageApi->createBucket(str_replace('c-', '', $bucketArr[1]), SapiClient::STAGE_IN, 'DB Extractor data bucket');
}
if (!$this->storageApi->tableExists($outputTable)) {
$this->storageApi->createTableAsync($bucketId, $tableName, $csv, array('primaryKey' => $primaryKey));
} else {
// handle unexpected temporary errors like "unable to fork()"
$success = false;
$exception = null;
for ($i = 0; $i < 2 && !$success; $i++) {
try {
$this->storageApi->writeTableAsync($outputTable, $csv, array('incremental' => $incremental));
$success = true;
} catch (\Exception $e) {
$exception = $e;
$this->logger->warning("Error writing to SAPI", ['exception' => $exception]);
}
sleep(1);
}
if (!$success) {
throw $exception;
}
}
} catch (ClientException $e) {
if ($e->getCode() < 500) {
throw new UserException($e->getMessage(), $e);
} else {
throw new ApplicationException($e->getMessage(), $e);
}
}
$this->logger->info("Table " . $tableName . " imported to Storage API");
}
public function testReadTablesS3Redshift()
{
// Create bucket
if (!$this->client->bucketExists("in.c-docker-test-redshift")) {
$this->client->createBucket("docker-test-redshift", Client::STAGE_IN, "Docker Testsuite", "redshift");
}
// Create table
if (!$this->client->tableExists("in.c-docker-test-redshift.test")) {
$csv = new CsvFile($this->tmpDir . "/upload.csv");
$csv->writeRow(["Id", "Name"]);
$csv->writeRow(["test", "test"]);
$this->client->createTableAsync("in.c-docker-test-redshift", "test", $csv);
$this->client->setTableAttribute("in.c-docker-test-redshift.test", "attr1", "val2");
}
$root = $this->tmpDir;
$reader = new Reader($this->client);
$configuration = [["source" => "in.c-docker-test-redshift.test", "destination" => "test-redshift.csv"]];
$reader->downloadTables($configuration, $root . "/download", "s3");
$adapter = new TableManifestAdapter();
$manifest = $adapter->readFromFile($root . "/download/test-redshift.csv.manifest");
$this->assertEquals("in.c-docker-test-redshift.test", $manifest["id"]);
$this->assertEquals("val2", $manifest["attributes"][0]["value"]);
$this->assertS3info($manifest);
}
public function initDataBucket($accountId)
{
if (!$this->storageApi->bucketExists(self::IN_PREFIX . $this->componentName . '-' . $accountId)) {
$this->storageApi->createBucket($this->componentName . '-' . $accountId, 'in', 'Google Drive Account bucket');
}
}
public function create()
{
$this->storageApi->createBucket($this->componentName, 'sys', 'GoogleDrive Writer');
}
public function testCreateProjectStorageToken()
{
$organization = $this->client->createOrganization($this->testMaintainerId, ['name' => 'My org']);
$project = $this->client->createProject($organization['id'], ['name' => 'My test']);
// token without permissions
$token = $this->client->createProjectStorageToken($project['id'], ['description' => 'test', 'expiresIn' => 60]);
$client = new Client(['url' => getenv('KBC_MANAGE_API_URL'), 'token' => $token['token']]);
$verified = $client->verifyToken();
$this->assertEquals($project['id'], $verified['owner']['id']);
$this->assertFalse($verified['canManageBuckets']);
$this->assertFalse($verified['canManageTokens']);
$this->assertFalse($verified['canReadAllFileUploads']);
$this->assertEmpty($verified['bucketPermissions']);
// new token with more permissions
$token = $this->client->createProjectStorageToken($project['id'], ['description' => 'test', 'expiresIn' => 60, 'canManageBuckets' => true, 'canReadAllFileUploads' => true]);
$client = new Client(['url' => getenv('KBC_MANAGE_API_URL'), 'token' => $token['token']]);
$verified = $client->verifyToken();
$this->assertEquals($project['id'], $verified['owner']['id']);
$this->assertTrue($verified['canManageBuckets']);
$this->assertFalse($verified['canManageTokens']);
$this->assertTrue($verified['canReadAllFileUploads']);
// test bucket permissions
// let's create some bucket with previous token
$newBucketId = $client->createBucket('test', 'in');
$token = $this->client->createProjectStorageToken($project['id'], ['description' => 'test', 'expiresIn' => 60, 'bucketPermissions' => [$newBucketId => 'read']]);
$client = new Client(['url' => getenv('KBC_MANAGE_API_URL'), 'token' => $token['token']]);
$verified = $client->verifyToken();
$this->assertEquals($project['id'], $verified['owner']['id']);
$this->assertFalse($verified['canManageBuckets']);
$this->assertFalse($verified['canManageTokens']);
$this->assertFalse($verified['canReadAllFileUploads']);
$this->assertEquals([$newBucketId => 'read'], $verified['bucketPermissions']);
}
