public function initDataBucket($accountId) { $bucketId = self::IN_PREFIX . $this->componentName . '-' . $accountId; if (!$this->storageApi->bucketExists(self::IN_PREFIX . $this->componentName . '-' . $accountId)) { $this->storageApi->createBucket($this->componentName . '-' . $accountId, 'in', 'GoodData Account bucket'); } return $bucketId; }
public function initDataBucket($tableId) { $tableArr = explode('.', $tableId); if (strtolower($tableArr[0]) != 'in') { throw new UserException("Extractor can only import to 'IN' stage bucket"); } $bucketId = $tableArr[0] . '.' . $tableArr[1]; if (!$this->storageApi->bucketExists($bucketId)) { $bucketName = str_replace('c-', '', $tableArr[1]); $this->storageApi->createBucket($bucketName, 'in', 'Google Drive data bucket'); } }
public function init() { try { $this->storageApi->createBucket($this->componentName, 'sys', 'DB extractor'); } catch (\Exception $e) { // bucket exists } return $this; }
public function createWriter($name, $description = '') { $bucketName = $this->componentName . '-' . $name; $bucketId = self::SYS_PREFIX . $bucketName; if (!$this->storageApi->bucketExists($bucketId)) { $this->storageApi->createBucket($bucketName, StorageApi::STAGE_SYS, $description); } $this->storageApi->setBucketAttribute($bucketId, 'writer', 'db'); $this->storageApi->setBucketAttribute($bucketId, 'driver', $this->driver); $this->storageApi->setBucketAttribute($bucketId, 'writerId', $name); if (!empty($description)) { $this->storageApi->setBucketAttribute($bucketId, 'description', $description); } return ['id' => $name, 'name' => $name, 'description' => $description]; }
protected function writeTable(CsvFile $csv, $outputTable, $incremental, $primaryKey) { try { $tableNameArr = explode('.', $outputTable); $bucketId = $tableNameArr[0] . "." . $tableNameArr[1]; $tableName = $tableNameArr[2]; } catch (ContextErrorException $e) { throw new UserException("Wrong output table name.", $e); } if (!count($csv->getHeader())) { throw new ApplicationException("Trying to upload an empty file"); } try { if (!$this->storageApi->bucketExists($bucketId)) { $bucketArr = explode('.', $bucketId); $this->storageApi->createBucket(str_replace('c-', '', $bucketArr[1]), SapiClient::STAGE_IN, 'DB Extractor data bucket'); } if (!$this->storageApi->tableExists($outputTable)) { $this->storageApi->createTableAsync($bucketId, $tableName, $csv, array('primaryKey' => $primaryKey)); } else { // handle unexpected temporary errors like "unable to fork()" $success = false; $exception = null; for ($i = 0; $i < 2 && !$success; $i++) { try { $this->storageApi->writeTableAsync($outputTable, $csv, array('incremental' => $incremental)); $success = true; } catch (\Exception $e) { $exception = $e; $this->logger->warning("Error writing to SAPI", ['exception' => $exception]); } sleep(1); } if (!$success) { throw $exception; } } } catch (ClientException $e) { if ($e->getCode() < 500) { throw new UserException($e->getMessage(), $e); } else { throw new ApplicationException($e->getMessage(), $e); } } $this->logger->info("Table " . $tableName . " imported to Storage API"); }
public function testReadTablesS3Redshift() { // Create bucket if (!$this->client->bucketExists("in.c-docker-test-redshift")) { $this->client->createBucket("docker-test-redshift", Client::STAGE_IN, "Docker Testsuite", "redshift"); } // Create table if (!$this->client->tableExists("in.c-docker-test-redshift.test")) { $csv = new CsvFile($this->tmpDir . "/upload.csv"); $csv->writeRow(["Id", "Name"]); $csv->writeRow(["test", "test"]); $this->client->createTableAsync("in.c-docker-test-redshift", "test", $csv); $this->client->setTableAttribute("in.c-docker-test-redshift.test", "attr1", "val2"); } $root = $this->tmpDir; $reader = new Reader($this->client); $configuration = [["source" => "in.c-docker-test-redshift.test", "destination" => "test-redshift.csv"]]; $reader->downloadTables($configuration, $root . "/download", "s3"); $adapter = new TableManifestAdapter(); $manifest = $adapter->readFromFile($root . "/download/test-redshift.csv.manifest"); $this->assertEquals("in.c-docker-test-redshift.test", $manifest["id"]); $this->assertEquals("val2", $manifest["attributes"][0]["value"]); $this->assertS3info($manifest); }
public function initDataBucket($accountId) { if (!$this->storageApi->bucketExists(self::IN_PREFIX . $this->componentName . '-' . $accountId)) { $this->storageApi->createBucket($this->componentName . '-' . $accountId, 'in', 'Google Drive Account bucket'); } }
public function create() { $this->storageApi->createBucket($this->componentName, 'sys', 'GoogleDrive Writer'); }
public function testCreateProjectStorageToken() { $organization = $this->client->createOrganization($this->testMaintainerId, ['name' => 'My org']); $project = $this->client->createProject($organization['id'], ['name' => 'My test']); // token without permissions $token = $this->client->createProjectStorageToken($project['id'], ['description' => 'test', 'expiresIn' => 60]); $client = new Client(['url' => getenv('KBC_MANAGE_API_URL'), 'token' => $token['token']]); $verified = $client->verifyToken(); $this->assertEquals($project['id'], $verified['owner']['id']); $this->assertFalse($verified['canManageBuckets']); $this->assertFalse($verified['canManageTokens']); $this->assertFalse($verified['canReadAllFileUploads']); $this->assertEmpty($verified['bucketPermissions']); // new token with more permissions $token = $this->client->createProjectStorageToken($project['id'], ['description' => 'test', 'expiresIn' => 60, 'canManageBuckets' => true, 'canReadAllFileUploads' => true]); $client = new Client(['url' => getenv('KBC_MANAGE_API_URL'), 'token' => $token['token']]); $verified = $client->verifyToken(); $this->assertEquals($project['id'], $verified['owner']['id']); $this->assertTrue($verified['canManageBuckets']); $this->assertFalse($verified['canManageTokens']); $this->assertTrue($verified['canReadAllFileUploads']); // test bucket permissions // let's create some bucket with previous token $newBucketId = $client->createBucket('test', 'in'); $token = $this->client->createProjectStorageToken($project['id'], ['description' => 'test', 'expiresIn' => 60, 'bucketPermissions' => [$newBucketId => 'read']]); $client = new Client(['url' => getenv('KBC_MANAGE_API_URL'), 'token' => $token['token']]); $verified = $client->verifyToken(); $this->assertEquals($project['id'], $verified['owner']['id']); $this->assertFalse($verified['canManageBuckets']); $this->assertFalse($verified['canManageTokens']); $this->assertFalse($verified['canReadAllFileUploads']); $this->assertEquals([$newBucketId => 'read'], $verified['bucketPermissions']); }