/**
* Reads configuration from StorageApi
* could be empty - extractor with no configuration
*/
public function getConfig()
{
Reader::$client = $this->storageApi;
if ($this->storageApi->bucketExists('sys.c-' . $this->componentName)) {
return Reader::read('sys.c-' . $this->componentName);
} else {
return [];
}
}
/**
* @throws \Exception
* @throws \Keboola\StorageApi\ClientException
*/
protected function clearBucket()
{
// Delete tables and bucket
if ($this->client->bucketExists("out.c-tde-test")) {
foreach ($this->client->listTables("out.c-tde-test") as $table) {
$this->client->dropTable($table["id"]);
}
// Delete bucket
$this->client->dropBucket("out.c-tde-test");
}
}
public function createWriter($name, $description = '')
{
$bucketName = $this->componentName . '-' . $name;
$bucketId = self::SYS_PREFIX . $bucketName;
if (!$this->storageApi->bucketExists($bucketId)) {
$this->storageApi->createBucket($bucketName, StorageApi::STAGE_SYS, $description);
}
$this->storageApi->setBucketAttribute($bucketId, 'writer', 'db');
$this->storageApi->setBucketAttribute($bucketId, 'driver', $this->driver);
$this->storageApi->setBucketAttribute($bucketId, 'writerId', $name);
if (!empty($description)) {
$this->storageApi->setBucketAttribute($bucketId, 'description', $description);
}
return ['id' => $name, 'name' => $name, 'description' => $description];
}
public function getSysBucketId()
{
if ($this->storageApi->bucketExists('sys.c-' . $this->componentName)) {
return 'sys.c-' . $this->componentName;
} else {
if ($this->storageApi->bucketExists('sys.' . $this->componentName)) {
return 'sys.' . $this->componentName;
}
}
throw new ConfigurationException("SYS bucket don't exists");
}
protected function setUp($driver = null)
{
self::$client = static::createClient();
$this->container = self::$client->getContainer();
$sapiToken = $this->container->getParameter('storage_api.test.token');
$sapiUrl = $this->container->getParameter('storage_api.test.url');
self::$client->setServerParameters(['HTTP_X-StorageApi-Token' => $sapiToken]);
$this->storageApi = new SapiClient(['token' => $sapiToken, 'url' => $sapiUrl, 'userAgent' => $this->componentName]);
if ($driver != null) {
$this->configuration = new Configuration($this->componentName . '-' . $driver, $this->storageApi, $driver);
} else {
$this->configuration = new Configuration($this->componentName, $this->storageApi);
}
// Cleanup
$sysBucketId = $this->configuration->getSysBucketId($this->writerId);
if ($this->storageApi->bucketExists($sysBucketId)) {
$accTables = $this->storageApi->listTables($sysBucketId);
foreach ($accTables as $table) {
$this->storageApi->dropTable($table['id']);
}
$this->storageApi->dropBucket($sysBucketId);
}
}
protected function writeTable(CsvFile $csv, $outputTable, $incremental, $primaryKey)
{
try {
$tableNameArr = explode('.', $outputTable);
$bucketId = $tableNameArr[0] . "." . $tableNameArr[1];
$tableName = $tableNameArr[2];
} catch (ContextErrorException $e) {
throw new UserException("Wrong output table name.", $e);
}
if (!count($csv->getHeader())) {
throw new ApplicationException("Trying to upload an empty file");
}
try {
if (!$this->storageApi->bucketExists($bucketId)) {
$bucketArr = explode('.', $bucketId);
$this->storageApi->createBucket(str_replace('c-', '', $bucketArr[1]), SapiClient::STAGE_IN, 'DB Extractor data bucket');
}
if (!$this->storageApi->tableExists($outputTable)) {
$this->storageApi->createTableAsync($bucketId, $tableName, $csv, array('primaryKey' => $primaryKey));
} else {
// handle unexpected temporary errors like "unable to fork()"
$success = false;
$exception = null;
for ($i = 0; $i < 2 && !$success; $i++) {
try {
$this->storageApi->writeTableAsync($outputTable, $csv, array('incremental' => $incremental));
$success = true;
} catch (\Exception $e) {
$exception = $e;
$this->logger->warning("Error writing to SAPI", ['exception' => $exception]);
}
sleep(1);
}
if (!$success) {
throw $exception;
}
}
} catch (ClientException $e) {
if ($e->getCode() < 500) {
throw new UserException($e->getMessage(), $e);
} else {
throw new ApplicationException($e->getMessage(), $e);
}
}
$this->logger->info("Table " . $tableName . " imported to Storage API");
}
public function testReadTablesS3Redshift()
{
// Create bucket
if (!$this->client->bucketExists("in.c-docker-test-redshift")) {
$this->client->createBucket("docker-test-redshift", Client::STAGE_IN, "Docker Testsuite", "redshift");
}
// Create table
if (!$this->client->tableExists("in.c-docker-test-redshift.test")) {
$csv = new CsvFile($this->tmpDir . "/upload.csv");
$csv->writeRow(["Id", "Name"]);
$csv->writeRow(["test", "test"]);
$this->client->createTableAsync("in.c-docker-test-redshift", "test", $csv);
$this->client->setTableAttribute("in.c-docker-test-redshift.test", "attr1", "val2");
}
$root = $this->tmpDir;
$reader = new Reader($this->client);
$configuration = [["source" => "in.c-docker-test-redshift.test", "destination" => "test-redshift.csv"]];
$reader->downloadTables($configuration, $root . "/download", "s3");
$adapter = new TableManifestAdapter();
$manifest = $adapter->readFromFile($root . "/download/test-redshift.csv.manifest");
$this->assertEquals("in.c-docker-test-redshift.test", $manifest["id"]);
$this->assertEquals("val2", $manifest["attributes"][0]["value"]);
$this->assertS3info($manifest);
}
