/**
* Remove account
*
* @param $accountId
*/
public function removeAccount($accountId)
{
$tableId = $this->getSysBucketId() . '.' . $accountId;
if ($this->storageApi->tableExists($tableId)) {
$this->storageApi->dropTable($tableId);
}
unset($this->accounts[$accountId]);
}
protected function setUp()
{
self::$client = static::createClient();
$container = self::$client->getContainer();
$sapiToken = $container->getParameter('storage_api.test.token');
$sapiUrl = $container->getParameter('storage_api.test.url');
self::$client->setServerParameters(array('HTTP_X-StorageApi-Token' => $sapiToken));
$this->storageApi = new SapiClient(['token' => $sapiToken, 'url' => $sapiUrl, 'userAgent' => $this->componentName]);
$this->encryptor = $container->get('syrup.encryptor');
$this->configuration = $container->get('ex_google_drive.configuration');
$this->configuration->setStorageApi($this->storageApi);
try {
$this->configuration->create();
} catch (\Exception $e) {
// bucket exists
}
// Cleanup
$sysBucketId = $this->configuration->getSysBucketId();
$tableId = $sysBucketId . '.' . $this->accountId;
if ($this->storageApi->tableExists($tableId)) {
$this->storageApi->dropTable($tableId);
}
}
public function addAccount($name, $desc = null)
{
$accountId = $this->nameToId($name);
$tableId = $this->getSysBucketId() . '.' . $accountId;
if ($this->storageApi->tableExists($tableId)) {
throw new ConfigurationException('Table `' . $tableId . '` already exists. Choose different name.');
}
$table = $this->getAccountTable($accountId);
$table->setAttribute('accountId', $accountId);
$table->setAttribute('name', $name);
if ($desc != null) {
$table->setAttribute('desc', $desc);
}
$table->save();
return ["id" => $accountId, "name" => $name, "description" => $desc];
}
protected function writeTable(CsvFile $csv, $outputTable, $incremental, $primaryKey)
{
try {
$tableNameArr = explode('.', $outputTable);
$bucketId = $tableNameArr[0] . "." . $tableNameArr[1];
$tableName = $tableNameArr[2];
} catch (ContextErrorException $e) {
throw new UserException("Wrong output table name.", $e);
}
if (!count($csv->getHeader())) {
throw new ApplicationException("Trying to upload an empty file");
}
try {
if (!$this->storageApi->bucketExists($bucketId)) {
$bucketArr = explode('.', $bucketId);
$this->storageApi->createBucket(str_replace('c-', '', $bucketArr[1]), SapiClient::STAGE_IN, 'DB Extractor data bucket');
}
if (!$this->storageApi->tableExists($outputTable)) {
$this->storageApi->createTableAsync($bucketId, $tableName, $csv, array('primaryKey' => $primaryKey));
} else {
// handle unexpected temporary errors like "unable to fork()"
$success = false;
$exception = null;
for ($i = 0; $i < 2 && !$success; $i++) {
try {
$this->storageApi->writeTableAsync($outputTable, $csv, array('incremental' => $incremental));
$success = true;
} catch (\Exception $e) {
$exception = $e;
$this->logger->warning("Error writing to SAPI", ['exception' => $exception]);
}
sleep(1);
}
if (!$success) {
throw $exception;
}
}
} catch (ClientException $e) {
if ($e->getCode() < 500) {
throw new UserException($e->getMessage(), $e);
} else {
throw new ApplicationException($e->getMessage(), $e);
}
}
$this->logger->info("Table " . $tableName . " imported to Storage API");
}
/**
*
* Upload CSV file to storage API
*
* @param $queryNumber
* @param $table
* @return bool
* @throws \Exception
* @throws \Keboola\StorageApi\ClientException
*/
private function _uploadCsvFile($queryNumber, $table)
{
if (!isset($this->_csvFiles[$queryNumber])) {
return false;
}
$csvHandle = $this->_csvFiles[$queryNumber]["handle"];
$tableId = $this->storageApiBucket . '.' . $table;
if (!isset($this->_sapiTableCache[$tableId]) && $this->storageApi->tableExists($tableId)) {
$this->_sapiTableCache[$tableId] = $this->storageApi->getTable($tableId);
}
try {
if (isset($this->_sapiTableCache[$tableId])) {
$this->storageApi->writeTableAsync($tableId, $csvHandle, array("incremental" => true));
} else {
$this->storageApi->createTableAsync($this->storageApiBucket, $table, $csvHandle, array("primaryKey" => "ex__primary"));
$this->_sapiTableCache[$tableId] = $this->storageApi->getTable($tableId);
}
} catch (Exception $e) {
throw new UserException($e->getMessage(), $e);
}
return true;
}
public function testReadTablesS3Redshift()
{
// Create bucket
if (!$this->client->bucketExists("in.c-docker-test-redshift")) {
$this->client->createBucket("docker-test-redshift", Client::STAGE_IN, "Docker Testsuite", "redshift");
}
// Create table
if (!$this->client->tableExists("in.c-docker-test-redshift.test")) {
$csv = new CsvFile($this->tmpDir . "/upload.csv");
$csv->writeRow(["Id", "Name"]);
$csv->writeRow(["test", "test"]);
$this->client->createTableAsync("in.c-docker-test-redshift", "test", $csv);
$this->client->setTableAttribute("in.c-docker-test-redshift.test", "attr1", "val2");
}
$root = $this->tmpDir;
$reader = new Reader($this->client);
$configuration = [["source" => "in.c-docker-test-redshift.test", "destination" => "test-redshift.csv"]];
$reader->downloadTables($configuration, $root . "/download", "s3");
$adapter = new TableManifestAdapter();
$manifest = $adapter->readFromFile($root . "/download/test-redshift.csv.manifest");
$this->assertEquals("in.c-docker-test-redshift.test", $manifest["id"]);
$this->assertEquals("val2", $manifest["attributes"][0]["value"]);
$this->assertS3info($manifest);
}
