/** * Remove account * * @param $accountId */ public function removeAccount($accountId) { $tableId = $this->getSysBucketId() . '.' . $accountId; if ($this->storageApi->tableExists($tableId)) { $this->storageApi->dropTable($tableId); } unset($this->accounts[$accountId]); }
protected function setUp() { self::$client = static::createClient(); $container = self::$client->getContainer(); $sapiToken = $container->getParameter('storage_api.test.token'); $sapiUrl = $container->getParameter('storage_api.test.url'); self::$client->setServerParameters(array('HTTP_X-StorageApi-Token' => $sapiToken)); $this->storageApi = new SapiClient(['token' => $sapiToken, 'url' => $sapiUrl, 'userAgent' => $this->componentName]); $this->encryptor = $container->get('syrup.encryptor'); $this->configuration = $container->get('ex_google_drive.configuration'); $this->configuration->setStorageApi($this->storageApi); try { $this->configuration->create(); } catch (\Exception $e) { // bucket exists } // Cleanup $sysBucketId = $this->configuration->getSysBucketId(); $tableId = $sysBucketId . '.' . $this->accountId; if ($this->storageApi->tableExists($tableId)) { $this->storageApi->dropTable($tableId); } }
public function addAccount($name, $desc = null) { $accountId = $this->nameToId($name); $tableId = $this->getSysBucketId() . '.' . $accountId; if ($this->storageApi->tableExists($tableId)) { throw new ConfigurationException('Table `' . $tableId . '` already exists. Choose different name.'); } $table = $this->getAccountTable($accountId); $table->setAttribute('accountId', $accountId); $table->setAttribute('name', $name); if ($desc != null) { $table->setAttribute('desc', $desc); } $table->save(); return ["id" => $accountId, "name" => $name, "description" => $desc]; }
protected function writeTable(CsvFile $csv, $outputTable, $incremental, $primaryKey) { try { $tableNameArr = explode('.', $outputTable); $bucketId = $tableNameArr[0] . "." . $tableNameArr[1]; $tableName = $tableNameArr[2]; } catch (ContextErrorException $e) { throw new UserException("Wrong output table name.", $e); } if (!count($csv->getHeader())) { throw new ApplicationException("Trying to upload an empty file"); } try { if (!$this->storageApi->bucketExists($bucketId)) { $bucketArr = explode('.', $bucketId); $this->storageApi->createBucket(str_replace('c-', '', $bucketArr[1]), SapiClient::STAGE_IN, 'DB Extractor data bucket'); } if (!$this->storageApi->tableExists($outputTable)) { $this->storageApi->createTableAsync($bucketId, $tableName, $csv, array('primaryKey' => $primaryKey)); } else { // handle unexpected temporary errors like "unable to fork()" $success = false; $exception = null; for ($i = 0; $i < 2 && !$success; $i++) { try { $this->storageApi->writeTableAsync($outputTable, $csv, array('incremental' => $incremental)); $success = true; } catch (\Exception $e) { $exception = $e; $this->logger->warning("Error writing to SAPI", ['exception' => $exception]); } sleep(1); } if (!$success) { throw $exception; } } } catch (ClientException $e) { if ($e->getCode() < 500) { throw new UserException($e->getMessage(), $e); } else { throw new ApplicationException($e->getMessage(), $e); } } $this->logger->info("Table " . $tableName . " imported to Storage API"); }
/** * * Upload CSV file to storage API * * @param $queryNumber * @param $table * @return bool * @throws \Exception * @throws \Keboola\StorageApi\ClientException */ private function _uploadCsvFile($queryNumber, $table) { if (!isset($this->_csvFiles[$queryNumber])) { return false; } $csvHandle = $this->_csvFiles[$queryNumber]["handle"]; $tableId = $this->storageApiBucket . '.' . $table; if (!isset($this->_sapiTableCache[$tableId]) && $this->storageApi->tableExists($tableId)) { $this->_sapiTableCache[$tableId] = $this->storageApi->getTable($tableId); } try { if (isset($this->_sapiTableCache[$tableId])) { $this->storageApi->writeTableAsync($tableId, $csvHandle, array("incremental" => true)); } else { $this->storageApi->createTableAsync($this->storageApiBucket, $table, $csvHandle, array("primaryKey" => "ex__primary")); $this->_sapiTableCache[$tableId] = $this->storageApi->getTable($tableId); } } catch (Exception $e) { throw new UserException($e->getMessage(), $e); } return true; }
public function testReadTablesS3Redshift() { // Create bucket if (!$this->client->bucketExists("in.c-docker-test-redshift")) { $this->client->createBucket("docker-test-redshift", Client::STAGE_IN, "Docker Testsuite", "redshift"); } // Create table if (!$this->client->tableExists("in.c-docker-test-redshift.test")) { $csv = new CsvFile($this->tmpDir . "/upload.csv"); $csv->writeRow(["Id", "Name"]); $csv->writeRow(["test", "test"]); $this->client->createTableAsync("in.c-docker-test-redshift", "test", $csv); $this->client->setTableAttribute("in.c-docker-test-redshift.test", "attr1", "val2"); } $root = $this->tmpDir; $reader = new Reader($this->client); $configuration = [["source" => "in.c-docker-test-redshift.test", "destination" => "test-redshift.csv"]]; $reader->downloadTables($configuration, $root . "/download", "s3"); $adapter = new TableManifestAdapter(); $manifest = $adapter->readFromFile($root . "/download/test-redshift.csv.manifest"); $this->assertEquals("in.c-docker-test-redshift.test", $manifest["id"]); $this->assertEquals("val2", $manifest["attributes"][0]["value"]); $this->assertS3info($manifest); }