protected function writeTable(CsvFile $csv, $outputTable, $incremental, $primaryKey)
{
try {
$tableNameArr = explode('.', $outputTable);
$bucketId = $tableNameArr[0] . "." . $tableNameArr[1];
$tableName = $tableNameArr[2];
} catch (ContextErrorException $e) {
throw new UserException("Wrong output table name.", $e);
}
if (!count($csv->getHeader())) {
throw new ApplicationException("Trying to upload an empty file");
}
try {
if (!$this->storageApi->bucketExists($bucketId)) {
$bucketArr = explode('.', $bucketId);
$this->storageApi->createBucket(str_replace('c-', '', $bucketArr[1]), SapiClient::STAGE_IN, 'DB Extractor data bucket');
}
if (!$this->storageApi->tableExists($outputTable)) {
$this->storageApi->createTableAsync($bucketId, $tableName, $csv, array('primaryKey' => $primaryKey));
} else {
// handle unexpected temporary errors like "unable to fork()"
$success = false;
$exception = null;
for ($i = 0; $i < 2 && !$success; $i++) {
try {
$this->storageApi->writeTableAsync($outputTable, $csv, array('incremental' => $incremental));
$success = true;
} catch (\Exception $e) {
$exception = $e;
$this->logger->warning("Error writing to SAPI", ['exception' => $exception]);
}
sleep(1);
}
if (!$success) {
throw $exception;
}
}
} catch (ClientException $e) {
if ($e->getCode() < 500) {
throw new UserException($e->getMessage(), $e);
} else {
throw new ApplicationException($e->getMessage(), $e);
}
}
$this->logger->info("Table " . $tableName . " imported to Storage API");
}
/**
*
* Upload CSV file to storage API
*
* @param $queryNumber
* @param $table
* @return bool
* @throws \Exception
* @throws \Keboola\StorageApi\ClientException
*/
private function _uploadCsvFile($queryNumber, $table)
{
if (!isset($this->_csvFiles[$queryNumber])) {
return false;
}
$csvHandle = $this->_csvFiles[$queryNumber]["handle"];
$tableId = $this->storageApiBucket . '.' . $table;
if (!isset($this->_sapiTableCache[$tableId]) && $this->storageApi->tableExists($tableId)) {
$this->_sapiTableCache[$tableId] = $this->storageApi->getTable($tableId);
}
try {
if (isset($this->_sapiTableCache[$tableId])) {
$this->storageApi->writeTableAsync($tableId, $csvHandle, array("incremental" => true));
} else {
$this->storageApi->createTableAsync($this->storageApiBucket, $table, $csvHandle, array("primaryKey" => "ex__primary"));
$this->_sapiTableCache[$tableId] = $this->storageApi->getTable($tableId);
}
} catch (Exception $e) {
throw new UserException($e->getMessage(), $e);
}
return true;
}
public function testReadTablesS3Redshift()
{
// Create bucket
if (!$this->client->bucketExists("in.c-docker-test-redshift")) {
$this->client->createBucket("docker-test-redshift", Client::STAGE_IN, "Docker Testsuite", "redshift");
}
// Create table
if (!$this->client->tableExists("in.c-docker-test-redshift.test")) {
$csv = new CsvFile($this->tmpDir . "/upload.csv");
$csv->writeRow(["Id", "Name"]);
$csv->writeRow(["test", "test"]);
$this->client->createTableAsync("in.c-docker-test-redshift", "test", $csv);
$this->client->setTableAttribute("in.c-docker-test-redshift.test", "attr1", "val2");
}
$root = $this->tmpDir;
$reader = new Reader($this->client);
$configuration = [["source" => "in.c-docker-test-redshift.test", "destination" => "test-redshift.csv"]];
$reader->downloadTables($configuration, $root . "/download", "s3");
$adapter = new TableManifestAdapter();
$manifest = $adapter->readFromFile($root . "/download/test-redshift.csv.manifest");
$this->assertEquals("in.c-docker-test-redshift.test", $manifest["id"]);
$this->assertEquals("val2", $manifest["attributes"][0]["value"]);
$this->assertS3info($manifest);
}
