public static function getTemp()
{
if (!self::$temp) {
self::$temp = new Temp();
self::$temp->initRunFolder();
}
return self::$temp;
}
public function testCleanupFilePreserve()
{
$temp = new Temp();
$file = $temp->createFile('file', true);
unset($temp);
self::assertFileExists($file->getPathname());
unlink($file->getPathname());
rmdir(dirname($file->getPathname()));
}
public function setUp()
{
// Create folders
$temp = new Temp('docker');
$temp->initRunFolder();
$this->temp = $temp;
$this->tmpDir = $temp->getTmpFolder();
$fs = new Filesystem();
$fs->mkdir($this->tmpDir . "/download");
$this->client = new Client(["token" => STORAGE_API_TOKEN]);
}
protected function writeRawCsv($data, Sheet $sheet)
{
$fileName = $sheet->getGoogleId() . "_" . $sheet->getSheetId() . "_" . date('Y-m-d') . '-' . uniqid() . ".csv";
/** @var SplFileInfo $fileInfo */
$fileInfo = $this->temp->createFile($fileName);
$fh = fopen($fileInfo->getPathname(), 'w+');
if (!$fh) {
throw new \Exception("Can't write to file " . $fileInfo->getPathname());
}
/* @var Stream $data */
fwrite($fh, $data->getContents());
fclose($fh);
return $fileInfo->getPathname();
}
/**
* @covers \Keboola\Syrup\Aws\S3\Uploader::getClient
* @covers \Keboola\Syrup\Aws\S3\Uploader::uploadString
* @covers \Keboola\Syrup\Aws\S3\Uploader::uploadFile
*/
public function testS3Uploader()
{
$s3Uploader = new Uploader(['aws-access-key' => SYRUP_AWS_KEY, 'aws-secret-key' => SYRUP_AWS_SECRET, 'aws-region' => SYRUP_AWS_REGION, 's3-upload-path' => SYRUP_S3_BUCKET]);
$fileName = uniqid();
$resultUrl = $s3Uploader->uploadString($fileName, uniqid());
$this->assertStringStartsWith('https://connection.keboola.com/admin/utils/logs?file=', $resultUrl);
$this->assertStringEndsWith($fileName, $resultUrl);
$temp = new Temp();
$fileInfo = $temp->createTmpFile();
$file = $fileInfo->openFile('a');
$file->fwrite(uniqid());
$resultUrl = $s3Uploader->uploadFile($fileInfo->getRealPath());
$this->assertStringStartsWith('https://connection.keboola.com/admin/utils/logs?file=', $resultUrl);
$this->assertStringEndsWith($fileInfo->getFilename(), $resultUrl);
}
/**
* @brief Create a CSV file, and optionally set its header
*
* @param string $name File name Suffix
* @param array $header A header line to write into created file
* @param \Keboola\Temp\Temp $temp
* @return \Keboola\CsvTable\Table
*/
public static function create($name = '', array $header = array(), Temp $temp = null)
{
if ($temp == null) {
$temp = new Temp('csv-table');
}
$tmpFile = $temp->createTmpFile($name);
$csvFile = new self($tmpFile->getPathname());
// Write header
if (!empty($header)) {
$csvFile->writeRow($header);
}
// Preserve Temp to prevent deletion!
$csvFile->setTemp($temp);
$csvFile->name = $name;
return $csvFile;
}
/**
* @expectedException \Exception
* @expectedExceptionMessage Found orphaned file manifest: 'file1.manifest'
*/
public function testWriteFilesOrphanedManifest()
{
$root = $this->tmp->getTmpFolder();
file_put_contents($root . "/upload/file1.manifest", "tags: [\"tde-exporter-php-test-xxx\"]\nis_public: true");
$writer = new Uploader($this->client);
$writer->uploadFiles($root . "/upload");
}
protected function getOutputCsv($tableName, Profile $profile)
{
$fileName = str_replace(' ', '-', $tableName) . '-' . str_replace('/', '', $profile->getGoogleId()) . "-" . microtime() . "-" . uniqid("", true) . ".csv";
$tmpFileInfo = $this->temp->createFile($fileName);
return new CsvFile($tmpFileInfo->getPathname());
}
public function createSSLFile($sslCa)
{
$filename = $this->temp->createTmpFile('ssl');
file_put_contents($filename, $sslCa);
return realpath($filename);
}
public function execute(Job $job)
{
$this->configuration->setStorageApi($this->storageApi);
$accounts = $this->configuration->getAccounts();
$options = $job->getParams();
$status = [];
if (isset($options['external'])) {
// load files by tag from SAPI
if (!isset($options['external']['account'])) {
throw new UserException("Missing field 'account'");
}
try {
$this->configuration->create();
} catch (\Exception $e) {
// create configuration if not exists
}
$account = new Account($this->configuration, uniqid('external'));
$account->fromArray($options['external']['account']);
$writer = $this->writerFactory->create($account);
if (!isset($options['external']['query'])) {
throw new UserException("Missing field 'query'");
}
$queryString = $options['external']['query'];
$queryString .= ' -tags:wr-google-drive-processed';
if (isset($options['external']['filterByRunId']) && $options['external']['filterByRunId']) {
$parentRunId = $this->getParentRunId($job->getRunId());
if ($parentRunId) {
$queryString .= ' +tags:runId-' . $parentRunId;
}
}
$uploadedFiles = $this->storageApi->listFiles((new ListFilesOptions())->setQuery($queryString));
if (empty($uploadedFiles)) {
throw new UserException("No file matches your query '" . $queryString . "'");
}
foreach ($uploadedFiles as $uploadedFile) {
$tmpFile = $this->temp->createTmpFile('wr-gd');
file_put_contents($tmpFile->getPathname(), fopen($uploadedFile['url'], 'r'));
$file = new File(['id' => $uploadedFile['id'], 'title' => $uploadedFile['name'], 'targetFolder' => isset($options['external']['targetFolder']) ? $options['external']['targetFolder'] : null, 'type' => File::TYPE_FILE, 'pathname' => $tmpFile, 'size' => $uploadedFile['sizeBytes']]);
$gdFiles = $writer->listFiles(['q' => "trashed=false and name='" . $uploadedFile['name'] . "'"]);
if (!empty($gdFiles['files'])) {
$lastGdFile = array_shift($gdFiles['files']);
$file->setGoogleId($lastGdFile['id']);
}
$file = $writer->process($file);
// tag file 'wr-google-drive-processed'
$this->storageApi->addFileTag($uploadedFile['id'], 'wr-google-drive-processed');
}
// delete temporary account
$this->configuration->removeAccount($account->getAccountId());
} else {
$fileFilter = null;
if (isset($options['config'])) {
if (!isset($accounts[$options['config']])) {
throw new UserException("Config '" . $options['config'] . "' does not exist.");
}
$accounts = [$options['config'] => $accounts[$options['config']]];
if (isset($options['file'])) {
/** @var Account $account */
$account = $accounts[$options['config']];
if (null == $account->getFile($options['file'])) {
throw new UserException("File '" . $options['file'] . "' not found");
}
$fileFilter = $options['file'];
}
}
/** @var Account $account */
foreach ($accounts as $accountId => $account) {
$writer = $this->writerFactory->create($account);
$files = $account->getFiles();
/** @var File $file */
foreach ($files as $file) {
if ($fileFilter != null && $file->getId() != $fileFilter) {
continue;
}
$file->setPathname($this->temp->createTmpFile()->getPathname());
try {
$tableExporter = new TableExporter($this->storageApi);
$tableExporter->exportTable($file->getTableId(), $file->getPathname(), []);
} catch (ClientException $e) {
throw new UserException($e->getMessage(), $e, ['file' => $file->toArray()]);
}
$file = $writer->process($file);
$status[$account->getAccountName()][$file->getTitle()] = 'ok';
}
// updated changes to files
$account->save();
}
}
return $status;
}
public function testCreateSandboxInputMapping()
{
$temp = new Temp();
$tmpDir = $temp->getTmpFolder();
$temp->initRunFolder();
$csv = new CsvFile($tmpDir . "/upload.csv");
$csv->writeRow(["Id", "Name"]);
$csv->writeRow(["test", "test"]);
$this->client->createTableAsync("in.c-sandbox", "test", $csv);
unset($csv);
$this->createECSServer();
$jobData = ['params' => ['backend' => 'docker', 'type' => 'rstudio', 'input' => ['tables' => [['source' => 'in.c-sandbox.test', 'destination' => 'source.csv']]]]];
$data = $this->executeJob('create', $jobData);
$this->assertArrayHasKey('credentials', $data);
$this->assertArrayHasKey('id', $data['credentials']);
$doctrine = self::$kernel->getContainer()->get("doctrine");
$server = new Docker($doctrine, new Client(["token" => "dummy"]));
$account = $server->getActiveAccount($data["credentials"]["id"]);
/** @var $account \Keboola\ProvisioningBundle\Entity\Account\Docker */
$this->assertNotEmpty($account->getCredentials());
$credentials = $account->getCredentials();
$this->assertArrayHasKey('hostname', $credentials);
$this->assertArrayHasKey('port', $credentials);
$this->assertArrayHasKey('user', $credentials);
$this->assertArrayHasKey('password', $credentials);
$client = $this->getGuzzleClient();
$url = $credentials['hostname'] . ':' . $credentials['port'];
$res = $client->request('GET', $url);
$response = (string) $res->getBody();
$this->assertContains('RStudio: Browser Not Supported', $response);
}
