public static function getTemp() { if (!self::$temp) { self::$temp = new Temp(); self::$temp->initRunFolder(); } return self::$temp; }
public function testCleanupFilePreserve() { $temp = new Temp(); $file = $temp->createFile('file', true); unset($temp); self::assertFileExists($file->getPathname()); unlink($file->getPathname()); rmdir(dirname($file->getPathname())); }
public function setUp() { // Create folders $temp = new Temp('docker'); $temp->initRunFolder(); $this->temp = $temp; $this->tmpDir = $temp->getTmpFolder(); $fs = new Filesystem(); $fs->mkdir($this->tmpDir . "/download"); $this->client = new Client(["token" => STORAGE_API_TOKEN]); }
protected function writeRawCsv($data, Sheet $sheet) { $fileName = $sheet->getGoogleId() . "_" . $sheet->getSheetId() . "_" . date('Y-m-d') . '-' . uniqid() . ".csv"; /** @var SplFileInfo $fileInfo */ $fileInfo = $this->temp->createFile($fileName); $fh = fopen($fileInfo->getPathname(), 'w+'); if (!$fh) { throw new \Exception("Can't write to file " . $fileInfo->getPathname()); } /* @var Stream $data */ fwrite($fh, $data->getContents()); fclose($fh); return $fileInfo->getPathname(); }
/** * @covers \Keboola\Syrup\Aws\S3\Uploader::getClient * @covers \Keboola\Syrup\Aws\S3\Uploader::uploadString * @covers \Keboola\Syrup\Aws\S3\Uploader::uploadFile */ public function testS3Uploader() { $s3Uploader = new Uploader(['aws-access-key' => SYRUP_AWS_KEY, 'aws-secret-key' => SYRUP_AWS_SECRET, 'aws-region' => SYRUP_AWS_REGION, 's3-upload-path' => SYRUP_S3_BUCKET]); $fileName = uniqid(); $resultUrl = $s3Uploader->uploadString($fileName, uniqid()); $this->assertStringStartsWith('https://connection.keboola.com/admin/utils/logs?file=', $resultUrl); $this->assertStringEndsWith($fileName, $resultUrl); $temp = new Temp(); $fileInfo = $temp->createTmpFile(); $file = $fileInfo->openFile('a'); $file->fwrite(uniqid()); $resultUrl = $s3Uploader->uploadFile($fileInfo->getRealPath()); $this->assertStringStartsWith('https://connection.keboola.com/admin/utils/logs?file=', $resultUrl); $this->assertStringEndsWith($fileInfo->getFilename(), $resultUrl); }
/** * @brief Create a CSV file, and optionally set its header * * @param string $name File name Suffix * @param array $header A header line to write into created file * @param \Keboola\Temp\Temp $temp * @return \Keboola\CsvTable\Table */ public static function create($name = '', array $header = array(), Temp $temp = null) { if ($temp == null) { $temp = new Temp('csv-table'); } $tmpFile = $temp->createTmpFile($name); $csvFile = new self($tmpFile->getPathname()); // Write header if (!empty($header)) { $csvFile->writeRow($header); } // Preserve Temp to prevent deletion! $csvFile->setTemp($temp); $csvFile->name = $name; return $csvFile; }
/** * @expectedException \Exception * @expectedExceptionMessage Found orphaned file manifest: 'file1.manifest' */ public function testWriteFilesOrphanedManifest() { $root = $this->tmp->getTmpFolder(); file_put_contents($root . "/upload/file1.manifest", "tags: [\"tde-exporter-php-test-xxx\"]\nis_public: true"); $writer = new Uploader($this->client); $writer->uploadFiles($root . "/upload"); }
protected function getOutputCsv($tableName, Profile $profile) { $fileName = str_replace(' ', '-', $tableName) . '-' . str_replace('/', '', $profile->getGoogleId()) . "-" . microtime() . "-" . uniqid("", true) . ".csv"; $tmpFileInfo = $this->temp->createFile($fileName); return new CsvFile($tmpFileInfo->getPathname()); }
public function createSSLFile($sslCa) { $filename = $this->temp->createTmpFile('ssl'); file_put_contents($filename, $sslCa); return realpath($filename); }
public function execute(Job $job) { $this->configuration->setStorageApi($this->storageApi); $accounts = $this->configuration->getAccounts(); $options = $job->getParams(); $status = []; if (isset($options['external'])) { // load files by tag from SAPI if (!isset($options['external']['account'])) { throw new UserException("Missing field 'account'"); } try { $this->configuration->create(); } catch (\Exception $e) { // create configuration if not exists } $account = new Account($this->configuration, uniqid('external')); $account->fromArray($options['external']['account']); $writer = $this->writerFactory->create($account); if (!isset($options['external']['query'])) { throw new UserException("Missing field 'query'"); } $queryString = $options['external']['query']; $queryString .= ' -tags:wr-google-drive-processed'; if (isset($options['external']['filterByRunId']) && $options['external']['filterByRunId']) { $parentRunId = $this->getParentRunId($job->getRunId()); if ($parentRunId) { $queryString .= ' +tags:runId-' . $parentRunId; } } $uploadedFiles = $this->storageApi->listFiles((new ListFilesOptions())->setQuery($queryString)); if (empty($uploadedFiles)) { throw new UserException("No file matches your query '" . $queryString . "'"); } foreach ($uploadedFiles as $uploadedFile) { $tmpFile = $this->temp->createTmpFile('wr-gd'); file_put_contents($tmpFile->getPathname(), fopen($uploadedFile['url'], 'r')); $file = new File(['id' => $uploadedFile['id'], 'title' => $uploadedFile['name'], 'targetFolder' => isset($options['external']['targetFolder']) ? $options['external']['targetFolder'] : null, 'type' => File::TYPE_FILE, 'pathname' => $tmpFile, 'size' => $uploadedFile['sizeBytes']]); $gdFiles = $writer->listFiles(['q' => "trashed=false and name='" . $uploadedFile['name'] . "'"]); if (!empty($gdFiles['files'])) { $lastGdFile = array_shift($gdFiles['files']); $file->setGoogleId($lastGdFile['id']); } $file = $writer->process($file); // tag file 'wr-google-drive-processed' $this->storageApi->addFileTag($uploadedFile['id'], 'wr-google-drive-processed'); } // delete temporary account $this->configuration->removeAccount($account->getAccountId()); } else { $fileFilter = null; if (isset($options['config'])) { if (!isset($accounts[$options['config']])) { throw new UserException("Config '" . $options['config'] . "' does not exist."); } $accounts = [$options['config'] => $accounts[$options['config']]]; if (isset($options['file'])) { /** @var Account $account */ $account = $accounts[$options['config']]; if (null == $account->getFile($options['file'])) { throw new UserException("File '" . $options['file'] . "' not found"); } $fileFilter = $options['file']; } } /** @var Account $account */ foreach ($accounts as $accountId => $account) { $writer = $this->writerFactory->create($account); $files = $account->getFiles(); /** @var File $file */ foreach ($files as $file) { if ($fileFilter != null && $file->getId() != $fileFilter) { continue; } $file->setPathname($this->temp->createTmpFile()->getPathname()); try { $tableExporter = new TableExporter($this->storageApi); $tableExporter->exportTable($file->getTableId(), $file->getPathname(), []); } catch (ClientException $e) { throw new UserException($e->getMessage(), $e, ['file' => $file->toArray()]); } $file = $writer->process($file); $status[$account->getAccountName()][$file->getTitle()] = 'ok'; } // updated changes to files $account->save(); } } return $status; }
public function testCreateSandboxInputMapping() { $temp = new Temp(); $tmpDir = $temp->getTmpFolder(); $temp->initRunFolder(); $csv = new CsvFile($tmpDir . "/upload.csv"); $csv->writeRow(["Id", "Name"]); $csv->writeRow(["test", "test"]); $this->client->createTableAsync("in.c-sandbox", "test", $csv); unset($csv); $this->createECSServer(); $jobData = ['params' => ['backend' => 'docker', 'type' => 'rstudio', 'input' => ['tables' => [['source' => 'in.c-sandbox.test', 'destination' => 'source.csv']]]]]; $data = $this->executeJob('create', $jobData); $this->assertArrayHasKey('credentials', $data); $this->assertArrayHasKey('id', $data['credentials']); $doctrine = self::$kernel->getContainer()->get("doctrine"); $server = new Docker($doctrine, new Client(["token" => "dummy"])); $account = $server->getActiveAccount($data["credentials"]["id"]); /** @var $account \Keboola\ProvisioningBundle\Entity\Account\Docker */ $this->assertNotEmpty($account->getCredentials()); $credentials = $account->getCredentials(); $this->assertArrayHasKey('hostname', $credentials); $this->assertArrayHasKey('port', $credentials); $this->assertArrayHasKey('user', $credentials); $this->assertArrayHasKey('password', $credentials); $client = $this->getGuzzleClient(); $url = $credentials['hostname'] . ':' . $credentials['port']; $res = $client->request('GET', $url); $response = (string) $res->getBody(); $this->assertContains('RStudio: Browser Not Supported', $response); }