public function testDataImport($gzipped = false) { $data = []; for ($i = 1; $i <= 5; ++$i) { $row = []; for ($j = 1; $j <= 7; ++$j) { $row['a' . $j] = $j + $i * 10; } $data[] = $row; } $filename = $gzipped ? "data.gz" : "data"; $out = fopen('php://memory', 'r+'); $writer = new DrdStreamWriter($out, self::FIELDS); foreach ($data as $row) { $writer->writeRecord($row); } rewind($out); $data = stream_get_contents($out); if ($gzipped) { $data = gzencode($data); } self::$localFs->put($filename, $data); fclose($out); $importer = new RedshiftImporter(self::$rs, self::$localFs, self::$s3Fs, self::$s3Region, self::$sts); $importer->importFromFile('data', 'php_redshift_test', self::FIELDS, $gzipped, true); $stmt = self::$rs->prepare("SELECT COUNT(*) FROM php_redshift_test"); $stmt->execute(); $result = $stmt->fetchColumn(); self::$localFs->delete($filename); self::assertEquals(5, $result); }
public function testOutput() { $fh = fopen(self::$file, 'w'); $writer = new DrdStreamWriter($fh, self::$fields); foreach (self::$objects as $obj) { $writer->writeRecord($obj); } fclose($fh); $content = <<<FILE Josh|13|The o'looka police Martin|3| O'riel|21|pub \\| priv Nanting||glad to\\ win |0|not yet born || FILE; $this->assertStringEqualsFile(self::$file, $content); return $content; }
use Oasis\Mlib\AwsWrappers\StsClient; use Oasis\Mlib\FlysystemWrappers\ExtendedAwsS3Adapter; use Oasis\Mlib\FlysystemWrappers\ExtendedFilesystem; use Oasis\Mlib\FlysystemWrappers\ExtendedLocal; use Oasis\Mlib\Redshift\DrdStreamWriter; use Oasis\Mlib\Redshift\RedshiftConnection; use Oasis\Mlib\Redshift\RedshiftExporter; use Oasis\Mlib\Redshift\RedshiftImporter; $awsConfig = ['profile' => "oasis-minhao", 'region' => 'ap-northeast-1']; $sts = new StsClient(['profile' => 'oasis-minhao', 'region' => 'ap-northeast-1']); $rs = RedshiftConnection::getConnection(["host" => "oas-dmp-test.cikskyn4dlgm.ap-northeast-1.redshift.amazonaws.com", "port" => 5439, "dbname" => "oasdmp", "user" => "oasdmp", "password" => "NU9qEG3nR8"]); $localFs = new ExtendedFilesystem(new ExtendedLocal('/tmp')); $s3Fs = new ExtendedFilesystem(new ExtendedAwsS3Adapter(new S3Client($awsConfig), "minhao-dev", "/tmp")); $importer = new RedshiftImporter($rs, $localFs, $s3Fs, 'ap-northeast-1', $sts); $exporter = new RedshiftExporter($rs, $localFs, $s3Fs, 'ap-northeast-1', $sts); $columns = explode(",", "a1,a2,a3,a4,a5,a6,a7"); $dataPath = 'data'; $localFs->put($dataPath, ''); $drd_os = $localFs->appendStream($dataPath); $writer = new DrdStreamWriter($drd_os, $columns); for ($i = 0; $i < 10; ++$i) { $data = []; for ($j = 0; $j < 7; ++$j) { $data['a' . ($j + 1)] = mt_rand(1, 10) + $j * 10; } $writer->writeRecord($data); } fclose($drd_os); //$importer->importFromFile('/out/testing', 'test', $columns, true, true); //$importer->importFromFile('ddd', 'test', $columns, true, true); $exporter->exportToFile('/out//testing', 'SELECT * FROM test', false, true, true);