예제 #1
0
use Oasis\Mlib\AwsWrappers\S3Client;
use Oasis\Mlib\AwsWrappers\StsClient;
use Oasis\Mlib\FlysystemWrappers\ExtendedAwsS3Adapter;
use Oasis\Mlib\FlysystemWrappers\ExtendedFilesystem;
use Oasis\Mlib\FlysystemWrappers\ExtendedLocal;
use Oasis\Mlib\Redshift\DrdStreamWriter;
use Oasis\Mlib\Redshift\RedshiftConnection;
use Oasis\Mlib\Redshift\RedshiftExporter;
use Oasis\Mlib\Redshift\RedshiftImporter;
$awsConfig = ['profile' => "oasis-minhao", 'region' => 'ap-northeast-1'];
$sts = new StsClient(['profile' => 'oasis-minhao', 'region' => 'ap-northeast-1']);
$rs = RedshiftConnection::getConnection(["host" => "oas-dmp-test.cikskyn4dlgm.ap-northeast-1.redshift.amazonaws.com", "port" => 5439, "dbname" => "oasdmp", "user" => "oasdmp", "password" => "NU9qEG3nR8"]);
$localFs = new ExtendedFilesystem(new ExtendedLocal('/tmp'));
$s3Fs = new ExtendedFilesystem(new ExtendedAwsS3Adapter(new S3Client($awsConfig), "minhao-dev", "/tmp"));
$importer = new RedshiftImporter($rs, $localFs, $s3Fs, 'ap-northeast-1', $sts);
$exporter = new RedshiftExporter($rs, $localFs, $s3Fs, 'ap-northeast-1', $sts);
$columns = explode(",", "a1,a2,a3,a4,a5,a6,a7");
$dataPath = 'data';
$localFs->put($dataPath, '');
$drd_os = $localFs->appendStream($dataPath);
$writer = new DrdStreamWriter($drd_os, $columns);
for ($i = 0; $i < 10; ++$i) {
    $data = [];
    for ($j = 0; $j < 7; ++$j) {
        $data['a' . ($j + 1)] = mt_rand(1, 10) + $j * 10;
    }
    $writer->writeRecord($data);
}
fclose($drd_os);
//$importer->importFromFile('/out/testing', 'test', $columns, true, true);
//$importer->importFromFile('ddd', 'test', $columns, true, true);
 public function testDataImportExportWithGzipAndParallel()
 {
     $exportPrefix = "redshift_ut_" . time();
     $this->testDataImport(true);
     $exporter = new RedshiftExporter(self::$rs, self::$localFs, self::$s3Fs, self::$s3Region, self::$sts);
     $exporter->exportToFile($exportPrefix, "SELECT * FROM php_redshift_test", true, true, true);
     $exportedCount = 0;
     $finder = self::$localFs->getFinder();
     $finder->path("#^" . preg_quote($exportPrefix, "#") . "#");
     $unloaded = [];
     foreach ($finder as $splFileInfo) {
         $relativePathname = $splFileInfo->getRelativePathname();
         $unloaded[] = $relativePathname;
         $content = self::$localFs->read($relativePathname);
         mdebug(gzdecode($content));
         $fh = fopen('php://memory', 'r+');
         fwrite($fh, gzdecode($content));
         rewind($fh);
         $reader = new DrdStreamReader($fh, self::FIELDS);
         while ($reader->readRecord()) {
             $exportedCount++;
         }
         fclose($fh);
     }
     self::assertEquals(5, $exportedCount);
     // test import of parallel data
     $importer = new RedshiftImporter(self::$rs, self::$localFs, self::$s3Fs, self::$s3Region, self::$sts);
     $importer->importFromFile($exportPrefix, 'php_redshift_test', self::FIELDS, true, true);
     $stmt = self::$rs->prepare("SELECT COUNT(*) FROM php_redshift_test");
     $stmt->execute();
     $result = $stmt->fetchColumn();
     self::assertEquals(10, $result);
     foreach ($unloaded as $relativePathname) {
         self::$localFs->delete($relativePathname);
     }
 }