public function exportToFile($path, $query, $gzip = false, $isParallel = true, $overwriteExistingFiles = false)
{
$path = ltrim(preg_replace('#/+#', "/", $path), "/");
$s3path = $this->s3Fs->getRealpath($path);
mdebug("export file to temp s3, prefix = %s", $s3path);
$suffix = $gzip ? "\\.gz" : "";
$clearPattern = "#^" . preg_quote($path, "#") . "#";
$downloadPattern = "#^" . preg_quote($path, "#") . "([0-9]{2,})?(_part_[0-9]{2,})?{$suffix}\$#";
mdebug("Clear pattern for %s is %s", $path, $clearPattern);
mdebug("Download pattern for %s is %s", $path, $downloadPattern);
// clear remote path
try {
$finder = $this->s3Fs->getFinder();
$finder->path($clearPattern);
if ($finder->count() > 0) {
if ($overwriteExistingFiles) {
foreach ($finder as $splFileInfo) {
$this->s3Fs->delete($splFileInfo->getRelativePathname());
}
} else {
throw new \RuntimeException(sprintf("The path is not empty on remote end, path = %s", $path));
}
}
} catch (\InvalidArgumentException $e) {
if (strpos($e->getMessage(), "directory does not exist") === false) {
throw $e;
}
}
// clear local path
$finder = $this->localFs->getFinder();
$finder->path($clearPattern);
if ($finder->count() > 0) {
if ($overwriteExistingFiles) {
foreach ($finder as $splFileInfo) {
$this->localFs->delete($splFileInfo->getRelativePathname());
}
} else {
throw new \RuntimeException(sprintf("The path is not empty locally, path = %s", $path));
}
}
$tempCredential = $this->sts->getTemporaryCredential();
$this->connection->unloadToS3($query, $s3path, $tempCredential, true, $gzip, $isParallel);
$finder = $this->s3Fs->getFinder();
$finder->path($downloadPattern);
foreach ($finder as $splFileInfo) {
//var_dump($splFileInfo->getRelativePathname());
$partName = $splFileInfo->getRelativePathname();
$fh = $this->s3Fs->readStream($partName);
$this->localFs->putStream($partName, $fh);
fclose($fh);
$this->s3Fs->delete($partName);
}
}
public function importFromFile($path, $table, $columns, $gzip = false, $overwriteS3Files = false)
{
$timestamp = microtime(true) . getmypid();
$path = ltrim(preg_replace('#/+#', "/", $path), "/");
$suffix = $gzip ? "\\.gz" : "";
$uploadPattern = "#^" . preg_quote($path, "#") . "([0-9]{2,})?(_part_[0-9]{2,})?{$suffix}\$#";
$clearPattern = "#^" . preg_quote($path, "#") . ".*/" . $timestamp . "\$#";
mdebug("Upload pattern is %s", $uploadPattern);
mdebug("Clear pattern is %s", $clearPattern);
$localFinder = $this->localFs->getFinder();
$localFinder->path($uploadPattern);
if ($localFinder->count() == 0) {
throw new \RuntimeException(sprintf("No import files found at path: %s, pattern = %s", $path, $uploadPattern));
}
try {
$s3Finder = $this->s3Fs->getFinder();
$s3Finder->path($clearPattern);
if ($s3Finder->count() > 0) {
if ($overwriteS3Files) {
foreach ($s3Finder as $splFileInfo) {
$this->s3Fs->delete($splFileInfo->getRelativePathname());
}
} else {
throw new \RuntimeException(sprintf("The path is not empty on remote end, path = %s", $path));
}
}
} catch (\InvalidArgumentException $e) {
if (strpos($e->getMessage(), "directory does not exist") === false) {
throw $e;
}
}
$uploaded = [];
foreach ($localFinder as $splFileInfo) {
$relativePathname = $splFileInfo->getRelativePathname();
$remoteName = $relativePathname . "/" . $timestamp;
$fh = $this->localFs->readStream($relativePathname);
// IMPORTANT: use write stream so that s3fs doesn't check for key exisistence, which in turn would bread the strong consistency of s3
$this->s3Fs->writeStream($remoteName, $fh);
fclose($fh);
$uploaded[] = $remoteName;
mdebug("Uploaded %s to %s", $relativePathname, $remoteName);
}
//$inStream = $this->localFs->readStream($path);
//$this->s3Fs->putStream($path, $inStream);
$s3path = $this->s3Fs->getRealpath($path);
mdebug("uploaded file to temp s3, path = %s", $s3path);
$tempCredential = $this->sts->getTemporaryCredential();
$this->connection->copyFromS3($table, $columns, $s3path, $this->s3Region, $tempCredential, true, $gzip);
foreach ($uploaded as $relativePathname) {
$this->s3Fs->delete($relativePathname);
}
}
/**
* NOTE: region of s3 bucket must be the same as redshift cluster
*
* @param $selectStatement
* @param $s3path
* @param TemporaryCredential $tempCredential
* @param bool $escaped
* @param bool $gzip
* @param bool $parallel
*
* @throws \Doctrine\DBAL\DBALException
*/
public function unloadToS3($selectStatement, $s3path, TemporaryCredential $tempCredential, $escaped = true, $gzip = false, $parallel = true)
{
$stmt_template = <<<SQL
UNLOAD (%s)
TO '%s'
CREDENTIALS 'aws_access_key_id=%s;aws_secret_access_key=%s;token=%s'
%s
%s
%s
ALLOWOVERWRITE
SQL;
$stmt = sprintf($stmt_template, $this->normalizeSingleQuotedValue($selectStatement), $this->normalizeS3Path($s3path), $tempCredential->accessKeyId, $tempCredential->secretAccessKey, $tempCredential->sessionToken, $escaped ? "ESCAPE" : "", $gzip ? "GZIP" : "", $parallel ? "" : "PARALLEL OFF");
mdebug("Unloading using stmt:\n%s", $stmt);
$prepared_statement = $this->prepare($stmt);
$prepared_statement->execute();
}
public function count($conditions, array $fields, array $params, $index_name = self::NO_INDEX, $consistent_read = false)
{
$usingScan = $index_name === self::NO_INDEX;
$command = $usingScan ? "scan" : "query";
$requestArgs = ["TableName" => $this->tableName, "Select" => "COUNT"];
if ($conditions) {
$conditionKey = $usingScan ? "FilterExpression" : "KeyConditionExpression";
$requestArgs[$conditionKey] = $conditions;
if ($fields) {
$requestArgs['ExpressionAttributeNames'] = $fields;
}
if ($params) {
$paramsItem = DynamoDbItem::createFromArray($params);
$requestArgs['ExpressionAttributeValues'] = $paramsItem->getData();
}
}
if (!$usingScan) {
$requestArgs['ConsistentRead'] = $consistent_read;
if ($index_name !== self::PRIMARY_INDEX) {
$requestArgs['IndexName'] = $index_name;
}
}
$count = 0;
$scanned = 0;
$last_key = null;
do {
if ($last_key) {
$requestArgs['ExclusiveStartKey'] = $last_key;
}
$result = call_user_func([$this->dbClient, $command], $requestArgs);
$last_key = isset($result['LastEvaluatedKey']) ? $result['LastEvaluatedKey'] : null;
$count += intval($result['Count']);
$scanned += intval($result['ScannedCount']);
} while ($last_key != null);
mdebug("Count = {$count} from total scanned {$scanned}");
return $count;
}
protected function onChildProcessExit($pid, $exitStatus, InputInterface $input, OutputInterface $output)
{
if (($key = array_search($pid, $this->pids)) !== false) {
//mdebug("Child process $pid exit with code: %x", $exitStatus);
array_splice($this->pids, $key, 1);
switch ($exitStatus) {
case self::EXIT_CODE_RESTART:
//mdebug("Child process $pid requires restart");
$newPid = $this->doFork($input, $output);
$this->pids[] = $newPid;
mdebug("Child process {$pid} restarted, new pid = {$newPid}");
break;
case self::EXIT_CODE_OK:
break;
default:
$this->isFailed = true;
}
} else {
mwarning("Un-managed child process {$pid} exit with code: %d", $exitStatus);
}
}
protected function waitForBackgroundProcesses()
{
//$lastMemory = memory_get_usage(true);
while (true) {
//$memory = memory_get_usage(true);
//if ($memory != $lastMemory) {
// echo(
// sprintf("memory change: %d, from %d to %d", $memory - $lastMemory, $lastMemory, $memory)
// . PHP_EOL
// );
//}
//$lastMemory = $memory;
pcntl_signal_dispatch();
$status = 0;
$pid = pcntl_waitpid(-1, $status, WNOHANG);
if ($pid == 0) {
// no child process has quit
usleep(200 * 1000);
} else {
if ($pid > 0) {
// child process with pid = $pid exits
$exitStatus = pcntl_wexitstatus($status);
$runner = $this->runningProcesses[$pid];
if (!$runner instanceof CommandRunner) {
throw new \LogicException("Cannot find command runner for process pid = %d", $pid);
}
unset($this->runningProcesses[$pid]);
$runner->onProcessExit($exitStatus, $pid);
$newPid = $runner->run();
if ($newPid > 0) {
$this->runningProcesses[$newPid] = $runner;
}
} else {
// error
$errno = pcntl_get_last_error();
if ($errno == PCNTL_ECHILD) {
// all children finished
mdebug("No more BackgroundProcessRunner children, continue ...");
break;
} else {
// some other error
throw new \RuntimeException("Error waiting for process, error = " . pcntl_strerror($errno));
}
}
}
}
}
public function run()
{
if ($this->stopped) {
return 0;
}
$pid = pcntl_fork();
if ($pid < 0) {
$errno = pcntl_get_last_error();
throw new \RuntimeException("Cannot fork process, error = " . pcntl_strerror($errno));
} elseif ($pid == 0) {
// in child process
$now = time();
if ($now < $this->nextRun) {
if ($this->traceEnabled) {
mdebug("Will wait %d seconds for next run of %s", $this->nextRun - $now, $this->name);
}
sleep($this->nextRun - $now);
}
// run using application
$this->application->setAutoExit(false);
// we will handle exit on our own
$this->application->setCatchExceptions(false);
// we will catch on our own
try {
$ret = $this->application->run($this->input, $this->output);
} catch (\Exception $e) {
mtrace($e, "Exception while running command {$this->name}", "error");
$ret = AbstractDaemonSentinelCommand::EXIT_CODE_COMMON_ERROR;
}
// Check if we should alert
if ($ret != AbstractDaemonSentinelCommand::EXIT_CODE_OK && $this->alert) {
// alert in child process is better, because we can get more trace here
malert("Daemon command %s failed with exit code = %d", $this->name, $ret);
exit(AbstractDaemonSentinelCommand::EXIT_CODE_OK);
// exit OK because alert is already sent
} else {
exit($ret);
}
} else {
$this->lastRun = $this->nextRun;
return $pid;
}
}
public function testDataImportExportWithGzipAndParallel()
{
$exportPrefix = "redshift_ut_" . time();
$this->testDataImport(true);
$exporter = new RedshiftExporter(self::$rs, self::$localFs, self::$s3Fs, self::$s3Region, self::$sts);
$exporter->exportToFile($exportPrefix, "SELECT * FROM php_redshift_test", true, true, true);
$exportedCount = 0;
$finder = self::$localFs->getFinder();
$finder->path("#^" . preg_quote($exportPrefix, "#") . "#");
$unloaded = [];
foreach ($finder as $splFileInfo) {
$relativePathname = $splFileInfo->getRelativePathname();
$unloaded[] = $relativePathname;
$content = self::$localFs->read($relativePathname);
mdebug(gzdecode($content));
$fh = fopen('php://memory', 'r+');
fwrite($fh, gzdecode($content));
rewind($fh);
$reader = new DrdStreamReader($fh, self::FIELDS);
while ($reader->readRecord()) {
$exportedCount++;
}
fclose($fh);
}
self::assertEquals(5, $exportedCount);
// test import of parallel data
$importer = new RedshiftImporter(self::$rs, self::$localFs, self::$s3Fs, self::$s3Region, self::$sts);
$importer->importFromFile($exportPrefix, 'php_redshift_test', self::FIELDS, true, true);
$stmt = self::$rs->prepare("SELECT COUNT(*) FROM php_redshift_test");
$stmt->execute();
$result = $stmt->fetchColumn();
self::assertEquals(10, $result);
foreach ($unloaded as $relativePathname) {
self::$localFs->delete($relativePathname);
}
}
function int_reg($int)
{
global $registers;
if ($int <= 32767) {
return $int;
}
$reg = $int % 32768;
mdebug("REG {$reg} = " . $registers[$reg]);
return $registers[$reg];
}
<?php /** * Created by PhpStorm. * User: minhao * Date: 2016-03-17 * Time: 20:24 */ use Oasis\SlimApp\SlimApp; require_once __DIR__ . "/bootstrap.php"; mdebug(getcwd()); SlimApp::app()->getHttpKernel()->run();
