public function exportToFile($path, $query, $gzip = false, $isParallel = true, $overwriteExistingFiles = false) { $path = ltrim(preg_replace('#/+#', "/", $path), "/"); $s3path = $this->s3Fs->getRealpath($path); mdebug("export file to temp s3, prefix = %s", $s3path); $suffix = $gzip ? "\\.gz" : ""; $clearPattern = "#^" . preg_quote($path, "#") . "#"; $downloadPattern = "#^" . preg_quote($path, "#") . "([0-9]{2,})?(_part_[0-9]{2,})?{$suffix}\$#"; mdebug("Clear pattern for %s is %s", $path, $clearPattern); mdebug("Download pattern for %s is %s", $path, $downloadPattern); // clear remote path try { $finder = $this->s3Fs->getFinder(); $finder->path($clearPattern); if ($finder->count() > 0) { if ($overwriteExistingFiles) { foreach ($finder as $splFileInfo) { $this->s3Fs->delete($splFileInfo->getRelativePathname()); } } else { throw new \RuntimeException(sprintf("The path is not empty on remote end, path = %s", $path)); } } } catch (\InvalidArgumentException $e) { if (strpos($e->getMessage(), "directory does not exist") === false) { throw $e; } } // clear local path $finder = $this->localFs->getFinder(); $finder->path($clearPattern); if ($finder->count() > 0) { if ($overwriteExistingFiles) { foreach ($finder as $splFileInfo) { $this->localFs->delete($splFileInfo->getRelativePathname()); } } else { throw new \RuntimeException(sprintf("The path is not empty locally, path = %s", $path)); } } $tempCredential = $this->sts->getTemporaryCredential(); $this->connection->unloadToS3($query, $s3path, $tempCredential, true, $gzip, $isParallel); $finder = $this->s3Fs->getFinder(); $finder->path($downloadPattern); foreach ($finder as $splFileInfo) { //var_dump($splFileInfo->getRelativePathname()); $partName = $splFileInfo->getRelativePathname(); $fh = $this->s3Fs->readStream($partName); $this->localFs->putStream($partName, $fh); fclose($fh); $this->s3Fs->delete($partName); } }
public function importFromFile($path, $table, $columns, $gzip = false, $overwriteS3Files = false) { $timestamp = microtime(true) . getmypid(); $path = ltrim(preg_replace('#/+#', "/", $path), "/"); $suffix = $gzip ? "\\.gz" : ""; $uploadPattern = "#^" . preg_quote($path, "#") . "([0-9]{2,})?(_part_[0-9]{2,})?{$suffix}\$#"; $clearPattern = "#^" . preg_quote($path, "#") . ".*/" . $timestamp . "\$#"; mdebug("Upload pattern is %s", $uploadPattern); mdebug("Clear pattern is %s", $clearPattern); $localFinder = $this->localFs->getFinder(); $localFinder->path($uploadPattern); if ($localFinder->count() == 0) { throw new \RuntimeException(sprintf("No import files found at path: %s, pattern = %s", $path, $uploadPattern)); } try { $s3Finder = $this->s3Fs->getFinder(); $s3Finder->path($clearPattern); if ($s3Finder->count() > 0) { if ($overwriteS3Files) { foreach ($s3Finder as $splFileInfo) { $this->s3Fs->delete($splFileInfo->getRelativePathname()); } } else { throw new \RuntimeException(sprintf("The path is not empty on remote end, path = %s", $path)); } } } catch (\InvalidArgumentException $e) { if (strpos($e->getMessage(), "directory does not exist") === false) { throw $e; } } $uploaded = []; foreach ($localFinder as $splFileInfo) { $relativePathname = $splFileInfo->getRelativePathname(); $remoteName = $relativePathname . "/" . $timestamp; $fh = $this->localFs->readStream($relativePathname); // IMPORTANT: use write stream so that s3fs doesn't check for key exisistence, which in turn would bread the strong consistency of s3 $this->s3Fs->writeStream($remoteName, $fh); fclose($fh); $uploaded[] = $remoteName; mdebug("Uploaded %s to %s", $relativePathname, $remoteName); } //$inStream = $this->localFs->readStream($path); //$this->s3Fs->putStream($path, $inStream); $s3path = $this->s3Fs->getRealpath($path); mdebug("uploaded file to temp s3, path = %s", $s3path); $tempCredential = $this->sts->getTemporaryCredential(); $this->connection->copyFromS3($table, $columns, $s3path, $this->s3Region, $tempCredential, true, $gzip); foreach ($uploaded as $relativePathname) { $this->s3Fs->delete($relativePathname); } }
/** * NOTE: region of s3 bucket must be the same as redshift cluster * * @param $selectStatement * @param $s3path * @param TemporaryCredential $tempCredential * @param bool $escaped * @param bool $gzip * @param bool $parallel * * @throws \Doctrine\DBAL\DBALException */ public function unloadToS3($selectStatement, $s3path, TemporaryCredential $tempCredential, $escaped = true, $gzip = false, $parallel = true) { $stmt_template = <<<SQL UNLOAD (%s) TO '%s' CREDENTIALS 'aws_access_key_id=%s;aws_secret_access_key=%s;token=%s' %s %s %s ALLOWOVERWRITE SQL; $stmt = sprintf($stmt_template, $this->normalizeSingleQuotedValue($selectStatement), $this->normalizeS3Path($s3path), $tempCredential->accessKeyId, $tempCredential->secretAccessKey, $tempCredential->sessionToken, $escaped ? "ESCAPE" : "", $gzip ? "GZIP" : "", $parallel ? "" : "PARALLEL OFF"); mdebug("Unloading using stmt:\n%s", $stmt); $prepared_statement = $this->prepare($stmt); $prepared_statement->execute(); }
public function count($conditions, array $fields, array $params, $index_name = self::NO_INDEX, $consistent_read = false) { $usingScan = $index_name === self::NO_INDEX; $command = $usingScan ? "scan" : "query"; $requestArgs = ["TableName" => $this->tableName, "Select" => "COUNT"]; if ($conditions) { $conditionKey = $usingScan ? "FilterExpression" : "KeyConditionExpression"; $requestArgs[$conditionKey] = $conditions; if ($fields) { $requestArgs['ExpressionAttributeNames'] = $fields; } if ($params) { $paramsItem = DynamoDbItem::createFromArray($params); $requestArgs['ExpressionAttributeValues'] = $paramsItem->getData(); } } if (!$usingScan) { $requestArgs['ConsistentRead'] = $consistent_read; if ($index_name !== self::PRIMARY_INDEX) { $requestArgs['IndexName'] = $index_name; } } $count = 0; $scanned = 0; $last_key = null; do { if ($last_key) { $requestArgs['ExclusiveStartKey'] = $last_key; } $result = call_user_func([$this->dbClient, $command], $requestArgs); $last_key = isset($result['LastEvaluatedKey']) ? $result['LastEvaluatedKey'] : null; $count += intval($result['Count']); $scanned += intval($result['ScannedCount']); } while ($last_key != null); mdebug("Count = {$count} from total scanned {$scanned}"); return $count; }
protected function onChildProcessExit($pid, $exitStatus, InputInterface $input, OutputInterface $output) { if (($key = array_search($pid, $this->pids)) !== false) { //mdebug("Child process $pid exit with code: %x", $exitStatus); array_splice($this->pids, $key, 1); switch ($exitStatus) { case self::EXIT_CODE_RESTART: //mdebug("Child process $pid requires restart"); $newPid = $this->doFork($input, $output); $this->pids[] = $newPid; mdebug("Child process {$pid} restarted, new pid = {$newPid}"); break; case self::EXIT_CODE_OK: break; default: $this->isFailed = true; } } else { mwarning("Un-managed child process {$pid} exit with code: %d", $exitStatus); } }
protected function waitForBackgroundProcesses() { //$lastMemory = memory_get_usage(true); while (true) { //$memory = memory_get_usage(true); //if ($memory != $lastMemory) { // echo( // sprintf("memory change: %d, from %d to %d", $memory - $lastMemory, $lastMemory, $memory) // . PHP_EOL // ); //} //$lastMemory = $memory; pcntl_signal_dispatch(); $status = 0; $pid = pcntl_waitpid(-1, $status, WNOHANG); if ($pid == 0) { // no child process has quit usleep(200 * 1000); } else { if ($pid > 0) { // child process with pid = $pid exits $exitStatus = pcntl_wexitstatus($status); $runner = $this->runningProcesses[$pid]; if (!$runner instanceof CommandRunner) { throw new \LogicException("Cannot find command runner for process pid = %d", $pid); } unset($this->runningProcesses[$pid]); $runner->onProcessExit($exitStatus, $pid); $newPid = $runner->run(); if ($newPid > 0) { $this->runningProcesses[$newPid] = $runner; } } else { // error $errno = pcntl_get_last_error(); if ($errno == PCNTL_ECHILD) { // all children finished mdebug("No more BackgroundProcessRunner children, continue ..."); break; } else { // some other error throw new \RuntimeException("Error waiting for process, error = " . pcntl_strerror($errno)); } } } } }
public function run() { if ($this->stopped) { return 0; } $pid = pcntl_fork(); if ($pid < 0) { $errno = pcntl_get_last_error(); throw new \RuntimeException("Cannot fork process, error = " . pcntl_strerror($errno)); } elseif ($pid == 0) { // in child process $now = time(); if ($now < $this->nextRun) { if ($this->traceEnabled) { mdebug("Will wait %d seconds for next run of %s", $this->nextRun - $now, $this->name); } sleep($this->nextRun - $now); } // run using application $this->application->setAutoExit(false); // we will handle exit on our own $this->application->setCatchExceptions(false); // we will catch on our own try { $ret = $this->application->run($this->input, $this->output); } catch (\Exception $e) { mtrace($e, "Exception while running command {$this->name}", "error"); $ret = AbstractDaemonSentinelCommand::EXIT_CODE_COMMON_ERROR; } // Check if we should alert if ($ret != AbstractDaemonSentinelCommand::EXIT_CODE_OK && $this->alert) { // alert in child process is better, because we can get more trace here malert("Daemon command %s failed with exit code = %d", $this->name, $ret); exit(AbstractDaemonSentinelCommand::EXIT_CODE_OK); // exit OK because alert is already sent } else { exit($ret); } } else { $this->lastRun = $this->nextRun; return $pid; } }
public function testDataImportExportWithGzipAndParallel() { $exportPrefix = "redshift_ut_" . time(); $this->testDataImport(true); $exporter = new RedshiftExporter(self::$rs, self::$localFs, self::$s3Fs, self::$s3Region, self::$sts); $exporter->exportToFile($exportPrefix, "SELECT * FROM php_redshift_test", true, true, true); $exportedCount = 0; $finder = self::$localFs->getFinder(); $finder->path("#^" . preg_quote($exportPrefix, "#") . "#"); $unloaded = []; foreach ($finder as $splFileInfo) { $relativePathname = $splFileInfo->getRelativePathname(); $unloaded[] = $relativePathname; $content = self::$localFs->read($relativePathname); mdebug(gzdecode($content)); $fh = fopen('php://memory', 'r+'); fwrite($fh, gzdecode($content)); rewind($fh); $reader = new DrdStreamReader($fh, self::FIELDS); while ($reader->readRecord()) { $exportedCount++; } fclose($fh); } self::assertEquals(5, $exportedCount); // test import of parallel data $importer = new RedshiftImporter(self::$rs, self::$localFs, self::$s3Fs, self::$s3Region, self::$sts); $importer->importFromFile($exportPrefix, 'php_redshift_test', self::FIELDS, true, true); $stmt = self::$rs->prepare("SELECT COUNT(*) FROM php_redshift_test"); $stmt->execute(); $result = $stmt->fetchColumn(); self::assertEquals(10, $result); foreach ($unloaded as $relativePathname) { self::$localFs->delete($relativePathname); } }
function int_reg($int) { global $registers; if ($int <= 32767) { return $int; } $reg = $int % 32768; mdebug("REG {$reg} = " . $registers[$reg]); return $registers[$reg]; }
<?php /** * Created by PhpStorm. * User: minhao * Date: 2016-03-17 * Time: 20:24 */ use Oasis\SlimApp\SlimApp; require_once __DIR__ . "/bootstrap.php"; mdebug(getcwd()); SlimApp::app()->getHttpKernel()->run();