/** * @param string $backend * @return Server */ public function getServer($backend) { switch ($backend) { case 'docker': return new Docker($this->registry, $this->storageApi->getClient()); default: throw new UserException("Backend '{$backend}' not found. Possible typo in URL?"); } }
public function setUp() { $storageApiService = new StorageApiService(new RequestStack()); $storageApiService->setClient(new Client(['token' => SYRUP_SAPI_TEST_TOKEN, 'url' => SYRUP_SAPI_TEST_URL])); $uploader = new UploaderS3(['aws-access-key' => SYRUP_AWS_KEY, 'aws-secret-key' => SYRUP_AWS_SECRET, 's3-upload-path' => SYRUP_S3_BUCKET, 'aws-region' => SYRUP_AWS_REGION, 'url-prefix' => 'https://connection.keboola.com/admin/utils/logs?file=']); $this->testLogHandler = new TestHandler(); $this->testLogHandler->setFormatter(new JsonFormatter()); $this->testLogHandler->pushProcessor(new SyslogProcessor(SYRUP_APP_NAME, $storageApiService, $uploader)); $logger = new \Monolog\Logger('test', [$this->testLogHandler]); $this->listener = new SyrupExceptionListener(SYRUP_APP_NAME, $storageApiService, $logger); }
public function __construct($appName, StorageApiService $storageApiService, Logger $logger) { $this->appName = $appName; try { $storageApiClient = $storageApiService->getClient(); $this->runId = $storageApiClient->getRunId(); } catch (NoRequestException $e) { } catch (UserException $e) { } $this->logger = $logger; }
public function setUp() { $storageApiService = new StorageApiService(); $storageApiService->setClient(new Client(['token' => SYRUP_SAPI_TEST_TOKEN])); $uploader = new Uploader(['aws-access-key' => SYRUP_AWS_KEY, 'aws-secret-key' => SYRUP_AWS_SECRET, 's3-upload-path' => SYRUP_S3_BUCKET, 'aws-region' => SYRUP_AWS_REGION]); $this->testLogHandler = new TestHandler(); $this->testLogHandler->setFormatter(new JsonFormatter()); $this->testLogHandler->pushProcessor(new SyslogProcessor(SYRUP_APP_NAME, $storageApiService, $uploader)); $logger = new \Monolog\Logger('test', [$this->testLogHandler]); $this->listener = new SyrupExceptionListener(SYRUP_APP_NAME, $storageApiService, $logger); }
protected function initStorageApiClient() { try { $this->storageApiClient = $this->storageApiService->getClient(); } catch (NoRequestException $e) { // Ignore when no SAPI client setup } catch (UserException $e) { // Ignore when no SAPI client setup } catch (SapiException $e) { // Ignore when SAPI client setup is wrong } }
public function __construct($componentName, StorageApiService $storageApiService, Uploader $s3Uploader) { $this->componentName = $componentName; $this->s3Uploader = $s3Uploader; try { // does not work for some commands $storageApiClient = $storageApiService->getClient(); $this->tokenData = $storageApiClient->getLogData(); $this->runId = $storageApiClient->getRunId(); } catch (SyrupComponentException $e) { } catch (SapiException $e) { } }
public function create($command, array $params = [], $lockName = null) { $this->storageApiClient = $this->storageApiService->getClient(); $tokenData = $this->storageApiService->getTokenData(); $job = new Job($this->objectEncryptor, ['id' => $this->storageApiClient->generateId(), 'runId' => $this->storageApiClient->generateRunId($this->storageApiClient->getRunId()), 'project' => ['id' => $tokenData['owner']['id'], 'name' => $tokenData['owner']['name']], 'token' => ['id' => $tokenData['id'], 'description' => $tokenData['description'], 'token' => $this->objectEncryptor->encrypt($this->storageApiClient->getTokenString())], 'component' => $this->componentName, 'command' => $command, 'params' => $params, 'process' => ['host' => gethostname(), 'pid' => getmypid()], 'nestingLevel' => 0, 'createdTime' => date('c')], null, null, null); if ($lockName) { $job->setLockName($lockName); } $componentConfiguration = $this->getComponentConfiguration(); if (isset($componentConfiguration['flags']) && in_array('encrypt', $componentConfiguration['flags'])) { $job->setEncrypted(true); } return $job; }
private function isParallelLimitExceeded() { // skip validation for components without limit if (in_array($this->job->getComponent(), Limits::unlimitedComponents())) { $this->logger->debug('isParallelLimitExceeded - NO - unlimited component'); return false; } $maxLimit = Limits::getParallelLimit($this->storageApiService->getTokenData()); /** @var Search $elasticSearch */ $elasticSearch = $this->getContainer()->get('syrup.elasticsearch.search'); $jobs = $elasticSearch->getJobs(array('projectId' => $this->job->getProject()['id'], 'query' => sprintf('(%s) AND (%s)', sprintf('status:%s OR status:%s', Job::STATUS_PROCESSING, Job::STATUS_TERMINATING), implode(' AND ', array_map(function ($name) { return '-component:' . $name; }, Limits::unlimitedComponents()))))); if (count($jobs) < $maxLimit) { $this->logger->debug('isParallelLimitExceeded - NO - free workers ' . ($maxLimit - count($jobs))); return false; } $this->logger->debug('isParallelLimitExceeded - full workers ' . ($maxLimit - count($jobs))); if ($this->job->getNestingLevel() >= 1) { $runIds = explode('.', $this->job->getRunId()); unset($runIds[count($runIds) - 1]); $jobs = $elasticSearch->getJobs(array('projectId' => $this->job->getProject()['id'], 'query' => sprintf('(%s) AND (%s) AND (%s) AND (%s)', sprintf('status:%s OR status:%s', Job::STATUS_PROCESSING, Job::STATUS_TERMINATING), implode(' AND ', array_map(function ($name) { return '-component:' . $name; }, Limits::unlimitedComponents())), sprintf('runId:%s.*', implode('.', $runIds)), sprintf('nestingLevel:%s', $this->job->getNestingLevel())))); if (!count($jobs)) { $this->logger->debug('isParallelLimitExceeded - NO - free at nesting level'); return false; } } $this->logger->debug('isParallelLimitExceeded - YES - any free worker'); return true; }
private function initHandlerAndClient() { $request = new Request(); $request->headers->add(['x-storageapi-token' => SYRUP_SAPI_TEST_TOKEN]); $requestStack = new RequestStack(); $requestStack->push($request); $storageApiService = new StorageApiService('https://connection.keboola.com', $requestStack); $client = $storageApiService->getClient(); $client->setRunId(uniqid()); $handler = new StorageApiHandler(SYRUP_APP_NAME, $storageApiService); return [$client, $handler]; }
public function testProcessorInitInvalidTokenLong() { $s3Uploader = new Uploader(['aws-access-key' => SYRUP_AWS_KEY, 'aws-secret-key' => SYRUP_AWS_SECRET, 'aws-region' => SYRUP_AWS_REGION, 's3-upload-path' => SYRUP_S3_BUCKET]); $request = new Request(); $request->headers->add(['x-storageapi-token' => 'invalid']); $storageApiService = new StorageApiService(); $storageApiService->setRequest($request); $record = $this->getRecord(Logger::WARNING, str_repeat('batman', 1000)); // instantiation must not fail $processor = new SyslogProcessor(SYRUP_APP_NAME, $storageApiService, $s3Uploader); $newRecord = $processor($record); $this->assertEquals(7, count($newRecord)); $this->assertArrayHasKey('message', $newRecord); $this->assertArrayHasKey('component', $newRecord); $this->assertArrayHasKey('runId', $newRecord); $this->assertArrayHasKey('pid', $newRecord); $this->assertArrayHasKey('priority', $newRecord); $this->assertArrayHasKey('level', $newRecord); $this->assertArrayHasKey('attachment', $newRecord); $this->assertArrayNotHasKey('token', $newRecord); $this->assertEquals(SYRUP_APP_NAME, $newRecord['component']); $this->assertEquals(Logger::WARNING, $newRecord['level']); $this->assertEquals('WARNING', $newRecord['priority']); $this->assertLessThan(strlen($record['message']), strlen($newRecord['message'])); }