/**
* @param string $backend
* @return Server
*/
public function getServer($backend)
{
switch ($backend) {
case 'docker':
return new Docker($this->registry, $this->storageApi->getClient());
default:
throw new UserException("Backend '{$backend}' not found. Possible typo in URL?");
}
}
public function setUp()
{
$storageApiService = new StorageApiService(new RequestStack());
$storageApiService->setClient(new Client(['token' => SYRUP_SAPI_TEST_TOKEN, 'url' => SYRUP_SAPI_TEST_URL]));
$uploader = new UploaderS3(['aws-access-key' => SYRUP_AWS_KEY, 'aws-secret-key' => SYRUP_AWS_SECRET, 's3-upload-path' => SYRUP_S3_BUCKET, 'aws-region' => SYRUP_AWS_REGION, 'url-prefix' => 'https://connection.keboola.com/admin/utils/logs?file=']);
$this->testLogHandler = new TestHandler();
$this->testLogHandler->setFormatter(new JsonFormatter());
$this->testLogHandler->pushProcessor(new SyslogProcessor(SYRUP_APP_NAME, $storageApiService, $uploader));
$logger = new \Monolog\Logger('test', [$this->testLogHandler]);
$this->listener = new SyrupExceptionListener(SYRUP_APP_NAME, $storageApiService, $logger);
}
public function __construct($appName, StorageApiService $storageApiService, Logger $logger)
{
$this->appName = $appName;
try {
$storageApiClient = $storageApiService->getClient();
$this->runId = $storageApiClient->getRunId();
} catch (NoRequestException $e) {
} catch (UserException $e) {
}
$this->logger = $logger;
}
public function setUp()
{
$storageApiService = new StorageApiService();
$storageApiService->setClient(new Client(['token' => SYRUP_SAPI_TEST_TOKEN]));
$uploader = new Uploader(['aws-access-key' => SYRUP_AWS_KEY, 'aws-secret-key' => SYRUP_AWS_SECRET, 's3-upload-path' => SYRUP_S3_BUCKET, 'aws-region' => SYRUP_AWS_REGION]);
$this->testLogHandler = new TestHandler();
$this->testLogHandler->setFormatter(new JsonFormatter());
$this->testLogHandler->pushProcessor(new SyslogProcessor(SYRUP_APP_NAME, $storageApiService, $uploader));
$logger = new \Monolog\Logger('test', [$this->testLogHandler]);
$this->listener = new SyrupExceptionListener(SYRUP_APP_NAME, $storageApiService, $logger);
}
protected function initStorageApiClient()
{
try {
$this->storageApiClient = $this->storageApiService->getClient();
} catch (NoRequestException $e) {
// Ignore when no SAPI client setup
} catch (UserException $e) {
// Ignore when no SAPI client setup
} catch (SapiException $e) {
// Ignore when SAPI client setup is wrong
}
}
public function __construct($componentName, StorageApiService $storageApiService, Uploader $s3Uploader)
{
$this->componentName = $componentName;
$this->s3Uploader = $s3Uploader;
try {
// does not work for some commands
$storageApiClient = $storageApiService->getClient();
$this->tokenData = $storageApiClient->getLogData();
$this->runId = $storageApiClient->getRunId();
} catch (SyrupComponentException $e) {
} catch (SapiException $e) {
}
}
public function create($command, array $params = [], $lockName = null)
{
$this->storageApiClient = $this->storageApiService->getClient();
$tokenData = $this->storageApiService->getTokenData();
$job = new Job($this->objectEncryptor, ['id' => $this->storageApiClient->generateId(), 'runId' => $this->storageApiClient->generateRunId($this->storageApiClient->getRunId()), 'project' => ['id' => $tokenData['owner']['id'], 'name' => $tokenData['owner']['name']], 'token' => ['id' => $tokenData['id'], 'description' => $tokenData['description'], 'token' => $this->objectEncryptor->encrypt($this->storageApiClient->getTokenString())], 'component' => $this->componentName, 'command' => $command, 'params' => $params, 'process' => ['host' => gethostname(), 'pid' => getmypid()], 'nestingLevel' => 0, 'createdTime' => date('c')], null, null, null);
if ($lockName) {
$job->setLockName($lockName);
}
$componentConfiguration = $this->getComponentConfiguration();
if (isset($componentConfiguration['flags']) && in_array('encrypt', $componentConfiguration['flags'])) {
$job->setEncrypted(true);
}
return $job;
}
private function isParallelLimitExceeded()
{
// skip validation for components without limit
if (in_array($this->job->getComponent(), Limits::unlimitedComponents())) {
$this->logger->debug('isParallelLimitExceeded - NO - unlimited component');
return false;
}
$maxLimit = Limits::getParallelLimit($this->storageApiService->getTokenData());
/** @var Search $elasticSearch */
$elasticSearch = $this->getContainer()->get('syrup.elasticsearch.search');
$jobs = $elasticSearch->getJobs(array('projectId' => $this->job->getProject()['id'], 'query' => sprintf('(%s) AND (%s)', sprintf('status:%s OR status:%s', Job::STATUS_PROCESSING, Job::STATUS_TERMINATING), implode(' AND ', array_map(function ($name) {
return '-component:' . $name;
}, Limits::unlimitedComponents())))));
if (count($jobs) < $maxLimit) {
$this->logger->debug('isParallelLimitExceeded - NO - free workers ' . ($maxLimit - count($jobs)));
return false;
}
$this->logger->debug('isParallelLimitExceeded - full workers ' . ($maxLimit - count($jobs)));
if ($this->job->getNestingLevel() >= 1) {
$runIds = explode('.', $this->job->getRunId());
unset($runIds[count($runIds) - 1]);
$jobs = $elasticSearch->getJobs(array('projectId' => $this->job->getProject()['id'], 'query' => sprintf('(%s) AND (%s) AND (%s) AND (%s)', sprintf('status:%s OR status:%s', Job::STATUS_PROCESSING, Job::STATUS_TERMINATING), implode(' AND ', array_map(function ($name) {
return '-component:' . $name;
}, Limits::unlimitedComponents())), sprintf('runId:%s.*', implode('.', $runIds)), sprintf('nestingLevel:%s', $this->job->getNestingLevel()))));
if (!count($jobs)) {
$this->logger->debug('isParallelLimitExceeded - NO - free at nesting level');
return false;
}
}
$this->logger->debug('isParallelLimitExceeded - YES - any free worker');
return true;
}
private function initHandlerAndClient()
{
$request = new Request();
$request->headers->add(['x-storageapi-token' => SYRUP_SAPI_TEST_TOKEN]);
$requestStack = new RequestStack();
$requestStack->push($request);
$storageApiService = new StorageApiService('https://connection.keboola.com', $requestStack);
$client = $storageApiService->getClient();
$client->setRunId(uniqid());
$handler = new StorageApiHandler(SYRUP_APP_NAME, $storageApiService);
return [$client, $handler];
}
public function testProcessorInitInvalidTokenLong()
{
$s3Uploader = new Uploader(['aws-access-key' => SYRUP_AWS_KEY, 'aws-secret-key' => SYRUP_AWS_SECRET, 'aws-region' => SYRUP_AWS_REGION, 's3-upload-path' => SYRUP_S3_BUCKET]);
$request = new Request();
$request->headers->add(['x-storageapi-token' => 'invalid']);
$storageApiService = new StorageApiService();
$storageApiService->setRequest($request);
$record = $this->getRecord(Logger::WARNING, str_repeat('batman', 1000));
// instantiation must not fail
$processor = new SyslogProcessor(SYRUP_APP_NAME, $storageApiService, $s3Uploader);
$newRecord = $processor($record);
$this->assertEquals(7, count($newRecord));
$this->assertArrayHasKey('message', $newRecord);
$this->assertArrayHasKey('component', $newRecord);
$this->assertArrayHasKey('runId', $newRecord);
$this->assertArrayHasKey('pid', $newRecord);
$this->assertArrayHasKey('priority', $newRecord);
$this->assertArrayHasKey('level', $newRecord);
$this->assertArrayHasKey('attachment', $newRecord);
$this->assertArrayNotHasKey('token', $newRecord);
$this->assertEquals(SYRUP_APP_NAME, $newRecord['component']);
$this->assertEquals(Logger::WARNING, $newRecord['level']);
$this->assertEquals('WARNING', $newRecord['priority']);
$this->assertLessThan(strlen($record['message']), strlen($newRecord['message']));
}
