public function testFeature()
{
$this->assertTrue(defined('TEST_ORCHESTRATOR_HAS_FEATURE_SAPI_TOKEN'));
$this->assertNotEmpty(constant('TEST_ORCHESTRATOR_HAS_FEATURE_SAPI_TOKEN'));
$this->assertTrue(defined('TEST_ORCHESTRATOR_FEATURE'));
$this->assertNotEmpty(constant('TEST_ORCHESTRATOR_FEATURE'));
$token = new Token(new StorageApiClient(array('token' => TEST_ORCHESTRATOR_HAS_FEATURE_SAPI_TOKEN, 'url' => TEST_ORCHESTRATOR_SAPI_URL)));
$this->assertTrue($token->hasEnabledFeature(TEST_ORCHESTRATOR_FEATURE));
$this->assertFalse($token->hasEnabledFeature(md5(TEST_ORCHESTRATOR_FEATURE)));
}
public function __construct(StorageApiClient $storageApi, Rds\OrchestrationManager $orchestrationManager, StorageApi\Token $token)
{
$this->storageApi = $storageApi;
$this->orchestrationManager = $orchestrationManager;
$this->token = $token;
// token with token manipulation permission
if (!$this->token->canManageTokens()) {
throw new AccessDeniedHttpException('Token has no permissions for token manipulation');
}
}
/**
* Cancel waiting job
*
* @param $jobId
* @throws \Keboola\OrchestratorBundle\Exception\OrchestratorException
* @return Response
*/
public function deleteAction($jobId)
{
$job = $this->jobEsManager->findJobById($jobId);
if (!$job) {
$exception = new OrchestratorException(404, sprintf('Job %s not found', $jobId));
$exception->setExceptionCode('JOB_NOT_FOUND');
throw $exception;
}
if (!$job->isWaiting()) {
$exception = new OrchestratorException(403, 'Only waiting job can be cancelled');
$exception->setExceptionCode('INVALID_JOB_STATUS');
throw $exception;
}
$client = new GuzzleHttp\Client([]);
try {
$response = $client->post(sprintf($this->killUrl, $job->getId()), array('config' => array('curl' => array(CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_0)), 'headers' => array('X-StorageApi-Token' => $this->token, 'X-KBC-RunId' => $this->storageApi->getRunId(), 'X-User-Agent', KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME . " - API cancel"), 'timeout' => 60));
if ($response->getStatusCode() == 202) {
$this->logger->info(sprintf('Orchestration job %s cancelled', $job->getId()));
$this->logger->debug(sprintf('Orchestration job %s cancelled - old api', $job->getId()), array('orchestrationId' => $job->getOrchestrationId(), 'orchestration' => $job->getOrchestrationName(), 'projectId' => $job->getProjectId(), 'project' => $job->getTokenOwnerName(), 'tokenId' => $this->token->getId(), 'token' => $this->token->getDescription()));
} else {
throw new ApplicationException('Job cancel via syrup api failed', null, array('statusCode' => $response->getStatusCode(), 'body' => ResponseDecoder::decode($response)));
}
} catch (GuzzleHttp\Exception\RequestException $e) {
throw new ApplicationException('Job cancel via syrup api failed', $e);
}
$jsonResponse = $this->createJsonResponse(array(), 204);
return $jsonResponse;
}
/**
* Prepare environment for test
*/
protected function setUp()
{
$this->httpClient = static::createClient();
$this->container = $this->httpClient->getContainer();
if (!$this->storageApiToken) {
$this->storageApiToken = TEST_ORCHESTRATOR_SAPI_TOKEN;
}
$this->setupStorageApi();
$message = 'Testing token description must contains same email as TEST_ERROR_NOTIFICATION_EMAIL_1';
if (!\Swift_Validate::email($this->token->getDescription())) {
throw new \Exception($message);
}
if ($this->token->getDescription() !== TEST_ERROR_NOTIFICATION_EMAIL_1) {
throw new \Exception($message);
}
$this->deleteTestEnvironments();
}
/**
* @param Entity\Orchestration $orchestration
* @param Token $token
* @return OrchestrationTask[]
* @throws \InvalidArgumentException
*/
public function findTasks(Entity\Orchestration $orchestration, Token $token)
{
$clientToken = new Token($this->storageApi);
if ($clientToken->getProjectId() !== $orchestration->getProjectId()) {
throw new \InvalidArgumentException('Given orchestration does not belong to requested tasks');
}
if ($clientToken->getProjectId() !== $token->getProjectId()) {
throw new \InvalidArgumentException('Given orchestration does not belong to requested tasks');
}
$data = $this->components->getConfiguration(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, $orchestration->getId());
if (empty($data['configuration']['tasks'])) {
return array();
}
/** @var OrchestrationTask[] $rows */
$rows = array_map(function ($line) use($token) {
$task = new OrchestrationTask($token);
$task->fromArray($line);
return $task;
}, $data['configuration']['tasks']);
return $this->sortTasks($rows);
}
public function jobsCount(array $criteria, ElasticsearchClient $client, array $config)
{
$exportOptions = $this->buildExportOptions($criteria);
$component = 'orchestrator';
$filter = [];
$filter[] = ['term' => ['project.id' => $this->token->getProjectId()]];
$query = ['match_all' => []];
if ($exportOptions != null) {
$query = ['query_string' => ['allow_leading_wildcard' => 'false', 'default_operator' => 'AND', 'query' => $exportOptions]];
}
$params = [];
$params['index'] = $config['index_prefix'] . '_syrup_*';
if (!is_null($component)) {
$params['index'] = $config['index_prefix'] . '_syrup_' . $component;
}
$params['body'] = ['query' => ['filtered' => ['filter' => ['bool' => ['must' => $filter]], 'query' => $query]]];
$count = $client->count($params);
return $count['count'];
}
public function postCreateTokenAction($orchestrationId)
{
$orchestration = $this->dbOrchestrationManager->findOrchestrationById($orchestrationId, $this->token);
if (!$orchestration) {
$exception = new OrchestratorException(404, sprintf('Orchestration %s not found', $orchestrationId));
$exception->setExceptionCode('ORCHESTRATION_NOT_FOUND');
throw $exception;
}
try {
$tokenId = $this->storageApi->createToken('manage', sprintf('Orchestrator %s', $orchestration->getName()), null, true);
// fill token data
$token = $this->storageApi->getToken($tokenId);
$token = new StorageApi\Token(new Client(array('token' => $token['token'], 'url' => $this->storageApi->getApiUrl())));
$orchestration->setToken((string) $token)->setTokenId($token->getId())->setTokenDesc($token->getDescription())->setTokenOwnerName($token->getOwnerName())->setProjectId($token->getProjectId());
} catch (\Exception $e) {
$exception = new OrchestratorException(500, sprintf('Could not create new token'), $e);
$exception->setExceptionCode('TOKEN_VALIDATION');
throw $exception;
}
$orchestration = $this->dbOrchestrationManager->updateOrchestration($orchestration);
$this->logger->info('Orchestration token changed');
$data = $orchestration->toApiArray();
return $this->createJsonResponse($data['token'], 201);
}
/**
* Create new orchestration with full params
*
* @param StorageApiToken $token
* @return int orchestration ID
*/
private function createOrchestrationTest(StorageApiToken $token)
{
$url = '/orchestrator/orchestrations';
$errorMessage = sprintf("Response for call '%s %s' should return new orchestration.", 'POST', $url);
$data = array('name' => self::TESTING_NAME, 'crontabRecord' => '*/5 * * * *', 'tokenId' => $token->getId(), 'active' => false, 'notifications' => array(array('email' => TEST_ERROR_NOTIFICATION_EMAIL_1, 'channel' => 'error', 'parameters' => array()), array('email' => TEST_ERROR_NOTIFICATION_EMAIL_1, 'channel' => 'warning', 'parameters' => array())));
$response = $this->callApiPost($url, $data);
$this->assertArrayHasKey('id', $response, $errorMessage);
$this->assertArrayHasKey('crontabRecord', $response, $errorMessage);
return $response['id'];
}
/**
* Create new orchestration with full params
*
* @param StorageApiToken $token
* @return int orchestration ID
*/
private function createOrchestrationTest(StorageApiToken $token)
{
$url = '/orchestrator/orchestrations';
$errorMessage = sprintf("Response for call '%s %s' should return new orchestration.", 'POST', $url);
$data = array('name' => self::TESTING_NAME, 'crontabRecord' => '*/1 * * * *', 'tokenId' => $token->getId(), 'active' => true);
$response = $this->callApiPost($url, $data);
$this->assertArrayHasKey('id', $response, $errorMessage);
$this->assertArrayHasKey('crontabRecord', $response, $errorMessage);
return $response['id'];
}
protected function execute(InputInterface $input, OutputInterface $output)
{
parent::execute($input, $output);
$this->dbOrchestrationManager = $this->getContainer()->get('orchestrator.doctrine.orchestration_manager');
$projects = array();
if ($input->getOption('projectId')) {
$projects[(int) $input->getOption('projectId')] = (int) $input->getOption('projectId');
} else {
foreach ($this->dbOrchestrationManager->findOrchestrations(array()) as $orchestration) {
$projects[$orchestration->getProjectId()] = (int) $orchestration->getProjectId();
}
}
foreach ($projects as $projectId) {
$output->writeln(sprintf('<info>[%s] Check start</info>', $projectId));
$filter = array('projectId' => $projectId);
foreach ($this->dbOrchestrationManager->findOrchestrations($filter) as $orchestration) {
try {
$jobClient = $this->createProjectSapi($orchestration);
$token = new StorageApi\Token($jobClient);
if ($token->hasEnabledFeature(StorageApi\Token::ORCHESTRATOR_NEW_CONFIGURATION)) {
$output->writeln("\t" . sprintf('<comment>[%s] (%s) %s - New configuration already enabled</comment>', $orchestration->getProjectId(), $orchestration->getId(), $orchestration->getName()));
continue;
}
} catch (DisabledException $e) {
$message = $e->getMessage();
if ($e->getPrevious()) {
$message = sprintf('%s: %s', $message, $e->getPrevious()->getMessage());
}
$output->writeln("\t" . sprintf('<error>[%s] (%s) %s - ' . $message . '</error>', $orchestration->getProjectId(), $orchestration->getId(), $orchestration->getName()));
continue;
}
$orchestration = $this->dbOrchestrationManager->findOrchestrationById($orchestration->getId(), $token, false);
// validate configuration
try {
$this->validateToken($jobClient);
if (!$input->getOption('onlyErrors')) {
$output->writeln("\t" . sprintf('<info>[%s] (%s) %s - Checked</info>', $orchestration->getProjectId(), $orchestration->getId(), $orchestration->getName()));
}
} catch (StorageApi\InvalidStateException $e) {
$output->writeln("\t" . sprintf('<error>[%s] (%s) %s - %s</error>', $orchestration->getProjectId(), $orchestration->getId(), $orchestration->getName(), $e->getMessage()));
}
}
$output->writeln(sprintf('<info>[%s] Check finished</info>', $projectId));
}
}
/**
* Create new orchestration with full params
*
* @param StorageApiToken $token
* @param StorageApi\Notification[] $notifications
* @return int orchestration ID
*/
private function createCrontabOrchestrationTest(StorageApiToken $token, $notifications = array())
{
$url = '/orchestrator/orchestrations';
$errorMessage = sprintf("Response for call '%s %s' should return new orchestration.", 'POST', $url);
$notifications = array_map(function ($row) {
/**
* @var StorageApi\Notification $row
*/
return $row->toApiArray();
}, $notifications);
$data = array('name' => self::TESTING_NAME, 'crontabRecord' => '*/1 * * * *', 'tokenId' => $token->getId(), 'active' => true, 'notifications' => $notifications);
$response = $this->callApiPost($url, $data);
$this->assertArrayHasKey('id', $response, $errorMessage);
$this->assertArrayHasKey('crontabRecord', $response, $errorMessage);
return $response['id'];
}
/**
* Change orchestration
*
* @param $id orchestration ID
* @param StorageApiToken $token
* @return int orchestration ID
*/
private function putOrchestrationBadTokenTest($id, StorageApiToken $token)
{
$url = '/orchestrator/orchestrations/' . $id;
$errorMessage = sprintf("Response for call '%s %s' should return error for new orchestration token.", 'PUT', $url);
$data = array('tokenId' => $token->getId(), 'active' => true);
$response = $this->callApiPut($url, $data, 400);
$this->assertArrayHasKey('code', $response, $errorMessage);
$this->assertArrayHasKey('message', $response, $errorMessage);
$this->assertArrayHasKey('status', $response, $errorMessage);
$this->assertEquals('ORCHESTRATION_VALIDATION', $response['code'], $errorMessage);
$this->assertEquals('warning', $response['status'], $errorMessage);
$this->assertContains('token', $response['message'], $errorMessage, true);
$this->assertContains('permission', $response['message'], $errorMessage, true);
return (int) $id;
}
protected function execute(InputInterface $input, OutputInterface $output)
{
parent::execute($input, $output);
$this->dbOrchestrationManager = $this->getContainer()->get('orchestrator.doctrine.orchestration_manager');
$skippedCount = 0;
$successCount = 0;
$errorCount = 0;
$force = $input->getOption('force');
$projectId = $input->getArgument('projectId');
$this->logger->info('configuration-migration.start ' . $projectId, array('test' => $force ? true : false));
foreach ($this->loadProjectOrchestrations($projectId) as $orchestration) {
try {
$jobClient = $this->createProjectSapi($orchestration);
$token = new StorageApi\Token($jobClient);
if ($token->hasEnabledFeature(StorageApi\Token::ORCHESTRATOR_NEW_CONFIGURATION)) {
throw new DisabledException('Migration skipped: new configuration already enabled');
}
} catch (DisabledException $e) {
$message = $e->getMessage();
if ($e->getPrevious()) {
$message = sprintf('%s: %s', $message, $e->getMessage());
}
$output->writeln(sprintf('<error>' . $message . '</error>'));
$this->logger->info('configuration-migration.failed ' . $projectId, array('test' => $force ? true : false, 'reason' => $message));
$output->writeln(sprintf('<error>Finished with errors</error>'));
return JobCommand::STATUS_ERROR;
}
$orchestration = $this->dbOrchestrationManager->findOrchestrationById($orchestration->getId(), $token, true);
$components = new Components($jobClient);
// remove old configuration
$oldConfiguration = null;
try {
$oldConfiguration = $components->getConfiguration(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, sprintf("%s-%s", KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, $orchestration->getId()));
} catch (ClientException $e) {
if ($e->getCode() !== 404) {
throw $e;
}
}
if ($force) {
// create or update configuration
$orchestrationTaskManager = new StorageApi\OrchestrationTaskManager($jobClient);
$orchestrationTaskManager->updateTasks($orchestration);
// validate configuration
try {
$this->validateConfiguration($orchestration, $jobClient);
$output->writeln(sprintf('<info>(%s) %s - Migrated</info>', $orchestration->getId(), $orchestration->getName()));
if ($oldConfiguration) {
$components->deleteConfiguration(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, $oldConfiguration['id']);
$output->writeln(sprintf('<comment>(%s) - Old configuration deleted</comment>', $oldConfiguration['id']));
}
$successCount++;
} catch (StorageApi\InvalidStateException $e) {
$output->writeln(sprintf('<error>' . $e->getMessage() . '</error>'));
$errorCount++;
}
} else {
$output->writeln(sprintf('<comment>(%s) %s - Will be migrated</comment>', $orchestration->getId(), $orchestration->getName()));
if ($oldConfiguration) {
$output->writeln(sprintf('<comment>(%s) - Old configuration will be deleted</comment>', $oldConfiguration['id']));
}
$skippedCount++;
}
}
if ($errorCount) {
$output->writeln(sprintf('<error>Finished with errors</error>'));
} else {
$output->writeln(sprintf('<info>Finished</info>'));
}
$this->logger->info('configuration-migration.migrated ' . $projectId, array('test' => $force ? true : false, 'successCount' => $successCount, 'errorCount' => $errorCount, 'skippedCount' => $skippedCount));
if ($errorCount) {
return JobCommand::STATUS_ERROR;
}
}