public function testFeature() { $this->assertTrue(defined('TEST_ORCHESTRATOR_HAS_FEATURE_SAPI_TOKEN')); $this->assertNotEmpty(constant('TEST_ORCHESTRATOR_HAS_FEATURE_SAPI_TOKEN')); $this->assertTrue(defined('TEST_ORCHESTRATOR_FEATURE')); $this->assertNotEmpty(constant('TEST_ORCHESTRATOR_FEATURE')); $token = new Token(new StorageApiClient(array('token' => TEST_ORCHESTRATOR_HAS_FEATURE_SAPI_TOKEN, 'url' => TEST_ORCHESTRATOR_SAPI_URL))); $this->assertTrue($token->hasEnabledFeature(TEST_ORCHESTRATOR_FEATURE)); $this->assertFalse($token->hasEnabledFeature(md5(TEST_ORCHESTRATOR_FEATURE))); }
public function __construct(StorageApiClient $storageApi, Rds\OrchestrationManager $orchestrationManager, StorageApi\Token $token) { $this->storageApi = $storageApi; $this->orchestrationManager = $orchestrationManager; $this->token = $token; // token with token manipulation permission if (!$this->token->canManageTokens()) { throw new AccessDeniedHttpException('Token has no permissions for token manipulation'); } }
/** * Cancel waiting job * * @param $jobId * @throws \Keboola\OrchestratorBundle\Exception\OrchestratorException * @return Response */ public function deleteAction($jobId) { $job = $this->jobEsManager->findJobById($jobId); if (!$job) { $exception = new OrchestratorException(404, sprintf('Job %s not found', $jobId)); $exception->setExceptionCode('JOB_NOT_FOUND'); throw $exception; } if (!$job->isWaiting()) { $exception = new OrchestratorException(403, 'Only waiting job can be cancelled'); $exception->setExceptionCode('INVALID_JOB_STATUS'); throw $exception; } $client = new GuzzleHttp\Client([]); try { $response = $client->post(sprintf($this->killUrl, $job->getId()), array('config' => array('curl' => array(CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_0)), 'headers' => array('X-StorageApi-Token' => $this->token, 'X-KBC-RunId' => $this->storageApi->getRunId(), 'X-User-Agent', KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME . " - API cancel"), 'timeout' => 60)); if ($response->getStatusCode() == 202) { $this->logger->info(sprintf('Orchestration job %s cancelled', $job->getId())); $this->logger->debug(sprintf('Orchestration job %s cancelled - old api', $job->getId()), array('orchestrationId' => $job->getOrchestrationId(), 'orchestration' => $job->getOrchestrationName(), 'projectId' => $job->getProjectId(), 'project' => $job->getTokenOwnerName(), 'tokenId' => $this->token->getId(), 'token' => $this->token->getDescription())); } else { throw new ApplicationException('Job cancel via syrup api failed', null, array('statusCode' => $response->getStatusCode(), 'body' => ResponseDecoder::decode($response))); } } catch (GuzzleHttp\Exception\RequestException $e) { throw new ApplicationException('Job cancel via syrup api failed', $e); } $jsonResponse = $this->createJsonResponse(array(), 204); return $jsonResponse; }
/** * Prepare environment for test */ protected function setUp() { $this->httpClient = static::createClient(); $this->container = $this->httpClient->getContainer(); if (!$this->storageApiToken) { $this->storageApiToken = TEST_ORCHESTRATOR_SAPI_TOKEN; } $this->setupStorageApi(); $message = 'Testing token description must contains same email as TEST_ERROR_NOTIFICATION_EMAIL_1'; if (!\Swift_Validate::email($this->token->getDescription())) { throw new \Exception($message); } if ($this->token->getDescription() !== TEST_ERROR_NOTIFICATION_EMAIL_1) { throw new \Exception($message); } $this->deleteTestEnvironments(); }
/** * @param Entity\Orchestration $orchestration * @param Token $token * @return OrchestrationTask[] * @throws \InvalidArgumentException */ public function findTasks(Entity\Orchestration $orchestration, Token $token) { $clientToken = new Token($this->storageApi); if ($clientToken->getProjectId() !== $orchestration->getProjectId()) { throw new \InvalidArgumentException('Given orchestration does not belong to requested tasks'); } if ($clientToken->getProjectId() !== $token->getProjectId()) { throw new \InvalidArgumentException('Given orchestration does not belong to requested tasks'); } $data = $this->components->getConfiguration(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, $orchestration->getId()); if (empty($data['configuration']['tasks'])) { return array(); } /** @var OrchestrationTask[] $rows */ $rows = array_map(function ($line) use($token) { $task = new OrchestrationTask($token); $task->fromArray($line); return $task; }, $data['configuration']['tasks']); return $this->sortTasks($rows); }
public function jobsCount(array $criteria, ElasticsearchClient $client, array $config) { $exportOptions = $this->buildExportOptions($criteria); $component = 'orchestrator'; $filter = []; $filter[] = ['term' => ['project.id' => $this->token->getProjectId()]]; $query = ['match_all' => []]; if ($exportOptions != null) { $query = ['query_string' => ['allow_leading_wildcard' => 'false', 'default_operator' => 'AND', 'query' => $exportOptions]]; } $params = []; $params['index'] = $config['index_prefix'] . '_syrup_*'; if (!is_null($component)) { $params['index'] = $config['index_prefix'] . '_syrup_' . $component; } $params['body'] = ['query' => ['filtered' => ['filter' => ['bool' => ['must' => $filter]], 'query' => $query]]]; $count = $client->count($params); return $count['count']; }
public function postCreateTokenAction($orchestrationId) { $orchestration = $this->dbOrchestrationManager->findOrchestrationById($orchestrationId, $this->token); if (!$orchestration) { $exception = new OrchestratorException(404, sprintf('Orchestration %s not found', $orchestrationId)); $exception->setExceptionCode('ORCHESTRATION_NOT_FOUND'); throw $exception; } try { $tokenId = $this->storageApi->createToken('manage', sprintf('Orchestrator %s', $orchestration->getName()), null, true); // fill token data $token = $this->storageApi->getToken($tokenId); $token = new StorageApi\Token(new Client(array('token' => $token['token'], 'url' => $this->storageApi->getApiUrl()))); $orchestration->setToken((string) $token)->setTokenId($token->getId())->setTokenDesc($token->getDescription())->setTokenOwnerName($token->getOwnerName())->setProjectId($token->getProjectId()); } catch (\Exception $e) { $exception = new OrchestratorException(500, sprintf('Could not create new token'), $e); $exception->setExceptionCode('TOKEN_VALIDATION'); throw $exception; } $orchestration = $this->dbOrchestrationManager->updateOrchestration($orchestration); $this->logger->info('Orchestration token changed'); $data = $orchestration->toApiArray(); return $this->createJsonResponse($data['token'], 201); }
/** * Create new orchestration with full params * * @param StorageApiToken $token * @return int orchestration ID */ private function createOrchestrationTest(StorageApiToken $token) { $url = '/orchestrator/orchestrations'; $errorMessage = sprintf("Response for call '%s %s' should return new orchestration.", 'POST', $url); $data = array('name' => self::TESTING_NAME, 'crontabRecord' => '*/5 * * * *', 'tokenId' => $token->getId(), 'active' => false, 'notifications' => array(array('email' => TEST_ERROR_NOTIFICATION_EMAIL_1, 'channel' => 'error', 'parameters' => array()), array('email' => TEST_ERROR_NOTIFICATION_EMAIL_1, 'channel' => 'warning', 'parameters' => array()))); $response = $this->callApiPost($url, $data); $this->assertArrayHasKey('id', $response, $errorMessage); $this->assertArrayHasKey('crontabRecord', $response, $errorMessage); return $response['id']; }
/** * Create new orchestration with full params * * @param StorageApiToken $token * @return int orchestration ID */ private function createOrchestrationTest(StorageApiToken $token) { $url = '/orchestrator/orchestrations'; $errorMessage = sprintf("Response for call '%s %s' should return new orchestration.", 'POST', $url); $data = array('name' => self::TESTING_NAME, 'crontabRecord' => '*/1 * * * *', 'tokenId' => $token->getId(), 'active' => true); $response = $this->callApiPost($url, $data); $this->assertArrayHasKey('id', $response, $errorMessage); $this->assertArrayHasKey('crontabRecord', $response, $errorMessage); return $response['id']; }
protected function execute(InputInterface $input, OutputInterface $output) { parent::execute($input, $output); $this->dbOrchestrationManager = $this->getContainer()->get('orchestrator.doctrine.orchestration_manager'); $projects = array(); if ($input->getOption('projectId')) { $projects[(int) $input->getOption('projectId')] = (int) $input->getOption('projectId'); } else { foreach ($this->dbOrchestrationManager->findOrchestrations(array()) as $orchestration) { $projects[$orchestration->getProjectId()] = (int) $orchestration->getProjectId(); } } foreach ($projects as $projectId) { $output->writeln(sprintf('<info>[%s] Check start</info>', $projectId)); $filter = array('projectId' => $projectId); foreach ($this->dbOrchestrationManager->findOrchestrations($filter) as $orchestration) { try { $jobClient = $this->createProjectSapi($orchestration); $token = new StorageApi\Token($jobClient); if ($token->hasEnabledFeature(StorageApi\Token::ORCHESTRATOR_NEW_CONFIGURATION)) { $output->writeln("\t" . sprintf('<comment>[%s] (%s) %s - New configuration already enabled</comment>', $orchestration->getProjectId(), $orchestration->getId(), $orchestration->getName())); continue; } } catch (DisabledException $e) { $message = $e->getMessage(); if ($e->getPrevious()) { $message = sprintf('%s: %s', $message, $e->getPrevious()->getMessage()); } $output->writeln("\t" . sprintf('<error>[%s] (%s) %s - ' . $message . '</error>', $orchestration->getProjectId(), $orchestration->getId(), $orchestration->getName())); continue; } $orchestration = $this->dbOrchestrationManager->findOrchestrationById($orchestration->getId(), $token, false); // validate configuration try { $this->validateToken($jobClient); if (!$input->getOption('onlyErrors')) { $output->writeln("\t" . sprintf('<info>[%s] (%s) %s - Checked</info>', $orchestration->getProjectId(), $orchestration->getId(), $orchestration->getName())); } } catch (StorageApi\InvalidStateException $e) { $output->writeln("\t" . sprintf('<error>[%s] (%s) %s - %s</error>', $orchestration->getProjectId(), $orchestration->getId(), $orchestration->getName(), $e->getMessage())); } } $output->writeln(sprintf('<info>[%s] Check finished</info>', $projectId)); } }
/** * Create new orchestration with full params * * @param StorageApiToken $token * @param StorageApi\Notification[] $notifications * @return int orchestration ID */ private function createCrontabOrchestrationTest(StorageApiToken $token, $notifications = array()) { $url = '/orchestrator/orchestrations'; $errorMessage = sprintf("Response for call '%s %s' should return new orchestration.", 'POST', $url); $notifications = array_map(function ($row) { /** * @var StorageApi\Notification $row */ return $row->toApiArray(); }, $notifications); $data = array('name' => self::TESTING_NAME, 'crontabRecord' => '*/1 * * * *', 'tokenId' => $token->getId(), 'active' => true, 'notifications' => $notifications); $response = $this->callApiPost($url, $data); $this->assertArrayHasKey('id', $response, $errorMessage); $this->assertArrayHasKey('crontabRecord', $response, $errorMessage); return $response['id']; }
/** * Change orchestration * * @param $id orchestration ID * @param StorageApiToken $token * @return int orchestration ID */ private function putOrchestrationBadTokenTest($id, StorageApiToken $token) { $url = '/orchestrator/orchestrations/' . $id; $errorMessage = sprintf("Response for call '%s %s' should return error for new orchestration token.", 'PUT', $url); $data = array('tokenId' => $token->getId(), 'active' => true); $response = $this->callApiPut($url, $data, 400); $this->assertArrayHasKey('code', $response, $errorMessage); $this->assertArrayHasKey('message', $response, $errorMessage); $this->assertArrayHasKey('status', $response, $errorMessage); $this->assertEquals('ORCHESTRATION_VALIDATION', $response['code'], $errorMessage); $this->assertEquals('warning', $response['status'], $errorMessage); $this->assertContains('token', $response['message'], $errorMessage, true); $this->assertContains('permission', $response['message'], $errorMessage, true); return (int) $id; }
protected function execute(InputInterface $input, OutputInterface $output) { parent::execute($input, $output); $this->dbOrchestrationManager = $this->getContainer()->get('orchestrator.doctrine.orchestration_manager'); $skippedCount = 0; $successCount = 0; $errorCount = 0; $force = $input->getOption('force'); $projectId = $input->getArgument('projectId'); $this->logger->info('configuration-migration.start ' . $projectId, array('test' => $force ? true : false)); foreach ($this->loadProjectOrchestrations($projectId) as $orchestration) { try { $jobClient = $this->createProjectSapi($orchestration); $token = new StorageApi\Token($jobClient); if ($token->hasEnabledFeature(StorageApi\Token::ORCHESTRATOR_NEW_CONFIGURATION)) { throw new DisabledException('Migration skipped: new configuration already enabled'); } } catch (DisabledException $e) { $message = $e->getMessage(); if ($e->getPrevious()) { $message = sprintf('%s: %s', $message, $e->getMessage()); } $output->writeln(sprintf('<error>' . $message . '</error>')); $this->logger->info('configuration-migration.failed ' . $projectId, array('test' => $force ? true : false, 'reason' => $message)); $output->writeln(sprintf('<error>Finished with errors</error>')); return JobCommand::STATUS_ERROR; } $orchestration = $this->dbOrchestrationManager->findOrchestrationById($orchestration->getId(), $token, true); $components = new Components($jobClient); // remove old configuration $oldConfiguration = null; try { $oldConfiguration = $components->getConfiguration(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, sprintf("%s-%s", KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, $orchestration->getId())); } catch (ClientException $e) { if ($e->getCode() !== 404) { throw $e; } } if ($force) { // create or update configuration $orchestrationTaskManager = new StorageApi\OrchestrationTaskManager($jobClient); $orchestrationTaskManager->updateTasks($orchestration); // validate configuration try { $this->validateConfiguration($orchestration, $jobClient); $output->writeln(sprintf('<info>(%s) %s - Migrated</info>', $orchestration->getId(), $orchestration->getName())); if ($oldConfiguration) { $components->deleteConfiguration(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, $oldConfiguration['id']); $output->writeln(sprintf('<comment>(%s) - Old configuration deleted</comment>', $oldConfiguration['id'])); } $successCount++; } catch (StorageApi\InvalidStateException $e) { $output->writeln(sprintf('<error>' . $e->getMessage() . '</error>')); $errorCount++; } } else { $output->writeln(sprintf('<comment>(%s) %s - Will be migrated</comment>', $orchestration->getId(), $orchestration->getName())); if ($oldConfiguration) { $output->writeln(sprintf('<comment>(%s) - Old configuration will be deleted</comment>', $oldConfiguration['id'])); } $skippedCount++; } } if ($errorCount) { $output->writeln(sprintf('<error>Finished with errors</error>')); } else { $output->writeln(sprintf('<info>Finished</info>')); } $this->logger->info('configuration-migration.migrated ' . $projectId, array('test' => $force ? true : false, 'successCount' => $successCount, 'errorCount' => $errorCount, 'skippedCount' => $skippedCount)); if ($errorCount) { return JobCommand::STATUS_ERROR; } }