/** * @param Orchestration $orchestration * @return Client */ private function createProjectSapi(Orchestration $orchestration) { $client = new Client(array('token' => $orchestration->getToken(), 'url' => $this->getConfiguration()->getStorageApiUrl(), 'userAgent' => $this->appName)); try { $pingClient = new StorageApi\PingClient($client, $this->logger); if (!$pingClient->ping()) { throw new DisabledException('Project disabled'); } } catch (\Exception $e) { if ($e instanceof DisabledException) { throw $e; } throw new DisabledException('Check skipped', 0, $e); } return $client; }
/** * Process and validate form * * @return true if form is valid * @throws \Symfony\Component\HttpKernel\Exception\HttpException on invalid form */ public function process() { $this->form->submit($this->getPostJsonData()); // collection errors foreach ($this->form->get('tasks') as $key => $child) { if ($child->isValid()) { continue; } try { $this->throwFirstError($child); } catch (HttpException $e) { throw new HttpException(400, sprintf("[Task %s] %s", $key, $e->getMessage())); } } // retry task validation $data = $this->getPostJsonData(); if (array_key_exists('tasks', $data)) { if ($this->form->isValid()) { $orchestrationTasks = $this->orchestration->getTasks(); $requestTasks = $this->getTaskList(); if (count($requestTasks) !== count($orchestrationTasks)) { $this->form->addError(new FormError('Job specification must containts all orchestrations tasks')); } foreach ($requestTasks as $key => $requestTask) { if (!array_key_exists($key, $orchestrationTasks)) { $this->form->addError(new FormError('Job specification must containts all orchestrations tasks')); break; } $compareRequest = $requestTask->toCompareArray(); $compare = $orchestrationTasks[$key]->toCompareArray(); if (serialize($compareRequest) !== serialize($compare)) { $message = "Job task is different from orchestration task"; $this->logger->notice($message, array('taskPosition' => $key, 'old' => $compare, 'new' => $compareRequest)); $this->form->addError(new FormError(sprintf("[Task %s] %s", $key, $message))); break; } } } } // other form errors if (!$this->form->isValid()) { $this->throwFirstError($this->form); } $this->success = true; return true; }
/** * @param Orchestration $entity * @param \Keboola\StorageApi\Client $storageApi * @return Orchestration */ public function fillOrchestrationEntity(Orchestration $entity, Client $storageApi) { $oldTokenId = $entity->getTokenId(); $oldName = $entity->getName(); /** @var $entity Orchestration */ $entity = parent::fillEntity($entity); $name = $entity->getName(); if (empty($name)) { $entity->setName($oldName); } // token change if ($oldTokenId !== $entity->getTokenId()) { $token = $storageApi->getToken($entity->getTokenId()); $token = new Token(new Client(array('token' => $token['token'], 'url' => $storageApi->getApiUrl()))); $entity->setToken($token)->setTokenId($token->getId())->setTokenDesc($token->getDescription())->setTokenOwnerName($token->getOwnerName())->setProjectId($token->getProjectId()); } // notifications $data = $this->getPostJsonData(); if (!empty($data['notifications'])) { $data['notifications'] = array_map(function ($row) { $notification = new Notification(); $notification->fromArray($row); return $notification; }, $data['notifications']); $entity->setNotifications($data['notifications']); } // tasks $data = $this->getPostJsonData(); if (array_key_exists('tasks', $data)) { $entity->removeTasks(); $data['tasks'] = array_map(function ($row) { $notification = new OrchestrationTask(); $notification->fromArray($row); return $notification; }, $data['tasks']); foreach ($data['tasks'] as $task) { $entity->addTask($task); } } //@FIXME catch and throw exception return $entity; }
/** * @param Orchestration $orchestration * @param Client $storageApi * @return bool * @throws StorageApi\InvalidStateException */ private function validateConfiguration(Orchestration $orchestration, Client $storageApi) { $components = new Components($storageApi); $configuration = $components->getConfiguration(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, $orchestration->getId()); // same name if ($configuration['name'] !== $orchestration->getName()) { throw new StorageApi\InvalidStateException('Orchestration names are different'); } if (count($configuration['configuration']['tasks']) !== count($orchestration->getTasks())) { throw new StorageApi\InvalidStateException('Orchestration has same tasks count'); } $configurationTasks = $configuration['configuration']['tasks']; foreach ($orchestration->getTasks() as $i => $task) { $task = $task->toApiArray(); foreach ($task as $key => $value) { if ($configurationTasks[$i][$key] != $value) { throw new StorageApi\InvalidStateException(sprintf('Task %s is different: %s||%s', $task['id'], json_encode($task), $configurationTasks[$i])); } } } return true; }
/** * @param Elasticsearch\Job $job * @param Orchestration $orchestration * @param Elasticsearch\JobManager $jobManager * @param KbcComponentsList $components */ public function sendJobWarningMessage(Elasticsearch\Job $job, Orchestration $orchestration, Elasticsearch\JobManager $jobManager, KbcComponentsList $components) { /** * @var Notification[] $notifications */ $notifications = array_filter($orchestration->getNotifications(), function ($row) { /** * @var Notification $row */ if (!\Swift_Validate::email($row->getEmail())) { return false; } return $row->getChannel() === Job::STATUS_WARNING; }); $notificationsEmails = $jobManager->getWarningNotificationsEmails($job); if (!$notificationsEmails) { $notificationsEmails = array(); } // validating emails foreach ($notificationsEmails as $key => $notificationsEmail) { if (!\Swift_Validate::email($notificationsEmail)) { unset($notificationsEmails[$key]); } } if (!count($notifications) && !count($notificationsEmails)) { return; } $message = \Swift_Message::newInstance(); $message->setSubject(sprintf("[KBC] %s orchestrator %s warning", $job->getTokenOwnerName(), $job->getOrchestrationName())); $message->setFrom(self::MAIL_SENDER); foreach ($notificationsEmails as $notificationsEmail) { $message->addTo($notificationsEmail); } foreach ($notifications as $notification) { if ($job->getInitializedBy() !== 'manually') { if (!in_array($notification->getEmail(), $notificationsEmails)) { $message->addTo($notification->getEmail()); } } } $schedule = null; try { $cronSchedule = CronSchedule::fromCronString($orchestration->getCrontabRecord(), 'en'); $schedule = $cronSchedule->asNaturalLanguage(); } catch (\Exception $e) { } $tasks = $job->getResults(); if (empty($tasks->tasks)) { $tasks = array(); } else { $tasks = $tasks->tasks; } $jobUrl = $components->getJobUriTemplate(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME); $jobUrl = str_replace('&&projectId&&', $job->getProjectId(), $jobUrl); $jobUrl = str_replace('&&orchestrationId&&', $job->getOrchestrationId(), $jobUrl); $jobUrl = str_replace('&&jobId&&', $job->getId(), $jobUrl); $message->setBody($this->templating->render('KeboolaOrchestratorBundle:Email:jobWarning.email.html.twig', array('schedule' => $schedule, 'tasks' => $tasks, 'componentsIcons' => $components->getComponentsIcons(), 'componentsNames' => $components->getComponentsNames(), 'componentsTypes' => $this->filterComponentsTypes($components->getComponentsTypes()), 'job' => $job, 'jobUrl' => $jobUrl)), 'text/html'); $this->mailer->send($message); /** * @var \Swift_Spool $spool */ $spool = $this->mailer->getTransport()->getSpool(); $spool->flushQueue($this->mailerTransport); }
/** * @param Orchestration $orchestration * @param int $offset * @param int $limit * @return Metadata\Job[] */ private function loadOrchestrationSuccessJobs(Orchestration $orchestration, $offset = 0, $limit = 25) { /** * @var ComponentIndex $config */ $config = $this->getContainer()->get('syrup.elasticsearch.current_component_index'); $orchestrations = $this->loadActiveOrchestrationIds(); $filter = []; $filter[] = ['term' => ['status' => Metadata\Job::STATUS_SUCCESS]]; $filter[] = ['terms' => ['params.orchestration.id' => array($orchestration->getId())]]; $query = ['match_all' => []]; $params = []; $params['index'] = $config->getIndexPrefix() . '_syrup_' . KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME; $params['body'] = ['from' => (int) $offset, 'size' => (int) $limit, 'query' => ['filtered' => ['filter' => ['bool' => ['must' => $filter]], 'query' => $query]], 'sort' => ['id' => ['order' => 'desc']]]; $results = []; $hits = $this->elasticSearch->search($params); foreach ($hits['hits']['hits'] as $hit) { $res = $hit['_source']; $res['_index'] = $hit['_index']; $res['_type'] = $hit['_type']; $res['id'] = (int) $res['id']; $results[] = $res; } $objectEncryptor = $this->objectEncryptor; return array_map(function ($line) use($objectEncryptor) { return new Metadata\Job($objectEncryptor, $line, $line['_index'], $line['_type']); }, $results); }
public function beforeOrchestrationDelete(Entity\Orchestration $orchestration) { try { $this->components->deleteConfiguration(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, $orchestration->getId()); } catch (ClientException $e) { if ($e->getCode() !== 404) { throw $e; } } }
/** * Save tasks to configuration * * @param $id orchestration ID */ private function putOrchestrationTaskTest($id) { $url = '/orchestrator/orchestrations/' . $id . '/tasks'; $errorMessage = sprintf("Response for call '%s %s' should return two created tasks.", 'PUT', $url); $actionParams = array('fake' => 1); $data = array(array("some-extra-field" => "dummy value", "component" => "ex-google-drive", "action" => "run", "active" => true), array("some-extra-field" => "dummy value", "componentUrl" => "https://syrup.keboola.com/ex-google-drive/run", "actionParameters" => $actionParams, "active" => false)); $response = $this->callApiPut($url, $data); $this->assertCount(2, $response, $errorMessage); $this->assertArrayNotHasKey('some-extra-field', $response[0], $errorMessage); $this->assertArrayHasKey('actionParameters', $response[0], $errorMessage); $this->assertArrayHasKey('action', $response[0], $errorMessage); $this->assertArrayHasKey('component', $response[0], $errorMessage); $this->assertArrayNotHasKey('componentUrl', $response[0], $errorMessage); $this->assertEquals($data[0]['component'], $response[0]['component'], $errorMessage); $this->assertEquals($data[0]['action'], $response[0]['action'], $errorMessage); $this->assertArrayHasKey('actionParameters', $response[1], $errorMessage); $this->assertArrayNotHasKey('action', $response[1], $errorMessage); $this->assertArrayHasKey('componentUrl', $response[1], $errorMessage); $this->assertArrayNotHasKey('component', $response[1], $errorMessage); $this->assertEquals($actionParams, $response[1]['actionParameters'], $errorMessage); $this->assertEquals($data[1]['componentUrl'], $response[1]['componentUrl'], $errorMessage); $this->assertEquals($data[1]['active'], $response[1]['active'], $errorMessage); // Validation of tasks copy in sapi configuration $orchestration = new Orchestration(); $orchestration->fromArray(array('id' => $id)); $components = new Components($this->storageApi); $configuration = $components->getConfiguration(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, $orchestration->getId()); $this->assertArrayHasKey('configuration', $configuration); $this->assertArrayHasKey('tasks', $configuration['configuration']); $configurationTasks = $configuration['configuration']['tasks']; $this->assertCount(2, $configurationTasks); $this->assertArrayHasKey('actionParameters', $configurationTasks[0]); $this->assertArrayHasKey('action', $configurationTasks[0]); $this->assertArrayHasKey('componentUrl', $configurationTasks[0]); $this->assertArrayHasKey('component', $configurationTasks[0]); $this->assertEmpty($configurationTasks[0]['actionParameters']); $this->assertEquals($data[0]['action'], $configurationTasks[0]['action']); $this->assertEquals($data[0]['component'], $configurationTasks[0]['component']); $this->assertEquals($data[0]['active'], $configurationTasks[0]['active']); $this->assertEmpty($configurationTasks[0]['componentUrl']); $this->assertArrayHasKey('actionParameters', $configurationTasks[1]); $this->assertArrayHasKey('action', $configurationTasks[1]); $this->assertArrayHasKey('componentUrl', $configurationTasks[1]); $this->assertArrayHasKey('component', $configurationTasks[1]); $this->assertEquals($actionParams, $configurationTasks[1]['actionParameters']); $this->assertEquals($data[1]['componentUrl'], $configurationTasks[1]['componentUrl']); $this->assertEquals($data[1]['active'], $configurationTasks[1]['active']); $this->assertEmpty($configurationTasks[1]['action']); $this->assertEmpty($configurationTasks[1]['component']); }