/**
* @param Orchestration $orchestration
* @return Client
*/
private function createProjectSapi(Orchestration $orchestration)
{
$client = new Client(array('token' => $orchestration->getToken(), 'url' => $this->getConfiguration()->getStorageApiUrl(), 'userAgent' => $this->appName));
try {
$pingClient = new StorageApi\PingClient($client, $this->logger);
if (!$pingClient->ping()) {
throw new DisabledException('Project disabled');
}
} catch (\Exception $e) {
if ($e instanceof DisabledException) {
throw $e;
}
throw new DisabledException('Check skipped', 0, $e);
}
return $client;
}
/**
* Process and validate form
*
* @return true if form is valid
* @throws \Symfony\Component\HttpKernel\Exception\HttpException on invalid form
*/
public function process()
{
$this->form->submit($this->getPostJsonData());
// collection errors
foreach ($this->form->get('tasks') as $key => $child) {
if ($child->isValid()) {
continue;
}
try {
$this->throwFirstError($child);
} catch (HttpException $e) {
throw new HttpException(400, sprintf("[Task %s] %s", $key, $e->getMessage()));
}
}
// retry task validation
$data = $this->getPostJsonData();
if (array_key_exists('tasks', $data)) {
if ($this->form->isValid()) {
$orchestrationTasks = $this->orchestration->getTasks();
$requestTasks = $this->getTaskList();
if (count($requestTasks) !== count($orchestrationTasks)) {
$this->form->addError(new FormError('Job specification must containts all orchestrations tasks'));
}
foreach ($requestTasks as $key => $requestTask) {
if (!array_key_exists($key, $orchestrationTasks)) {
$this->form->addError(new FormError('Job specification must containts all orchestrations tasks'));
break;
}
$compareRequest = $requestTask->toCompareArray();
$compare = $orchestrationTasks[$key]->toCompareArray();
if (serialize($compareRequest) !== serialize($compare)) {
$message = "Job task is different from orchestration task";
$this->logger->notice($message, array('taskPosition' => $key, 'old' => $compare, 'new' => $compareRequest));
$this->form->addError(new FormError(sprintf("[Task %s] %s", $key, $message)));
break;
}
}
}
}
// other form errors
if (!$this->form->isValid()) {
$this->throwFirstError($this->form);
}
$this->success = true;
return true;
}
/**
* @param Orchestration $entity
* @param \Keboola\StorageApi\Client $storageApi
* @return Orchestration
*/
public function fillOrchestrationEntity(Orchestration $entity, Client $storageApi)
{
$oldTokenId = $entity->getTokenId();
$oldName = $entity->getName();
/** @var $entity Orchestration */
$entity = parent::fillEntity($entity);
$name = $entity->getName();
if (empty($name)) {
$entity->setName($oldName);
}
// token change
if ($oldTokenId !== $entity->getTokenId()) {
$token = $storageApi->getToken($entity->getTokenId());
$token = new Token(new Client(array('token' => $token['token'], 'url' => $storageApi->getApiUrl())));
$entity->setToken($token)->setTokenId($token->getId())->setTokenDesc($token->getDescription())->setTokenOwnerName($token->getOwnerName())->setProjectId($token->getProjectId());
}
// notifications
$data = $this->getPostJsonData();
if (!empty($data['notifications'])) {
$data['notifications'] = array_map(function ($row) {
$notification = new Notification();
$notification->fromArray($row);
return $notification;
}, $data['notifications']);
$entity->setNotifications($data['notifications']);
}
// tasks
$data = $this->getPostJsonData();
if (array_key_exists('tasks', $data)) {
$entity->removeTasks();
$data['tasks'] = array_map(function ($row) {
$notification = new OrchestrationTask();
$notification->fromArray($row);
return $notification;
}, $data['tasks']);
foreach ($data['tasks'] as $task) {
$entity->addTask($task);
}
}
//@FIXME catch and throw exception
return $entity;
}
/**
* @param Orchestration $orchestration
* @param Client $storageApi
* @return bool
* @throws StorageApi\InvalidStateException
*/
private function validateConfiguration(Orchestration $orchestration, Client $storageApi)
{
$components = new Components($storageApi);
$configuration = $components->getConfiguration(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, $orchestration->getId());
// same name
if ($configuration['name'] !== $orchestration->getName()) {
throw new StorageApi\InvalidStateException('Orchestration names are different');
}
if (count($configuration['configuration']['tasks']) !== count($orchestration->getTasks())) {
throw new StorageApi\InvalidStateException('Orchestration has same tasks count');
}
$configurationTasks = $configuration['configuration']['tasks'];
foreach ($orchestration->getTasks() as $i => $task) {
$task = $task->toApiArray();
foreach ($task as $key => $value) {
if ($configurationTasks[$i][$key] != $value) {
throw new StorageApi\InvalidStateException(sprintf('Task %s is different: %s||%s', $task['id'], json_encode($task), $configurationTasks[$i]));
}
}
}
return true;
}
/**
* @param Elasticsearch\Job $job
* @param Orchestration $orchestration
* @param Elasticsearch\JobManager $jobManager
* @param KbcComponentsList $components
*/
public function sendJobWarningMessage(Elasticsearch\Job $job, Orchestration $orchestration, Elasticsearch\JobManager $jobManager, KbcComponentsList $components)
{
/**
* @var Notification[] $notifications
*/
$notifications = array_filter($orchestration->getNotifications(), function ($row) {
/**
* @var Notification $row
*/
if (!\Swift_Validate::email($row->getEmail())) {
return false;
}
return $row->getChannel() === Job::STATUS_WARNING;
});
$notificationsEmails = $jobManager->getWarningNotificationsEmails($job);
if (!$notificationsEmails) {
$notificationsEmails = array();
}
// validating emails
foreach ($notificationsEmails as $key => $notificationsEmail) {
if (!\Swift_Validate::email($notificationsEmail)) {
unset($notificationsEmails[$key]);
}
}
if (!count($notifications) && !count($notificationsEmails)) {
return;
}
$message = \Swift_Message::newInstance();
$message->setSubject(sprintf("[KBC] %s orchestrator %s warning", $job->getTokenOwnerName(), $job->getOrchestrationName()));
$message->setFrom(self::MAIL_SENDER);
foreach ($notificationsEmails as $notificationsEmail) {
$message->addTo($notificationsEmail);
}
foreach ($notifications as $notification) {
if ($job->getInitializedBy() !== 'manually') {
if (!in_array($notification->getEmail(), $notificationsEmails)) {
$message->addTo($notification->getEmail());
}
}
}
$schedule = null;
try {
$cronSchedule = CronSchedule::fromCronString($orchestration->getCrontabRecord(), 'en');
$schedule = $cronSchedule->asNaturalLanguage();
} catch (\Exception $e) {
}
$tasks = $job->getResults();
if (empty($tasks->tasks)) {
$tasks = array();
} else {
$tasks = $tasks->tasks;
}
$jobUrl = $components->getJobUriTemplate(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME);
$jobUrl = str_replace('&&projectId&&', $job->getProjectId(), $jobUrl);
$jobUrl = str_replace('&&orchestrationId&&', $job->getOrchestrationId(), $jobUrl);
$jobUrl = str_replace('&&jobId&&', $job->getId(), $jobUrl);
$message->setBody($this->templating->render('KeboolaOrchestratorBundle:Email:jobWarning.email.html.twig', array('schedule' => $schedule, 'tasks' => $tasks, 'componentsIcons' => $components->getComponentsIcons(), 'componentsNames' => $components->getComponentsNames(), 'componentsTypes' => $this->filterComponentsTypes($components->getComponentsTypes()), 'job' => $job, 'jobUrl' => $jobUrl)), 'text/html');
$this->mailer->send($message);
/**
* @var \Swift_Spool $spool
*/
$spool = $this->mailer->getTransport()->getSpool();
$spool->flushQueue($this->mailerTransport);
}
/**
* @param Orchestration $orchestration
* @param int $offset
* @param int $limit
* @return Metadata\Job[]
*/
private function loadOrchestrationSuccessJobs(Orchestration $orchestration, $offset = 0, $limit = 25)
{
/**
* @var ComponentIndex $config
*/
$config = $this->getContainer()->get('syrup.elasticsearch.current_component_index');
$orchestrations = $this->loadActiveOrchestrationIds();
$filter = [];
$filter[] = ['term' => ['status' => Metadata\Job::STATUS_SUCCESS]];
$filter[] = ['terms' => ['params.orchestration.id' => array($orchestration->getId())]];
$query = ['match_all' => []];
$params = [];
$params['index'] = $config->getIndexPrefix() . '_syrup_' . KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME;
$params['body'] = ['from' => (int) $offset, 'size' => (int) $limit, 'query' => ['filtered' => ['filter' => ['bool' => ['must' => $filter]], 'query' => $query]], 'sort' => ['id' => ['order' => 'desc']]];
$results = [];
$hits = $this->elasticSearch->search($params);
foreach ($hits['hits']['hits'] as $hit) {
$res = $hit['_source'];
$res['_index'] = $hit['_index'];
$res['_type'] = $hit['_type'];
$res['id'] = (int) $res['id'];
$results[] = $res;
}
$objectEncryptor = $this->objectEncryptor;
return array_map(function ($line) use($objectEncryptor) {
return new Metadata\Job($objectEncryptor, $line, $line['_index'], $line['_type']);
}, $results);
}
public function beforeOrchestrationDelete(Entity\Orchestration $orchestration)
{
try {
$this->components->deleteConfiguration(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, $orchestration->getId());
} catch (ClientException $e) {
if ($e->getCode() !== 404) {
throw $e;
}
}
}
/**
* Save tasks to configuration
*
* @param $id orchestration ID
*/
private function putOrchestrationTaskTest($id)
{
$url = '/orchestrator/orchestrations/' . $id . '/tasks';
$errorMessage = sprintf("Response for call '%s %s' should return two created tasks.", 'PUT', $url);
$actionParams = array('fake' => 1);
$data = array(array("some-extra-field" => "dummy value", "component" => "ex-google-drive", "action" => "run", "active" => true), array("some-extra-field" => "dummy value", "componentUrl" => "https://syrup.keboola.com/ex-google-drive/run", "actionParameters" => $actionParams, "active" => false));
$response = $this->callApiPut($url, $data);
$this->assertCount(2, $response, $errorMessage);
$this->assertArrayNotHasKey('some-extra-field', $response[0], $errorMessage);
$this->assertArrayHasKey('actionParameters', $response[0], $errorMessage);
$this->assertArrayHasKey('action', $response[0], $errorMessage);
$this->assertArrayHasKey('component', $response[0], $errorMessage);
$this->assertArrayNotHasKey('componentUrl', $response[0], $errorMessage);
$this->assertEquals($data[0]['component'], $response[0]['component'], $errorMessage);
$this->assertEquals($data[0]['action'], $response[0]['action'], $errorMessage);
$this->assertArrayHasKey('actionParameters', $response[1], $errorMessage);
$this->assertArrayNotHasKey('action', $response[1], $errorMessage);
$this->assertArrayHasKey('componentUrl', $response[1], $errorMessage);
$this->assertArrayNotHasKey('component', $response[1], $errorMessage);
$this->assertEquals($actionParams, $response[1]['actionParameters'], $errorMessage);
$this->assertEquals($data[1]['componentUrl'], $response[1]['componentUrl'], $errorMessage);
$this->assertEquals($data[1]['active'], $response[1]['active'], $errorMessage);
// Validation of tasks copy in sapi configuration
$orchestration = new Orchestration();
$orchestration->fromArray(array('id' => $id));
$components = new Components($this->storageApi);
$configuration = $components->getConfiguration(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, $orchestration->getId());
$this->assertArrayHasKey('configuration', $configuration);
$this->assertArrayHasKey('tasks', $configuration['configuration']);
$configurationTasks = $configuration['configuration']['tasks'];
$this->assertCount(2, $configurationTasks);
$this->assertArrayHasKey('actionParameters', $configurationTasks[0]);
$this->assertArrayHasKey('action', $configurationTasks[0]);
$this->assertArrayHasKey('componentUrl', $configurationTasks[0]);
$this->assertArrayHasKey('component', $configurationTasks[0]);
$this->assertEmpty($configurationTasks[0]['actionParameters']);
$this->assertEquals($data[0]['action'], $configurationTasks[0]['action']);
$this->assertEquals($data[0]['component'], $configurationTasks[0]['component']);
$this->assertEquals($data[0]['active'], $configurationTasks[0]['active']);
$this->assertEmpty($configurationTasks[0]['componentUrl']);
$this->assertArrayHasKey('actionParameters', $configurationTasks[1]);
$this->assertArrayHasKey('action', $configurationTasks[1]);
$this->assertArrayHasKey('componentUrl', $configurationTasks[1]);
$this->assertArrayHasKey('component', $configurationTasks[1]);
$this->assertEquals($actionParams, $configurationTasks[1]['actionParameters']);
$this->assertEquals($data[1]['componentUrl'], $configurationTasks[1]['componentUrl']);
$this->assertEquals($data[1]['active'], $configurationTasks[1]['active']);
$this->assertEmpty($configurationTasks[1]['action']);
$this->assertEmpty($configurationTasks[1]['component']);
}
