/** * @return null|string */ public function getError() { if ($this->getStatus() === StorageApiEvent::TYPE_SUCCESS || $this->getStatus() === 'processing' || $this->getStatus() === null) { return null; } return $this->event->getDescription() ? $this->event->getDescription() : 'Task error'; }
public function handle(array $record) { $this->initStorageApiClient(); if (!$this->storageApiClient || $record['level'] == Logger::DEBUG) { return false; } $event = new Event(); if (!empty($record['component'])) { $event->setComponent($record['component']); } else { $event->setComponent($this->appName); } $event->setMessage(\Keboola\Utils\sanitizeUtf8($record['message'])); $event->setRunId($this->storageApiClient->getRunId()); $params = []; if (isset($record['http'])) { $params['http'] = $record['http']; } $event->setParams($params); $results = []; if (isset($record['context']['exceptionId'])) { $results['exceptionId'] = $record['context']['exceptionId']; } if (isset($record['context']['job'])) { $results['job'] = $record['context']['job']; } $event->setResults($results); switch ($record['level']) { case Logger::ERROR: $type = Event::TYPE_ERROR; break; case Logger::CRITICAL: case Logger::EMERGENCY: case Logger::ALERT: $type = Event::TYPE_ERROR; $event->setMessage("Application error"); $event->setDescription("Contact support@keboola.com"); break; case Logger::WARNING: case Logger::NOTICE: $type = Event::TYPE_WARN; break; case Logger::INFO: default: $type = Event::TYPE_INFO; break; } $event->setType($type); $this->storageApiClient->createEvent($event); return false; }
public function testLargeEvents() { $client = new StorageApiClient(array('token' => TEST_ORCHESTRATOR_SAPI_TOKEN, 'url' => TEST_ORCHESTRATOR_SAPI_URL)); $testLimit = 300 * 1024; $testMessage = 'This is sample text for large evenet testing.'; $event = new Event(); $event->setComponent(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME)->setMessage('Test - Large Event')->setResults(array('Message' => str_repeat($testMessage, (int) ($testLimit / mb_strlen($testMessage))))); $sizeError = false; try { $client->createEvent($event); } catch (ClientException $e) { $this->assertEquals(EventLogger::STATUS_CODE_REQUEST_TOO_LARGE, $e->getStringCode()); $sizeError = true; $eventId = EventLogger::create($event, $client); $this->assertNotEmpty($eventId); } $this->assertTrue($sizeError); }
protected function sendEventToSapi($type, $message, $componentName) { $sapiEvent = new SapiEvent(); $sapiEvent->setComponent($componentName); $sapiEvent->setMessage($message); $sapiEvent->setRunId($this->storageApi->getRunId()); $sapiEvent->setType($type); $this->storageApi->createEvent($sapiEvent); }
protected function execute(InputInterface $input, OutputInterface $output) { parent::execute($input, $output); $this->logger->info('scheduler.start', array()); $output->writeln('Current time: ' . $this->getStartTime()->format('Y-m-d H:i:s')); $this->lock->setLockName('scheduler-' . $this->getStartTime()->format('Y-m-d-H-i')); // skip locked try { if (!$this->lock->lock()) { $this->logger->info('scheduler.skipped', array('lockName' => $this->lock->getLockName())); return; } } catch (\Exception $e) { $this->logger->info('scheduler.error', array('message' => $e->getMessage(), 'lockName' => $this->lock->getLockName())); return; } $this->logger->info('scheduler.locked', array('lockName' => $this->lock->getLockName())); /** * @var Elasticsearch\JobManagerFactory $jobManagerFactory */ $jobManagerFactory = $this->getContainer()->get('orchestrator.job_manager.factory'); $this->queue = $this->getContainer()->get('syrup.queue_factory')->get(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME); $this->dbOrchestrationManager = $this->getContainer()->get('orchestrator.doctrine.orchestration_manager'); $newJobs = array(); $orchestrations = $this->dbOrchestrationManager->findOrchestrations(array()); foreach ($orchestrations as $orchestration) { // skip deleted and non-active orchestrations if ($orchestration->getDeleted() || !$orchestration->getActive() || !$orchestration->getCrontabRecord()) { continue; } $cronExpression = CronExpression::factory($orchestration->getCrontabRecord()); if (!$cronExpression->isDue($this->getStartTime())) { continue; } $jobClient = new Client(array('token' => $orchestration->getToken(), 'url' => $this->getConfiguration()->getStorageApiUrl(), 'userAgent' => $this->appName)); try { $pingClient = new StorageApi\PingClient($jobClient, $this->logger); if (!$pingClient->ping()) { $this->logger->info('scheduler.orchestration.skipped.disabled', array('lockName' => $this->lock->getLockName(), 'orchestrationId' => $orchestration->getId(), 'orchestrationName' => $orchestration->getName(), 'projectId' => $orchestration->getProjectId(), 'projectName' => $orchestration->getTokenOwnerName())); continue; } } catch (\Exception $e) { $this->logger->info('scheduler.orchestration.skipped.error', array('lockName' => $this->lock->getLockName(), 'message' => $e->getMessage(), 'orchestrationId' => $orchestration->getId(), 'orchestrationName' => $orchestration->getName(), 'projectId' => $orchestration->getProjectId(), 'projectName' => $orchestration->getTokenOwnerName())); continue; } $token = new StorageApi\Token($jobClient); $jobEsManager = $jobManagerFactory->createJobManager($token); $jobsStats = $jobEsManager->getWaitingStats(); // skip waiting if (array_key_exists($orchestration->getId(), $jobsStats) && $jobsStats[$orchestration->getId()] >= AppConfiguration::MAX_WAITING_JOBS_COUNT) { $this->logger->info('scheduler.orchestration.skipped', array('lockName' => $this->lock->getLockName(), 'waitingCount' => $jobsStats[$orchestration->getId()], 'limit' => AppConfiguration::MAX_WAITING_JOBS_COUNT, 'orchestrationId' => $orchestration->getId(), 'orchestrationName' => $orchestration->getName(), 'projectId' => $token->getProjectId(), 'projectName' => $token->getOwnerName())); continue; } $orchestrationWithTasks = $this->dbOrchestrationManager->findOrchestrationById($orchestration->getId(), new StorageApi\Token($jobClient), true); $tasks = array_map(function ($task) { /** @var StorageApi\OrchestrationTask $task */ return $task->toApiArray(); }, $orchestrationWithTasks->getTasks()); $orchestration = $this->dbOrchestrationManager->updateLastScheduledTime($orchestration, new \DateTime()); // save job to queue $job = new Elasticsearch\Job(); $job->setOrchestrationId($orchestration->getId())->setConfig($orchestration->getId())->setOrchestrationName($orchestration->getName())->setToken($orchestration->getToken())->setTokenId($orchestration->getTokenId())->setTokenDesc($orchestration->getTokenDesc())->setTokenOwnerName($token->getOwnerName())->setProjectId($token->getProjectId())->setInitializedBy('scheduler')->setTaks($tasks)->setInitiatorTokenId($orchestration->getTokenId())->setInitiatorTokenDesc($orchestration->getTokenDesc())->setInitiatorUserAgent($this->appName . ' Scheduler'); $job = $jobEsManager->saveJob($job, new StorageApi\UniqueManager($jobClient)); $newJobs[] = $job; // add job to queue try { if (is_null($job)) { throw new ApplicationException('Unable to save or retrieve job from ES', null, array('orchestrationId' => $orchestration->getId())); } $this->queue->enqueue($job->getId(), array('jobId' => $job->getId(), 'component' => KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME)); $output->writeln(sprintf('Orchestration %d with cron %s scheduled. Job id: %s', $orchestration->getId(), $orchestration->getCrontabRecord(), $job->getId())); } catch (\Exception $e) { $this->logger->error('Error during adding job to sqs', array('lockName' => $this->lock->getLockName(), 'jobId' => $job->getId(), 'exception' => $e)); continue; } // log event try { $event = new Event(); $event->setComponent($this->appName)->setRunId($job->getRunId())->setParams(array('orchestrationId' => $job->getOrchestrationId()))->setResults(array('jobId' => $job->getId()))->setMessage(sprintf('Orchestration job %s scheduled', $job->getId())); StorageApi\EventLogger::create($event, $jobClient); $this->logger->info($event->getMessage(), array('lockName' => $this->lock->getLockName(), 'configurationId' => $event->getConfigurationId(), 'runId' => $event->getRunId(), 'description' => $event->getDescription(), 'params' => $event->getParams(), 'results' => $event->getResults())); } catch (Exception $e) { //@TODO user notification when notification API will be available $this->logger->warning('Invalid token', array('lockName' => $this->lock->getLockName(), 'jobId' => $job->getId(), 'exception' => $e)); } } $this->logger->info('scheduler.end', array('lockName' => $this->lock->getLockName(), 'duration' => $this->getDuration(), 'orchestrationsCheckedCount' => count($orchestrations), 'scheduledJobsCount' => count($newJobs))); sleep(90); // trying fix double run // unlock $this->lock->unlock(); }
/** * @param Elasticsearch\Job $job * @param Client $sapi * @param StorageApi\Notification $notification * @param null $averageDuration * @throws Exception */ private function logLongProcessing(Elasticsearch\Job $job, Client $sapi, StorageApi\Notification $notification, $averageDuration = null) { $event = new Event(); $event->setComponent(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME)->setType(Event::TYPE_WARN)->setMessage(sprintf(self::EVENT_MESSAGE_LONG_PROCESSING, $job->getId()))->setParams(array('jobId' => $job->getId(), 'orchestrationId' => $job->getOrchestrationId(), 'createdTime' => $job->getCreatedTime()->format('c'), 'notificationEmail' => $notification->getEmail(), 'tolerance' => $notification->getParameter('tolerance'), 'averageDuration' => $averageDuration)); StorageApi\EventLogger::create($event, $sapi); $this->logger->error(sprintf(self::EVENT_MESSAGE_LONG_PROCESSING, $job->getId()), array('job' => $job->getId(), 'orchestrationId' => $job->getOrchestrationId(), 'projectId' => $job->getProjectId(), 'project' => $job->getTokenOwnerName(), 'averageDuration' => $averageDuration)); }
/** * @param StorageApiEvent $event * @return mixed|string Event ID */ private function save(StorageApiEvent $event) { $event->setParams(array_merge($event->getParams(), array('jobId' => $this->job->getId(), 'orchestrationId' => $this->job->getOrchestrationId(), 'pid' => getmypid()))); if ($event->getType() == StorageApiEvent::TYPE_ERROR) { $event->setParams(array_merge($event->getParams(), array('notificationEmails' => $this->job->getNotificationsEmails()))); } return EventLogger::create($event, $this->client); }
public function killAction($jobId) { /** @var Search $elasticsearch */ $elasticsearch = $this->container->get('syrup.elasticsearch.search'); $job = $elasticsearch->getJob($jobId); if ($job == null) { throw new SyrupComponentException(404, sprintf("Job '%s' not found", $jobId)); } $sapiData = $this->container->get('syrup.storage_api')->getTokenData(); $projectId = $sapiData['owner']['id']; if ($job->getProject()['id'] != $projectId) { throw new SyrupComponentException(404, sprintf("Job id '%s' not found in project", $jobId)); } $sapiEvent = new Event(); $sapiEvent->setComponent($job->getComponent()); $sapiEvent->setMessage('Requested job termination'); $sapiEvent->setRunId($this->storageApi->getRunId()); $sapiEvent->setType(Event::TYPE_INFO); $sapiEvent->setRunId($job->getRunId()); $sapiEvent->setParams(['jobId' => $jobId]); $this->storageApi->createEvent($sapiEvent); $snsConfig = $this->container->getParameter('sns'); $snsClient = new SnsClient(['credentials' => ['key' => $snsConfig['key'], 'secret' => $snsConfig['secret']], 'region' => isset($snsConfig['region']) ? $snsConfig['region'] : 'us-east-1', 'version' => '2010-03-31']); $snsClient->publish(['TopicArn' => $snsConfig['topic_arn'], 'Message' => json_encode(['jobId' => $jobId])]); $jobData = $job->getData(); if (array_key_exists('terminatedBy', $jobData)) { $job->setTerminatedBy(['id' => $sapiData['id'], 'description' => $sapiData['description']]); } $jobMapper = $this->getComponentJobMapper($job->getComponent()); $jobMapper->update($job); return $this->createJsonResponse(["message" => "job termination request accepted for processing"], 202); }