protected function execute(InputInterface $input, OutputInterface $output)
{
parent::execute($input, $output);
$this->dbOrchestrationManager = $this->getContainer()->get('orchestrator.doctrine.orchestration_manager');
$projects = array();
if ($input->getOption('projectId')) {
$projects[(int) $input->getOption('projectId')] = (int) $input->getOption('projectId');
} else {
foreach ($this->dbOrchestrationManager->findOrchestrations(array()) as $orchestration) {
$projects[$orchestration->getProjectId()] = (int) $orchestration->getProjectId();
}
}
foreach ($projects as $projectId) {
$output->writeln(sprintf('<info>[%s] Check start</info>', $projectId));
$filter = array('projectId' => $projectId);
foreach ($this->dbOrchestrationManager->findOrchestrations($filter) as $orchestration) {
try {
$jobClient = $this->createProjectSapi($orchestration);
$token = new StorageApi\Token($jobClient);
if ($token->hasEnabledFeature(StorageApi\Token::ORCHESTRATOR_NEW_CONFIGURATION)) {
$output->writeln("\t" . sprintf('<comment>[%s] (%s) %s - New configuration already enabled</comment>', $orchestration->getProjectId(), $orchestration->getId(), $orchestration->getName()));
continue;
}
} catch (DisabledException $e) {
$message = $e->getMessage();
if ($e->getPrevious()) {
$message = sprintf('%s: %s', $message, $e->getPrevious()->getMessage());
}
$output->writeln("\t" . sprintf('<error>[%s] (%s) %s - ' . $message . '</error>', $orchestration->getProjectId(), $orchestration->getId(), $orchestration->getName()));
continue;
}
$orchestration = $this->dbOrchestrationManager->findOrchestrationById($orchestration->getId(), $token, false);
// validate configuration
try {
$this->validateToken($jobClient);
if (!$input->getOption('onlyErrors')) {
$output->writeln("\t" . sprintf('<info>[%s] (%s) %s - Checked</info>', $orchestration->getProjectId(), $orchestration->getId(), $orchestration->getName()));
}
} catch (StorageApi\InvalidStateException $e) {
$output->writeln("\t" . sprintf('<error>[%s] (%s) %s - %s</error>', $orchestration->getProjectId(), $orchestration->getId(), $orchestration->getName(), $e->getMessage()));
}
}
$output->writeln(sprintf('<info>[%s] Check finished</info>', $projectId));
}
}
protected function execute(InputInterface $input, OutputInterface $output)
{
parent::execute($input, $output);
$force = $input->getOption('force');
/**
* @var QuestionHelper $helper
*/
$helper = $this->getHelper('question');
$question = new Question('Please enter your manage token: ');
$question->setHidden(true);
$question->setHiddenFallback(false);
$manageToken = $helper->ask($input, $output, $question);
$manageClient = new \Keboola\ManageApi\Client(["token" => $manageToken]);
$this->client = new \GuzzleHttp\Client(['base_uri' => 'https://connection.keboola.com/manage/', 'headers' => ['X-KBC-ManageApiToken' => $manageToken, 'Accept-Encoding' => 'gzip', 'User-Agent' => $this->getUserAgent()]]);
$this->dbOrchestrationManager = $this->getContainer()->get('orchestrator.doctrine.orchestration_manager');
foreach ($manageClient->listMaintainers() as $maintainer) {
$maintainerPrinted = false;
$output->write('M');
foreach ($this->loadOrganizations($maintainer) as $organization) {
$organizationPrinted = false;
$output->write('O');
foreach ($manageClient->listOrganizationProjects($organization['id']) as $project) {
if ($project['isDisabled']) {
$output->write('P');
continue;
}
if (!$this->loadProjectOrchestrations($project['id']) && !$this->loadProjectDeletedOrchestrations($project['id'])) {
$output->write('P');
continue;
}
// delete orchestrator bucket if exists
if (!$maintainerPrinted) {
$maintainerPrinted = true;
$output->writeln("\t" . $maintainer['name']);
}
if (!$organizationPrinted) {
$organizationPrinted = true;
$output->writeln("\t\t" . $organization['name']);
}
$output->writeln("\t\t\t" . $project['name'] . " (" . $project['id'] . ")");
$projectToken = $manageClient->createProjectStorageToken($project['id'], ["description" => "Orchestrator old table deletion", "canManageBuckets" => true, "canReadAllFileUploads" => true, "expiresIn" => 1800]);
$client = new Client(["token" => $projectToken["token"]]);
foreach ($client->listBuckets() as $bucket) {
$bucketId = 'sys.c-orchestrator';
if ($bucket['id'] != $bucketId) {
continue;
}
// bucket delete
if ($force) {
try {
$client->dropBucket($bucketId, array('force' => true));
$output->writeln("\t\t\t\tOrchestrator bucket deleted");
} catch (Exception $e) {
$output->writeln("\t\t\t\t" . $e->getMessage());
continue;
}
$manageClient->addNotification(array('type' => 'common', 'projectId' => $project['id'], 'title' => sprintf(' Orchestrator bucket %s was removed from your project -- %s', 'sys.c-orchestrator', $project['name']), 'message' => 'As we announced on our [blog](http://status.keboola.com/orchestrator-table-deletion-announcement), ' . 'we have deleted the unused orchestrator bucket from your KBC storage. ' . 'This message is just for your information and has no effect on your existing orchestrations.'));
$output->writeln("\t\t\t\tNotification created");
} else {
$output->writeln("\t\t\t\tOrchestrator bucket will be deleted");
}
}
}
}
}
}
protected function execute(InputInterface $input, OutputInterface $output)
{
parent::execute($input, $output);
$force = $input->getOption('force');
$maintainerId = $input->getOption('maintainer');
if (!$maintainerId) {
$output->writeln(sprintf('<comment>Command skipped - Missing maintainer ID</comment>'));
return;
}
/**
* @var QuestionHelper $helper
*/
$helper = $this->getHelper('question');
$question = new Question('Please insert manage token: ');
$question->setHidden(true);
$question->setHiddenFallback(false);
$token = $helper->ask($input, $output, $question);
if (!$token) {
$output->writeln(sprintf('<comment>Command skipped</comment>'));
return;
}
$this->client = new \GuzzleHttp\Client(['base_uri' => 'https://connection.keboola.com/manage/', 'headers' => ['X-KBC-ManageApiToken' => $token, 'Accept-Encoding' => 'gzip', 'User-Agent' => $this->getUserAgent()]]);
foreach ($this->loadMaintainers() as $maintainer) {
if ($maintainer['id'] != $maintainerId) {
continue;
}
$output->writeln(sprintf('<comment>[%s] - %s</comment>', $maintainer['id'], $maintainer['name']));
foreach ($this->loadOrganizations($maintainer) as $organization) {
$output->writeln(sprintf("\t" . '<comment>[%s] - %s</comment>', $organization['id'], $organization['name']));
foreach ($this->loadProjects($organization) as $project) {
if ($project['isDisabled']) {
$output->writeln(sprintf("\t\t" . '<comment>[%s] - %s - Disabled</comment>', $project['id'], $project['name']));
continue;
}
if (in_array('orchestrator-kbc-config', $project['features'])) {
$output->writeln(sprintf("\t\t" . '<comment>[%s] - %s - Already migrated</comment>', $project['id'], $project['name']));
continue;
}
$output->writeln(sprintf("\t\t" . '<comment>[%s] - %s</comment>', $project['id'], $project['name']));
// proces command
$exec = array();
$exec[] = $_SERVER["SCRIPT_FILENAME"];
$exec[] = $this->getMigrationCommand()->getName();
$exec[] = $project['id'];
if ($force) {
$exec[] = '--force';
}
$process = new Process(implode(' ', $exec));
$process->run();
if ($process->getExitCode() == JobCommand::STATUS_SUCCESS) {
$output->writeln($process->getOutput());
if (!$force) {
$output->writeln(sprintf("\t\t" . '<info>[%s] - %s - Migration Test OK</info>', $project['id'], $project['name']));
} else {
$output->writeln(sprintf("\t\t" . '<info>[%s] - %s - Migration OK</info>', $project['id'], $project['name']));
$this->enableNewConfiguration($project);
$output->writeln(sprintf("\t\t" . '<info>[%s] - %s - Feature enabled</info>', $project['id'], $project['name']));
}
} else {
$output->writeln($process->getOutput());
if (!$force) {
$output->writeln(sprintf("\t\t" . '<error>[%s] - %s - Migration Test Error</error>', $project['id'], $project['name']));
} else {
$output->writeln(sprintf("\t\t" . '<error>[%s] - %s - Migration Error</error>', $project['id'], $project['name']));
}
return JobCommand::STATUS_ERROR;
}
}
}
}
}
protected function execute(InputInterface $input, OutputInterface $output)
{
parent::execute($input, $output);
$this->logger->info('scheduler.start', array());
$output->writeln('Current time: ' . $this->getStartTime()->format('Y-m-d H:i:s'));
$this->lock->setLockName('scheduler-' . $this->getStartTime()->format('Y-m-d-H-i'));
// skip locked
try {
if (!$this->lock->lock()) {
$this->logger->info('scheduler.skipped', array('lockName' => $this->lock->getLockName()));
return;
}
} catch (\Exception $e) {
$this->logger->info('scheduler.error', array('message' => $e->getMessage(), 'lockName' => $this->lock->getLockName()));
return;
}
$this->logger->info('scheduler.locked', array('lockName' => $this->lock->getLockName()));
/**
* @var Elasticsearch\JobManagerFactory $jobManagerFactory
*/
$jobManagerFactory = $this->getContainer()->get('orchestrator.job_manager.factory');
$this->queue = $this->getContainer()->get('syrup.queue_factory')->get(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME);
$this->dbOrchestrationManager = $this->getContainer()->get('orchestrator.doctrine.orchestration_manager');
$newJobs = array();
$orchestrations = $this->dbOrchestrationManager->findOrchestrations(array());
foreach ($orchestrations as $orchestration) {
// skip deleted and non-active orchestrations
if ($orchestration->getDeleted() || !$orchestration->getActive() || !$orchestration->getCrontabRecord()) {
continue;
}
$cronExpression = CronExpression::factory($orchestration->getCrontabRecord());
if (!$cronExpression->isDue($this->getStartTime())) {
continue;
}
$jobClient = new Client(array('token' => $orchestration->getToken(), 'url' => $this->getConfiguration()->getStorageApiUrl(), 'userAgent' => $this->appName));
try {
$pingClient = new StorageApi\PingClient($jobClient, $this->logger);
if (!$pingClient->ping()) {
$this->logger->info('scheduler.orchestration.skipped.disabled', array('lockName' => $this->lock->getLockName(), 'orchestrationId' => $orchestration->getId(), 'orchestrationName' => $orchestration->getName(), 'projectId' => $orchestration->getProjectId(), 'projectName' => $orchestration->getTokenOwnerName()));
continue;
}
} catch (\Exception $e) {
$this->logger->info('scheduler.orchestration.skipped.error', array('lockName' => $this->lock->getLockName(), 'message' => $e->getMessage(), 'orchestrationId' => $orchestration->getId(), 'orchestrationName' => $orchestration->getName(), 'projectId' => $orchestration->getProjectId(), 'projectName' => $orchestration->getTokenOwnerName()));
continue;
}
$token = new StorageApi\Token($jobClient);
$jobEsManager = $jobManagerFactory->createJobManager($token);
$jobsStats = $jobEsManager->getWaitingStats();
// skip waiting
if (array_key_exists($orchestration->getId(), $jobsStats) && $jobsStats[$orchestration->getId()] >= AppConfiguration::MAX_WAITING_JOBS_COUNT) {
$this->logger->info('scheduler.orchestration.skipped', array('lockName' => $this->lock->getLockName(), 'waitingCount' => $jobsStats[$orchestration->getId()], 'limit' => AppConfiguration::MAX_WAITING_JOBS_COUNT, 'orchestrationId' => $orchestration->getId(), 'orchestrationName' => $orchestration->getName(), 'projectId' => $token->getProjectId(), 'projectName' => $token->getOwnerName()));
continue;
}
$orchestrationWithTasks = $this->dbOrchestrationManager->findOrchestrationById($orchestration->getId(), new StorageApi\Token($jobClient), true);
$tasks = array_map(function ($task) {
/** @var StorageApi\OrchestrationTask $task */
return $task->toApiArray();
}, $orchestrationWithTasks->getTasks());
$orchestration = $this->dbOrchestrationManager->updateLastScheduledTime($orchestration, new \DateTime());
// save job to queue
$job = new Elasticsearch\Job();
$job->setOrchestrationId($orchestration->getId())->setConfig($orchestration->getId())->setOrchestrationName($orchestration->getName())->setToken($orchestration->getToken())->setTokenId($orchestration->getTokenId())->setTokenDesc($orchestration->getTokenDesc())->setTokenOwnerName($token->getOwnerName())->setProjectId($token->getProjectId())->setInitializedBy('scheduler')->setTaks($tasks)->setInitiatorTokenId($orchestration->getTokenId())->setInitiatorTokenDesc($orchestration->getTokenDesc())->setInitiatorUserAgent($this->appName . ' Scheduler');
$job = $jobEsManager->saveJob($job, new StorageApi\UniqueManager($jobClient));
$newJobs[] = $job;
// add job to queue
try {
if (is_null($job)) {
throw new ApplicationException('Unable to save or retrieve job from ES', null, array('orchestrationId' => $orchestration->getId()));
}
$this->queue->enqueue($job->getId(), array('jobId' => $job->getId(), 'component' => KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME));
$output->writeln(sprintf('Orchestration %d with cron %s scheduled. Job id: %s', $orchestration->getId(), $orchestration->getCrontabRecord(), $job->getId()));
} catch (\Exception $e) {
$this->logger->error('Error during adding job to sqs', array('lockName' => $this->lock->getLockName(), 'jobId' => $job->getId(), 'exception' => $e));
continue;
}
// log event
try {
$event = new Event();
$event->setComponent($this->appName)->setRunId($job->getRunId())->setParams(array('orchestrationId' => $job->getOrchestrationId()))->setResults(array('jobId' => $job->getId()))->setMessage(sprintf('Orchestration job %s scheduled', $job->getId()));
StorageApi\EventLogger::create($event, $jobClient);
$this->logger->info($event->getMessage(), array('lockName' => $this->lock->getLockName(), 'configurationId' => $event->getConfigurationId(), 'runId' => $event->getRunId(), 'description' => $event->getDescription(), 'params' => $event->getParams(), 'results' => $event->getResults()));
} catch (Exception $e) {
//@TODO user notification when notification API will be available
$this->logger->warning('Invalid token', array('lockName' => $this->lock->getLockName(), 'jobId' => $job->getId(), 'exception' => $e));
}
}
$this->logger->info('scheduler.end', array('lockName' => $this->lock->getLockName(), 'duration' => $this->getDuration(), 'orchestrationsCheckedCount' => count($orchestrations), 'scheduledJobsCount' => count($newJobs)));
sleep(90);
// trying fix double run
// unlock
$this->lock->unlock();
}
protected function execute(InputInterface $input, OutputInterface $output)
{
parent::execute($input, $output);
$output->writeln('Current time: ' . $this->getStartTime()->format('Y-m-d H:i:s'));
$this->lock->setLockName('orchestrator-watchdog-' . $this->getStartTime()->format('Y-m-d-H-i'));
// skip locked
try {
if (!$this->lock->lock()) {
$this->logger->info('watchdog.skipped', array('lockName' => $this->lock->getLockName()));
return;
}
} catch (\Exception $e) {
$this->logger->info('watchdog.error', array('message' => $e->getMessage(), 'lockName' => $this->lock->getLockName()));
return;
}
$this->logger->info('watchdog.locked', array('lockName' => $this->lock->getLockName()));
$this->init();
$this->watchWaiting($input, $output);
$this->watchProcessing($input, $output);
sleep(90);
// trying fix double run
// unlock
$this->lock->unlock();
}
protected function execute(InputInterface $input, OutputInterface $output)
{
parent::execute($input, $output);
$dateTimeFrom = new \DateTime($input->getArgument('dateFrom'));
if ($input->getArgument('dateTo')) {
$dateTimeTo = new \DateTime($input->getArgument('dateTo'));
} else {
$dateTimeTo = new \DateTime();
}
/**
* @var Elasticsearch\JobManagerFactory $jobManagerFactory
*/
$jobManagerFactory = $this->getContainer()->get('orchestrator.job_manager.factory');
/**
* @var Rds\OrchestrationManager $orchestrationManager
*/
$orchestrationManager = $this->getContainer()->get('orchestrator.doctrine.orchestration_manager');
$orchestrations = $orchestrationManager->findOrchestrations(array());
foreach ($orchestrations as $orchestration) {
// skip deleted and non-active orchestrations
if ($orchestration->getDeleted() || !$orchestration->getActive()) {
continue;
}
$jobClient = new Client(array('token' => $orchestration->getToken(), 'url' => $this->getConfiguration()->getStorageApiUrl(), 'userAgent' => $this->appName));
try {
$pingClient = new StorageApi\PingClient($jobClient, $this->logger);
if (!$pingClient->ping()) {
$output->writeln(sprintf('<comment>(%s) %s - (%s) %s</comment>', $orchestration->getProjectId(), $orchestration->getTokenOwnerName(), $orchestration->getId(), $orchestration->getName()));
$output->writeln(sprintf('<error>Project disabled</error>'));
continue;
}
} catch (\Exception $e) {
$output->writeln(sprintf('<comment>(%s) %s - (%s) %s</comment>', $orchestration->getProjectId(), $orchestration->getTokenOwnerName(), $orchestration->getId(), $orchestration->getName()));
$output->writeln(sprintf('<error>Could not verify token</error>'));
continue;
}
$jobEsManager = $jobManagerFactory->createJobManager(new StorageApi\Token($jobClient));
$from = clone $dateTimeFrom;
$to = clone $dateTimeTo;
if (!$orchestration->getCrontabRecord()) {
$output->writeln(sprintf('<comment>(%s) %s - (%s) %s</comment>', $orchestration->getProjectId(), $orchestration->getTokenOwnerName(), $orchestration->getId(), $orchestration->getName()));
$output->writeln(sprintf('<info>Crontab does not specified</info>'));
continue;
}
$cron = CronExpression::factory($orchestration->getCrontabRecord());
$output->writeln(sprintf('<comment>(%s) %s - (%s) %s</comment>', $orchestration->getProjectId(), $orchestration->getTokenOwnerName(), $orchestration->getId(), $orchestration->getName()));
$jobs = $jobEsManager->findJobs(array('params.orchestration.id' => $orchestration->getId()));
/** @var \DateTime $oldRun */
$oldRun = null;
while ($from < $to) {
$nextRun = $cron->getNextRunDate($from, 0, true);
if ($nextRun->getTimestamp() >= $orchestration->getCreatedTime()->getTimestamp()) {
if (!$oldRun || $nextRun->getTimestamp() != $oldRun->getTimestamp()) {
if ($nextRun > $to) {
break;
}
$jobCreated = false;
foreach ($jobs as $job) {
if ($job->getCreatedTime()->format('Y-m-d H:i') == $nextRun->format('Y-m-d H:i')) {
$output->writeln(sprintf('<info>%s</info>', $nextRun->format('Y-m-d H:i')));
$jobCreated = true;
}
}
if (!$jobCreated) {
$output->writeln(sprintf('<error>%s</error>', $nextRun->format('Y-m-d H:i')));
}
$oldRun = clone $nextRun;
}
}
$from->modify('+1 minute');
}
unset($from, $to, $oldRun, $cron, $jobs);
}
}
protected function execute(InputInterface $input, OutputInterface $output)
{
parent::execute($input, $output);
$this->dbOrchestrationManager = $this->getContainer()->get('orchestrator.doctrine.orchestration_manager');
$skippedCount = 0;
$successCount = 0;
$errorCount = 0;
$force = $input->getOption('force');
$projectId = $input->getArgument('projectId');
$this->logger->info('configuration-migration.start ' . $projectId, array('test' => $force ? true : false));
foreach ($this->loadProjectOrchestrations($projectId) as $orchestration) {
try {
$jobClient = $this->createProjectSapi($orchestration);
$token = new StorageApi\Token($jobClient);
if ($token->hasEnabledFeature(StorageApi\Token::ORCHESTRATOR_NEW_CONFIGURATION)) {
throw new DisabledException('Migration skipped: new configuration already enabled');
}
} catch (DisabledException $e) {
$message = $e->getMessage();
if ($e->getPrevious()) {
$message = sprintf('%s: %s', $message, $e->getMessage());
}
$output->writeln(sprintf('<error>' . $message . '</error>'));
$this->logger->info('configuration-migration.failed ' . $projectId, array('test' => $force ? true : false, 'reason' => $message));
$output->writeln(sprintf('<error>Finished with errors</error>'));
return JobCommand::STATUS_ERROR;
}
$orchestration = $this->dbOrchestrationManager->findOrchestrationById($orchestration->getId(), $token, true);
$components = new Components($jobClient);
// remove old configuration
$oldConfiguration = null;
try {
$oldConfiguration = $components->getConfiguration(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, sprintf("%s-%s", KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, $orchestration->getId()));
} catch (ClientException $e) {
if ($e->getCode() !== 404) {
throw $e;
}
}
if ($force) {
// create or update configuration
$orchestrationTaskManager = new StorageApi\OrchestrationTaskManager($jobClient);
$orchestrationTaskManager->updateTasks($orchestration);
// validate configuration
try {
$this->validateConfiguration($orchestration, $jobClient);
$output->writeln(sprintf('<info>(%s) %s - Migrated</info>', $orchestration->getId(), $orchestration->getName()));
if ($oldConfiguration) {
$components->deleteConfiguration(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, $oldConfiguration['id']);
$output->writeln(sprintf('<comment>(%s) - Old configuration deleted</comment>', $oldConfiguration['id']));
}
$successCount++;
} catch (StorageApi\InvalidStateException $e) {
$output->writeln(sprintf('<error>' . $e->getMessage() . '</error>'));
$errorCount++;
}
} else {
$output->writeln(sprintf('<comment>(%s) %s - Will be migrated</comment>', $orchestration->getId(), $orchestration->getName()));
if ($oldConfiguration) {
$output->writeln(sprintf('<comment>(%s) - Old configuration will be deleted</comment>', $oldConfiguration['id']));
}
$skippedCount++;
}
}
if ($errorCount) {
$output->writeln(sprintf('<error>Finished with errors</error>'));
} else {
$output->writeln(sprintf('<info>Finished</info>'));
}
$this->logger->info('configuration-migration.migrated ' . $projectId, array('test' => $force ? true : false, 'successCount' => $successCount, 'errorCount' => $errorCount, 'skippedCount' => $skippedCount));
if ($errorCount) {
return JobCommand::STATUS_ERROR;
}
}
