protected function createJob() { $encryptedToken = self::$kernel->getContainer()->get('syrup.encryptor')->encrypt($this->storageApiToken); /** @var ObjectEncryptor $configEncryptor */ $configEncryptor = self::$kernel->getContainer()->get('syrup.object_encryptor'); return new Job($configEncryptor, ['id' => $this->storageApiClient->generateId(), 'runId' => $this->storageApiClient->generateId(), 'project' => ['id' => '123', 'name' => 'Syrup TEST'], 'token' => ['id' => '123', 'description' => 'fake token', 'token' => $encryptedToken], 'component' => 'syrup', 'command' => 'run', 'params' => [], 'process' => ['host' => gethostname(), 'pid' => getmypid()], 'createdTime' => date('c'), 'lockName' => 'test-' . microtime(true)], null, null, null); }
/** * @param string $secret String to encrypt * @param string $componentId * @param string $token SAPI token * @return string Encrypted $secret by application $componentId */ public static function encrypt($secret, $componentId, $token = null, $toConfig = false, $sapiUrl) { if (empty($sapiUrl)) { throw new ApplicationException("StorageApi url is empty and must be set"); } $storageApiClient = new StorageApi(["token" => $token, "userAgent" => 'oauth-v2', "url" => $sapiUrl]); $components = $storageApiClient->indexAction()["components"]; $syrupApiUrl = null; foreach ($components as $component) { if ($component["id"] == 'queue') { // strip the component uri to syrup api uri // eg https://syrup.keboola.com/docker/docker-demo => https://syrup.keboola.com $syrupApiUrl = substr($component["uri"], 0, strpos($component["uri"], "/", 8)); break; } } if (empty($syrupApiUrl)) { throw new ApplicationException("SyrupApi url is empty"); } $config = ['super' => 'docker', "url" => $syrupApiUrl]; if (!is_null($token)) { $config['token'] = $token; } $client = Client::factory($config); try { return $client->encryptString($componentId, $secret, $toConfig ? ["path" => "configs"] : []); } catch (ClientException $e) { throw new UserException("Component based encryption of the app secret failed: " . $e->getMessage()); } }
protected function init($jobId) { $this->jobMapper = $this->getContainer()->get('syrup.elasticsearch.current_component_job_mapper'); // Get job from ES $this->job = $this->jobMapper->get($jobId); if ($this->job == null) { $this->logger->error("Job id '" . $jobId . "' not found."); return self::STATUS_ERROR; } // SAPI init /** @var ObjectEncryptor $encryptor */ $encryptor = $this->getContainer()->get('syrup.object_encryptor'); $this->sapiClient = new SapiClient(['token' => $encryptor->decrypt($this->job->getToken()['token']), 'url' => $this->getContainer()->getParameter('storage_api.url'), 'userAgent' => $this->job->getComponent()]); $this->sapiClient->setRunId($this->job->getRunId()); /** @var \Keboola\Syrup\Service\StorageApi\StorageApiService $storageApiService */ $storageApiService = $this->getContainer()->get('syrup.storage_api'); $storageApiService->setClient($this->sapiClient); $this->storageApiService = $storageApiService; /** @var \Keboola\Syrup\Monolog\Processor\JobProcessor $logProcessor */ $logProcessor = $this->getContainer()->get('syrup.monolog.job_processor'); $logProcessor->setJob($this->job); /** @var \Keboola\Syrup\Monolog\Processor\SyslogProcessor $logProcessor */ $logProcessor = $this->getContainer()->get('syrup.monolog.syslog_processor'); $logProcessor->setRunId($this->job->getRunId()); $logProcessor->setTokenData($storageApiService->getTokenData()); // Lock DB /** @var Connection $conn */ $conn = $this->getContainer()->get('doctrine.dbal.lock_connection'); $conn->exec('SET wait_timeout = 31536000;'); $this->lock = new Lock($conn, $this->job->getLockName()); }
private function createJob() { $tokenData = self::$sapiClient->verifyToken(); /** @var ObjectEncryptor $configEncryptor */ $configEncryptor = self::$kernel->getContainer()->get('syrup.object_encryptor'); return new Job($configEncryptor, ['id' => self::$sapiClient->generateId(), 'runId' => self::$sapiClient->generateId(), 'project' => ['id' => $tokenData['owner']['id'], 'name' => $tokenData['owner']['name']], 'token' => ['id' => $tokenData['id'], 'description' => $tokenData['description'], 'token' => self::$encryptor->encrypt(self::$sapiClient->getTokenString())], 'component' => SYRUP_APP_NAME, 'command' => 'run', 'process' => ['host' => 'test', 'pid' => posix_getpid()], 'createdTime' => date('c')], null, null, null); }
public function setUp() { $this->client = OrchestratorApi::factory(array('url' => FUNCTIONAL_ORCHESTRATOR_API_URL, 'token' => FUNCTIONAL_ORCHESTRATOR_API_TOKEN)); $this->sapiClient = new StorageApi(array('token' => FUNCTIONAL_ORCHESTRATOR_API_TOKEN, 'url' => defined('FUNCTIONAL_SAPI_URL') ? FUNCTIONAL_SAPI_URL : null)); $this->sapiClient->verifyToken(); // clean old tests $this->cleanWorkspace(); }
private function getSapiServiceStub() { $storageApiClient = new Client(['url' => STORAGE_API_URL, 'token' => STORAGE_API_TOKEN, 'userAgent' => 'docker-bundle']); $tokenData = $storageApiClient->verifyToken(); $storageServiceStub = $this->getMockBuilder(StorageApiService::class)->disableOriginalConstructor()->getMock(); $storageServiceStub->expects($this->any())->method("getClient")->will($this->returnValue($this->client)); $storageServiceStub->expects($this->any())->method("getTokenData")->will($this->returnValue($tokenData)); return $storageServiceStub; }
protected function writerProxyPost($writerId, $query, $payload, $toFile = null) { if (is_array($payload)) { $payload = json_encode($payload); } $request = $this->getWriter()->post('/gooddata-writer/proxy', array('Content-Type' => 'application/json; charset=utf-8', 'Accept' => 'application/json', 'X-StorageApi-Token' => $this->sapiClient->getTokenString()), json_encode(array('wait' => 1, 'writerId' => $writerId, 'query' => $query, 'payload' => $payload))); if ($toFile != null) { $request->setResponseBody($toFile); } return $request->send()->json(); }
protected function getComponentConfiguration() { // Check list of components $components = $this->storageApiClient->indexAction(); foreach ($components["components"] as $c) { if ($c["id"] == $this->componentName) { return $c; } } // no component configuration found return []; }
/** * Get uri template for component * * @param $componentId * @return string */ public function getJobUriTemplate($componentId) { if ($componentId !== KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME) { throw new \InvalidArgumentException(sprintf('Cannot get uri for "%s" component', $componentId)); } $jobUrl = '#'; $this->loadData(); // only for orchestrator if (!empty($this->urlTemplates['orchestrationJob'])) { $jobUrl = sprintf("%s%s", preg_replace('/(\\/+)$/', '', $this->storageApi->getApiUrl()), $this->urlTemplates['orchestrationJob']); } return $jobUrl; }
/** * Parallel jobs limit of KBC project, null if unlimited * * @param Client $storageApi * @return int|null */ public static function getParallelLimit(Client $storageApi) { $data = $storageApi->getLogData(); if (!empty($data['owner']['features'])) { foreach ($data['owner']['features'] as $feature) { $matches = array(); if (preg_match('/^syrup\\-jobs\\-limit\\-([0-9]+)$/ui', $feature, $matches)) { return (int) $matches[1]; } } } return null; }
public function handle(array $record) { $this->initStorageApiClient(); if (!$this->storageApiClient || $record['level'] == Logger::DEBUG) { return false; } $event = new Event(); if (!empty($record['component'])) { $event->setComponent($record['component']); } else { $event->setComponent($this->appName); } $event->setMessage(\Keboola\Utils\sanitizeUtf8($record['message'])); $event->setRunId($this->storageApiClient->getRunId()); $params = []; if (isset($record['http'])) { $params['http'] = $record['http']; } $event->setParams($params); $results = []; if (isset($record['context']['exceptionId'])) { $results['exceptionId'] = $record['context']['exceptionId']; } if (isset($record['context']['job'])) { $results['job'] = $record['context']['job']; } $event->setResults($results); switch ($record['level']) { case Logger::ERROR: $type = Event::TYPE_ERROR; break; case Logger::CRITICAL: case Logger::EMERGENCY: case Logger::ALERT: $type = Event::TYPE_ERROR; $event->setMessage("Application error"); $event->setDescription("Contact support@keboola.com"); break; case Logger::WARNING: case Logger::NOTICE: $type = Event::TYPE_WARN; break; case Logger::INFO: default: $type = Event::TYPE_INFO; break; } $event->setType($type); $this->storageApiClient->createEvent($event); return false; }
/** * @param Event $event * @param Client $client * @return int * @throws ClientException * @deprecated Events are logged by syrup logger */ public static function create(Event $event, Client $client) { try { return $client->createEvent($event); } catch (ClientException $e) { if ($e->getStringCode() === self::STATUS_CODE_REQUEST_TOO_LARGE) { $newEvent = clone $event; $newEvent->setResults(array('Message' => 'Large result was stripped due maximum allowed size for event.')); return $client->createEvent($newEvent); } else { throw $e; } } }
/** * @param Elasticsearch\Job $job * @param Notification $notification * @param Client $sapi * @return array */ public static function longProcessingEvents(Elasticsearch\Job $job, Notification $notification, Client $sapi) { $query = array(sprintf("type:%s", Event::TYPE_WARN), sprintf("message:'%s'", sprintf(CronWatchdogCommand::EVENT_MESSAGE_LONG_PROCESSING, $job->getId()))); $params = array('q' => implode(' ', $query), 'component' => KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, 'limit' => 1000); return array_filter($sapi->listEvents($params), function ($row) use($notification) { if (empty($row['params'])) { return false; } if (empty($row['params']['notificationEmail'])) { return false; } return $row['params']['notificationEmail'] === $notification->getEmail(); }); }
public function createToken() { $permissions = array($this->getSysBucketId() => 'write'); $tokenId = $this->storageApi->createToken($permissions, 'External Authorization', $this->tokenExpiration); $token = $this->storageApi->getToken($tokenId); return $token; }
public function getClient() { $request = $this->requestStack->getCurrentRequest(); if ($this->client == null) { if ($request == null) { throw new NoRequestException(); } if (!$request->headers->has('X-StorageApi-Token')) { throw new UserException('Missing StorageAPI token'); } $this->setClient(new Client(['token' => $request->headers->get('X-StorageApi-Token'), 'url' => $this->storageApiUrl, 'userAgent' => explode('/', $request->getPathInfo())[1], 'backoffMaxTries' => $this->getBackoffTries(gethostname())])); if ($request->headers->has('X-KBC-RunId')) { $this->client->setRunId($request->headers->get('X-KBC-RunId')); } } return $this->client; }
public function addFile($accountId, $fileData) { $account = $this->getAccount($accountId); $fileData['id'] = $this->storageApi->generateId(); $account->addFile($fileData); $account->save(); return $fileData['id']; }
protected function sendEventToSapi($type, $message, $componentName) { $sapiEvent = new SapiEvent(); $sapiEvent->setComponent($componentName); $sapiEvent->setMessage($message); $sapiEvent->setRunId($this->storageApi->getRunId()); $sapiEvent->setType($type); $this->storageApi->createEvent($sapiEvent); }
public function testLargeEvents() { $client = new StorageApiClient(array('token' => TEST_ORCHESTRATOR_SAPI_TOKEN, 'url' => TEST_ORCHESTRATOR_SAPI_URL)); $testLimit = 300 * 1024; $testMessage = 'This is sample text for large evenet testing.'; $event = new Event(); $event->setComponent(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME)->setMessage('Test - Large Event')->setResults(array('Message' => str_repeat($testMessage, (int) ($testLimit / mb_strlen($testMessage))))); $sizeError = false; try { $client->createEvent($event); } catch (ClientException $e) { $this->assertEquals(EventLogger::STATUS_CODE_REQUEST_TOO_LARGE, $e->getStringCode()); $sizeError = true; $eventId = EventLogger::create($event, $client); $this->assertNotEmpty($eventId); } $this->assertTrue($sizeError); }
public function deleteTable($writerId, $id) { $table = $this->tableFactory->get($writerId, $id); try { $this->storageApi->dropTable($table->getId()); } catch (ClientException $e) { // table already deleted } }
/** * @param Orchestration $entity * @param \Keboola\StorageApi\Client $storageApi * @return Orchestration */ public function fillOrchestrationEntity(Orchestration $entity, Client $storageApi) { $oldTokenId = $entity->getTokenId(); $oldName = $entity->getName(); /** @var $entity Orchestration */ $entity = parent::fillEntity($entity); $name = $entity->getName(); if (empty($name)) { $entity->setName($oldName); } // token change if ($oldTokenId !== $entity->getTokenId()) { $token = $storageApi->getToken($entity->getTokenId()); $token = new Token(new Client(array('token' => $token['token'], 'url' => $storageApi->getApiUrl()))); $entity->setToken($token)->setTokenId($token->getId())->setTokenDesc($token->getDescription())->setTokenOwnerName($token->getOwnerName())->setProjectId($token->getProjectId()); } // notifications $data = $this->getPostJsonData(); if (!empty($data['notifications'])) { $data['notifications'] = array_map(function ($row) { $notification = new Notification(); $notification->fromArray($row); return $notification; }, $data['notifications']); $entity->setNotifications($data['notifications']); } // tasks $data = $this->getPostJsonData(); if (array_key_exists('tasks', $data)) { $entity->removeTasks(); $data['tasks'] = array_map(function ($row) { $notification = new OrchestrationTask(); $notification->fromArray($row); return $notification; }, $data['tasks']); foreach ($data['tasks'] as $task) { $entity->addTask($task); } } //@FIXME catch and throw exception return $entity; }
public function getSysBucketId() { if ($this->storageApi->bucketExists('sys.c-' . $this->componentName)) { return 'sys.c-' . $this->componentName; } else { if ($this->storageApi->bucketExists('sys.' . $this->componentName)) { return 'sys.' . $this->componentName; } } throw new ConfigurationException("SYS bucket don't exists"); }
public function ping() { $options = array('timeout' => 7200, 'headers' => array('X-StorageApi-Token' => $this->storageApi->getTokenString(), 'Accept-Encoding' => 'gzip', 'User-Agent' => $this->storageApi->getUserAgent())); if ($this->storageApi->getRunId()) { $options['headers']['X-KBC-RunId'] = $this->storageApi->getRunId(); } try { $response = $this->client->get('storage/tokens/verify', $options); } catch (RequestException $e) { $response = $e->getResponse(); if ($response && $response->getStatusCode() == 503) { return false; } else { throw $e; } } if ($response && $response->getStatusCode() == 200) { $body = ResponseDecoder::decode($response); return array_key_exists('token', $body); } return false; }
public function getClient() { //@todo remove in 2.6 - setRequest() will be removed if ($this->request != null) { $request = $this->request; } else { $request = $this->requestStack->getCurrentRequest(); } if ($this->client == null) { if ($request == null) { throw new NoRequestException(); } if (!$request->headers->has('X-StorageApi-Token')) { throw new UserException('Missing StorageAPI token'); } $this->setClient(new Client(['token' => $request->headers->get('X-StorageApi-Token'), 'url' => $this->storageApiUrl, 'userAgent' => explode('/', $request->getPathInfo())[1]])); if ($request->headers->has('X-KBC-RunId')) { $this->client->setRunId($request->headers->get('X-KBC-RunId')); } } return $this->client; }
protected function setUp() { self::$client = static::createClient(); $container = self::$client->getContainer(); $sapiToken = $container->getParameter('storage_api.test.token'); $sapiUrl = $container->getParameter('storage_api.test.url'); self::$client->setServerParameters(array('HTTP_X-StorageApi-Token' => $sapiToken)); $this->storageApi = new SapiClient(['token' => $sapiToken, 'url' => $sapiUrl, 'userAgent' => $this->componentName]); $this->encryptor = $container->get('syrup.encryptor'); $this->configuration = $container->get('ex_google_drive.configuration'); $this->configuration->setStorageApi($this->storageApi); try { $this->configuration->create(); } catch (\Exception $e) { // bucket exists } // Cleanup $sysBucketId = $this->configuration->getSysBucketId(); $tableId = $sysBucketId . '.' . $this->accountId; if ($this->storageApi->tableExists($tableId)) { $this->storageApi->dropTable($tableId); } }
protected function setUp($driver = null) { self::$client = static::createClient(); $this->container = self::$client->getContainer(); $sapiToken = $this->container->getParameter('storage_api.test.token'); $sapiUrl = $this->container->getParameter('storage_api.test.url'); self::$client->setServerParameters(['HTTP_X-StorageApi-Token' => $sapiToken]); $this->storageApi = new SapiClient(['token' => $sapiToken, 'url' => $sapiUrl, 'userAgent' => $this->componentName]); if ($driver != null) { $this->configuration = new Configuration($this->componentName . '-' . $driver, $this->storageApi, $driver); } else { $this->configuration = new Configuration($this->componentName, $this->storageApi); } // Cleanup $sysBucketId = $this->configuration->getSysBucketId($this->writerId); if ($this->storageApi->bucketExists($sysBucketId)) { $accTables = $this->storageApi->listTables($sysBucketId); foreach ($accTables as $table) { $this->storageApi->dropTable($table['id']); } $this->storageApi->dropBucket($sysBucketId); } }
public function testConnection() { $testing = $this->container->getParameter('testing'); $tokenData = $this->storageApi->verifyToken(); /** @var Encryptor $encryptor */ $encryptor = $this->container->get('syrup.encryptor'); /** @var ObjectEncryptor $objectEncryptor */ $objectEncryptor = $this->container->get('syrup.object_encryptor'); /** @var Executor $executor */ $executor = $this->container->get('syrup.job_executor'); $executor->setStorageApi($this->storageApi); $result = $executor->execute(new Job($objectEncryptor, ['id' => $this->storageApi->generateId(), 'runId' => $this->storageApi->generateId(), 'project' => ['id' => $tokenData['owner']['id'], 'name' => $tokenData['owner']['name']], 'token' => ['id' => $tokenData['id'], 'description' => $tokenData['description'], 'token' => $encryptor->encrypt($this->storageApi->getTokenString())], 'component' => $this->componentName, 'command' => 'test', 'params' => $testing['db']['mysql'], 'process' => ['host' => gethostname(), 'pid' => getmypid()], 'createdTime' => date('c')])); $this->assertArrayHasKey('status', $result); $this->assertEquals('ok', $result['status']); }
/** * @param Orchestration $entity * @param \Keboola\StorageApi\Client $storageApi * @return Orchestration */ public function fillOrchestrationEntity(Orchestration $entity, Client $storageApi) { /** @var $entity Orchestration */ $entity = parent::fillEntity($entity); if (!$entity->getTokenId()) { $entity->setTokenId($storageApi->createToken('manage', sprintf('Orchestrator %s', $entity->getName()), null, true)); } // fill token data $token = $storageApi->getToken($entity->getTokenId()); $token = new Token(new Client(array('token' => $token['token'], 'url' => $storageApi->getApiUrl()))); $entity->setToken($token)->setTokenId($token->getId())->setTokenDesc($token->getDescription())->setTokenOwnerName($token->getOwnerName())->setProjectId($token->getProjectId()); // notifications $data = $this->getPostJsonData(); if (!empty($data['notifications'])) { $data['notifications'] = array_map(function ($row) { $notification = new Notification(); $notification->fromArray($row); return $notification; }, $data['notifications']); $entity->setNotifications($data['notifications']); } // tasks $data = $this->getPostJsonData(); if (!empty($data['tasks'])) { $data['tasks'] = array_map(function ($row) { $notification = new OrchestrationTask(); $notification->fromArray($row); return $notification; }, $data['tasks']); foreach ($data['tasks'] as $task) { $entity->addTask($task); } } //@FIXME catch and throw exception return $entity; }
/** * Get parent runId to the current runId (defined by SAPI client) * @return string Parent part of hierarchical Id. */ public function getParentRunId() { $runId = $this->client->getRunId(); if (!empty($runId)) { if (($pos = strrpos($runId, '.')) === false) { // there is no parent $parentRunId = $runId; } else { $parentRunId = substr($runId, 0, $pos); } } else { // there is no runId $parentRunId = ''; } return $parentRunId; }
protected function writeTable(CsvFile $csv, $outputTable, $incremental, $primaryKey) { try { $tableNameArr = explode('.', $outputTable); $bucketId = $tableNameArr[0] . "." . $tableNameArr[1]; $tableName = $tableNameArr[2]; } catch (ContextErrorException $e) { throw new UserException("Wrong output table name.", $e); } if (!count($csv->getHeader())) { throw new ApplicationException("Trying to upload an empty file"); } try { if (!$this->storageApi->bucketExists($bucketId)) { $bucketArr = explode('.', $bucketId); $this->storageApi->createBucket(str_replace('c-', '', $bucketArr[1]), SapiClient::STAGE_IN, 'DB Extractor data bucket'); } if (!$this->storageApi->tableExists($outputTable)) { $this->storageApi->createTableAsync($bucketId, $tableName, $csv, array('primaryKey' => $primaryKey)); } else { // handle unexpected temporary errors like "unable to fork()" $success = false; $exception = null; for ($i = 0; $i < 2 && !$success; $i++) { try { $this->storageApi->writeTableAsync($outputTable, $csv, array('incremental' => $incremental)); $success = true; } catch (\Exception $e) { $exception = $e; $this->logger->warning("Error writing to SAPI", ['exception' => $exception]); } sleep(1); } if (!$success) { throw $exception; } } } catch (ClientException $e) { if ($e->getCode() < 500) { throw new UserException($e->getMessage(), $e); } else { throw new ApplicationException($e->getMessage(), $e); } } $this->logger->info("Table " . $tableName . " imported to Storage API"); }
public function addQuery($accountId, $params, $rowId = null) { $table = $this->getAccountTable($accountId); if ($rowId == null) { // post request $rowId = $this->storageApi->generateId(); } // BC if (!isset($params['name']) && isset($params['outputTable'])) { $outputTableArr = explode('.', $params['outputTable']); $params['name'] = array_pop($outputTableArr); } // outputTable can be derived from name $outputTable = isset($params['outputTable']) && !empty($params['outputTable']) ? $params['outputTable'] : "in.c-" . $this->componentName . "-" . $accountId . "." . $params['name']; $arrayToSave = ['id' => $rowId, 'name' => $params['name'], 'query' => $this->checkArray($params, 'query'), 'outputTable' => $outputTable, 'incremental' => $this->checkArray($params, 'incremental'), 'primaryKey' => $this->checkArray($params, 'primaryKey'), 'enabled' => isset($params['enabled']) ? intval($params['enabled']) : 1]; $table->setFromArray([$arrayToSave]); $table->save(); return $arrayToSave; }