protected function createJob()
{
$encryptedToken = self::$kernel->getContainer()->get('syrup.encryptor')->encrypt($this->storageApiToken);
/** @var ObjectEncryptor $configEncryptor */
$configEncryptor = self::$kernel->getContainer()->get('syrup.object_encryptor');
return new Job($configEncryptor, ['id' => $this->storageApiClient->generateId(), 'runId' => $this->storageApiClient->generateId(), 'project' => ['id' => '123', 'name' => 'Syrup TEST'], 'token' => ['id' => '123', 'description' => 'fake token', 'token' => $encryptedToken], 'component' => 'syrup', 'command' => 'run', 'params' => [], 'process' => ['host' => gethostname(), 'pid' => getmypid()], 'createdTime' => date('c'), 'lockName' => 'test-' . microtime(true)], null, null, null);
}
/**
* @param string $secret String to encrypt
* @param string $componentId
* @param string $token SAPI token
* @return string Encrypted $secret by application $componentId
*/
public static function encrypt($secret, $componentId, $token = null, $toConfig = false, $sapiUrl)
{
if (empty($sapiUrl)) {
throw new ApplicationException("StorageApi url is empty and must be set");
}
$storageApiClient = new StorageApi(["token" => $token, "userAgent" => 'oauth-v2', "url" => $sapiUrl]);
$components = $storageApiClient->indexAction()["components"];
$syrupApiUrl = null;
foreach ($components as $component) {
if ($component["id"] == 'queue') {
// strip the component uri to syrup api uri
// eg https://syrup.keboola.com/docker/docker-demo => https://syrup.keboola.com
$syrupApiUrl = substr($component["uri"], 0, strpos($component["uri"], "/", 8));
break;
}
}
if (empty($syrupApiUrl)) {
throw new ApplicationException("SyrupApi url is empty");
}
$config = ['super' => 'docker', "url" => $syrupApiUrl];
if (!is_null($token)) {
$config['token'] = $token;
}
$client = Client::factory($config);
try {
return $client->encryptString($componentId, $secret, $toConfig ? ["path" => "configs"] : []);
} catch (ClientException $e) {
throw new UserException("Component based encryption of the app secret failed: " . $e->getMessage());
}
}
protected function init($jobId)
{
$this->jobMapper = $this->getContainer()->get('syrup.elasticsearch.current_component_job_mapper');
// Get job from ES
$this->job = $this->jobMapper->get($jobId);
if ($this->job == null) {
$this->logger->error("Job id '" . $jobId . "' not found.");
return self::STATUS_ERROR;
}
// SAPI init
/** @var ObjectEncryptor $encryptor */
$encryptor = $this->getContainer()->get('syrup.object_encryptor');
$this->sapiClient = new SapiClient(['token' => $encryptor->decrypt($this->job->getToken()['token']), 'url' => $this->getContainer()->getParameter('storage_api.url'), 'userAgent' => $this->job->getComponent()]);
$this->sapiClient->setRunId($this->job->getRunId());
/** @var \Keboola\Syrup\Service\StorageApi\StorageApiService $storageApiService */
$storageApiService = $this->getContainer()->get('syrup.storage_api');
$storageApiService->setClient($this->sapiClient);
$this->storageApiService = $storageApiService;
/** @var \Keboola\Syrup\Monolog\Processor\JobProcessor $logProcessor */
$logProcessor = $this->getContainer()->get('syrup.monolog.job_processor');
$logProcessor->setJob($this->job);
/** @var \Keboola\Syrup\Monolog\Processor\SyslogProcessor $logProcessor */
$logProcessor = $this->getContainer()->get('syrup.monolog.syslog_processor');
$logProcessor->setRunId($this->job->getRunId());
$logProcessor->setTokenData($storageApiService->getTokenData());
// Lock DB
/** @var Connection $conn */
$conn = $this->getContainer()->get('doctrine.dbal.lock_connection');
$conn->exec('SET wait_timeout = 31536000;');
$this->lock = new Lock($conn, $this->job->getLockName());
}
private function createJob()
{
$tokenData = self::$sapiClient->verifyToken();
/** @var ObjectEncryptor $configEncryptor */
$configEncryptor = self::$kernel->getContainer()->get('syrup.object_encryptor');
return new Job($configEncryptor, ['id' => self::$sapiClient->generateId(), 'runId' => self::$sapiClient->generateId(), 'project' => ['id' => $tokenData['owner']['id'], 'name' => $tokenData['owner']['name']], 'token' => ['id' => $tokenData['id'], 'description' => $tokenData['description'], 'token' => self::$encryptor->encrypt(self::$sapiClient->getTokenString())], 'component' => SYRUP_APP_NAME, 'command' => 'run', 'process' => ['host' => 'test', 'pid' => posix_getpid()], 'createdTime' => date('c')], null, null, null);
}
public function setUp()
{
$this->client = OrchestratorApi::factory(array('url' => FUNCTIONAL_ORCHESTRATOR_API_URL, 'token' => FUNCTIONAL_ORCHESTRATOR_API_TOKEN));
$this->sapiClient = new StorageApi(array('token' => FUNCTIONAL_ORCHESTRATOR_API_TOKEN, 'url' => defined('FUNCTIONAL_SAPI_URL') ? FUNCTIONAL_SAPI_URL : null));
$this->sapiClient->verifyToken();
// clean old tests
$this->cleanWorkspace();
}
private function getSapiServiceStub()
{
$storageApiClient = new Client(['url' => STORAGE_API_URL, 'token' => STORAGE_API_TOKEN, 'userAgent' => 'docker-bundle']);
$tokenData = $storageApiClient->verifyToken();
$storageServiceStub = $this->getMockBuilder(StorageApiService::class)->disableOriginalConstructor()->getMock();
$storageServiceStub->expects($this->any())->method("getClient")->will($this->returnValue($this->client));
$storageServiceStub->expects($this->any())->method("getTokenData")->will($this->returnValue($tokenData));
return $storageServiceStub;
}
protected function writerProxyPost($writerId, $query, $payload, $toFile = null)
{
if (is_array($payload)) {
$payload = json_encode($payload);
}
$request = $this->getWriter()->post('/gooddata-writer/proxy', array('Content-Type' => 'application/json; charset=utf-8', 'Accept' => 'application/json', 'X-StorageApi-Token' => $this->sapiClient->getTokenString()), json_encode(array('wait' => 1, 'writerId' => $writerId, 'query' => $query, 'payload' => $payload)));
if ($toFile != null) {
$request->setResponseBody($toFile);
}
return $request->send()->json();
}
protected function getComponentConfiguration()
{
// Check list of components
$components = $this->storageApiClient->indexAction();
foreach ($components["components"] as $c) {
if ($c["id"] == $this->componentName) {
return $c;
}
}
// no component configuration found
return [];
}
/**
* Get uri template for component
*
* @param $componentId
* @return string
*/
public function getJobUriTemplate($componentId)
{
if ($componentId !== KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME) {
throw new \InvalidArgumentException(sprintf('Cannot get uri for "%s" component', $componentId));
}
$jobUrl = '#';
$this->loadData();
// only for orchestrator
if (!empty($this->urlTemplates['orchestrationJob'])) {
$jobUrl = sprintf("%s%s", preg_replace('/(\\/+)$/', '', $this->storageApi->getApiUrl()), $this->urlTemplates['orchestrationJob']);
}
return $jobUrl;
}
/**
* Parallel jobs limit of KBC project, null if unlimited
*
* @param Client $storageApi
* @return int|null
*/
public static function getParallelLimit(Client $storageApi)
{
$data = $storageApi->getLogData();
if (!empty($data['owner']['features'])) {
foreach ($data['owner']['features'] as $feature) {
$matches = array();
if (preg_match('/^syrup\\-jobs\\-limit\\-([0-9]+)$/ui', $feature, $matches)) {
return (int) $matches[1];
}
}
}
return null;
}
public function handle(array $record)
{
$this->initStorageApiClient();
if (!$this->storageApiClient || $record['level'] == Logger::DEBUG) {
return false;
}
$event = new Event();
if (!empty($record['component'])) {
$event->setComponent($record['component']);
} else {
$event->setComponent($this->appName);
}
$event->setMessage(\Keboola\Utils\sanitizeUtf8($record['message']));
$event->setRunId($this->storageApiClient->getRunId());
$params = [];
if (isset($record['http'])) {
$params['http'] = $record['http'];
}
$event->setParams($params);
$results = [];
if (isset($record['context']['exceptionId'])) {
$results['exceptionId'] = $record['context']['exceptionId'];
}
if (isset($record['context']['job'])) {
$results['job'] = $record['context']['job'];
}
$event->setResults($results);
switch ($record['level']) {
case Logger::ERROR:
$type = Event::TYPE_ERROR;
break;
case Logger::CRITICAL:
case Logger::EMERGENCY:
case Logger::ALERT:
$type = Event::TYPE_ERROR;
$event->setMessage("Application error");
$event->setDescription("Contact support@keboola.com");
break;
case Logger::WARNING:
case Logger::NOTICE:
$type = Event::TYPE_WARN;
break;
case Logger::INFO:
default:
$type = Event::TYPE_INFO;
break;
}
$event->setType($type);
$this->storageApiClient->createEvent($event);
return false;
}
/**
* @param Event $event
* @param Client $client
* @return int
* @throws ClientException
* @deprecated Events are logged by syrup logger
*/
public static function create(Event $event, Client $client)
{
try {
return $client->createEvent($event);
} catch (ClientException $e) {
if ($e->getStringCode() === self::STATUS_CODE_REQUEST_TOO_LARGE) {
$newEvent = clone $event;
$newEvent->setResults(array('Message' => 'Large result was stripped due maximum allowed size for event.'));
return $client->createEvent($newEvent);
} else {
throw $e;
}
}
}
/**
* @param Elasticsearch\Job $job
* @param Notification $notification
* @param Client $sapi
* @return array
*/
public static function longProcessingEvents(Elasticsearch\Job $job, Notification $notification, Client $sapi)
{
$query = array(sprintf("type:%s", Event::TYPE_WARN), sprintf("message:'%s'", sprintf(CronWatchdogCommand::EVENT_MESSAGE_LONG_PROCESSING, $job->getId())));
$params = array('q' => implode(' ', $query), 'component' => KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME, 'limit' => 1000);
return array_filter($sapi->listEvents($params), function ($row) use($notification) {
if (empty($row['params'])) {
return false;
}
if (empty($row['params']['notificationEmail'])) {
return false;
}
return $row['params']['notificationEmail'] === $notification->getEmail();
});
}
public function createToken()
{
$permissions = array($this->getSysBucketId() => 'write');
$tokenId = $this->storageApi->createToken($permissions, 'External Authorization', $this->tokenExpiration);
$token = $this->storageApi->getToken($tokenId);
return $token;
}
public function getClient()
{
$request = $this->requestStack->getCurrentRequest();
if ($this->client == null) {
if ($request == null) {
throw new NoRequestException();
}
if (!$request->headers->has('X-StorageApi-Token')) {
throw new UserException('Missing StorageAPI token');
}
$this->setClient(new Client(['token' => $request->headers->get('X-StorageApi-Token'), 'url' => $this->storageApiUrl, 'userAgent' => explode('/', $request->getPathInfo())[1], 'backoffMaxTries' => $this->getBackoffTries(gethostname())]));
if ($request->headers->has('X-KBC-RunId')) {
$this->client->setRunId($request->headers->get('X-KBC-RunId'));
}
}
return $this->client;
}
public function addFile($accountId, $fileData)
{
$account = $this->getAccount($accountId);
$fileData['id'] = $this->storageApi->generateId();
$account->addFile($fileData);
$account->save();
return $fileData['id'];
}
protected function sendEventToSapi($type, $message, $componentName)
{
$sapiEvent = new SapiEvent();
$sapiEvent->setComponent($componentName);
$sapiEvent->setMessage($message);
$sapiEvent->setRunId($this->storageApi->getRunId());
$sapiEvent->setType($type);
$this->storageApi->createEvent($sapiEvent);
}
public function testLargeEvents()
{
$client = new StorageApiClient(array('token' => TEST_ORCHESTRATOR_SAPI_TOKEN, 'url' => TEST_ORCHESTRATOR_SAPI_URL));
$testLimit = 300 * 1024;
$testMessage = 'This is sample text for large evenet testing.';
$event = new Event();
$event->setComponent(KeboolaOrchestratorBundle::SYRUP_COMPONENT_NAME)->setMessage('Test - Large Event')->setResults(array('Message' => str_repeat($testMessage, (int) ($testLimit / mb_strlen($testMessage)))));
$sizeError = false;
try {
$client->createEvent($event);
} catch (ClientException $e) {
$this->assertEquals(EventLogger::STATUS_CODE_REQUEST_TOO_LARGE, $e->getStringCode());
$sizeError = true;
$eventId = EventLogger::create($event, $client);
$this->assertNotEmpty($eventId);
}
$this->assertTrue($sizeError);
}
public function deleteTable($writerId, $id)
{
$table = $this->tableFactory->get($writerId, $id);
try {
$this->storageApi->dropTable($table->getId());
} catch (ClientException $e) {
// table already deleted
}
}
/**
* @param Orchestration $entity
* @param \Keboola\StorageApi\Client $storageApi
* @return Orchestration
*/
public function fillOrchestrationEntity(Orchestration $entity, Client $storageApi)
{
$oldTokenId = $entity->getTokenId();
$oldName = $entity->getName();
/** @var $entity Orchestration */
$entity = parent::fillEntity($entity);
$name = $entity->getName();
if (empty($name)) {
$entity->setName($oldName);
}
// token change
if ($oldTokenId !== $entity->getTokenId()) {
$token = $storageApi->getToken($entity->getTokenId());
$token = new Token(new Client(array('token' => $token['token'], 'url' => $storageApi->getApiUrl())));
$entity->setToken($token)->setTokenId($token->getId())->setTokenDesc($token->getDescription())->setTokenOwnerName($token->getOwnerName())->setProjectId($token->getProjectId());
}
// notifications
$data = $this->getPostJsonData();
if (!empty($data['notifications'])) {
$data['notifications'] = array_map(function ($row) {
$notification = new Notification();
$notification->fromArray($row);
return $notification;
}, $data['notifications']);
$entity->setNotifications($data['notifications']);
}
// tasks
$data = $this->getPostJsonData();
if (array_key_exists('tasks', $data)) {
$entity->removeTasks();
$data['tasks'] = array_map(function ($row) {
$notification = new OrchestrationTask();
$notification->fromArray($row);
return $notification;
}, $data['tasks']);
foreach ($data['tasks'] as $task) {
$entity->addTask($task);
}
}
//@FIXME catch and throw exception
return $entity;
}
public function getSysBucketId()
{
if ($this->storageApi->bucketExists('sys.c-' . $this->componentName)) {
return 'sys.c-' . $this->componentName;
} else {
if ($this->storageApi->bucketExists('sys.' . $this->componentName)) {
return 'sys.' . $this->componentName;
}
}
throw new ConfigurationException("SYS bucket don't exists");
}
public function ping()
{
$options = array('timeout' => 7200, 'headers' => array('X-StorageApi-Token' => $this->storageApi->getTokenString(), 'Accept-Encoding' => 'gzip', 'User-Agent' => $this->storageApi->getUserAgent()));
if ($this->storageApi->getRunId()) {
$options['headers']['X-KBC-RunId'] = $this->storageApi->getRunId();
}
try {
$response = $this->client->get('storage/tokens/verify', $options);
} catch (RequestException $e) {
$response = $e->getResponse();
if ($response && $response->getStatusCode() == 503) {
return false;
} else {
throw $e;
}
}
if ($response && $response->getStatusCode() == 200) {
$body = ResponseDecoder::decode($response);
return array_key_exists('token', $body);
}
return false;
}
public function getClient()
{
//@todo remove in 2.6 - setRequest() will be removed
if ($this->request != null) {
$request = $this->request;
} else {
$request = $this->requestStack->getCurrentRequest();
}
if ($this->client == null) {
if ($request == null) {
throw new NoRequestException();
}
if (!$request->headers->has('X-StorageApi-Token')) {
throw new UserException('Missing StorageAPI token');
}
$this->setClient(new Client(['token' => $request->headers->get('X-StorageApi-Token'), 'url' => $this->storageApiUrl, 'userAgent' => explode('/', $request->getPathInfo())[1]]));
if ($request->headers->has('X-KBC-RunId')) {
$this->client->setRunId($request->headers->get('X-KBC-RunId'));
}
}
return $this->client;
}
protected function setUp()
{
self::$client = static::createClient();
$container = self::$client->getContainer();
$sapiToken = $container->getParameter('storage_api.test.token');
$sapiUrl = $container->getParameter('storage_api.test.url');
self::$client->setServerParameters(array('HTTP_X-StorageApi-Token' => $sapiToken));
$this->storageApi = new SapiClient(['token' => $sapiToken, 'url' => $sapiUrl, 'userAgent' => $this->componentName]);
$this->encryptor = $container->get('syrup.encryptor');
$this->configuration = $container->get('ex_google_drive.configuration');
$this->configuration->setStorageApi($this->storageApi);
try {
$this->configuration->create();
} catch (\Exception $e) {
// bucket exists
}
// Cleanup
$sysBucketId = $this->configuration->getSysBucketId();
$tableId = $sysBucketId . '.' . $this->accountId;
if ($this->storageApi->tableExists($tableId)) {
$this->storageApi->dropTable($tableId);
}
}
protected function setUp($driver = null)
{
self::$client = static::createClient();
$this->container = self::$client->getContainer();
$sapiToken = $this->container->getParameter('storage_api.test.token');
$sapiUrl = $this->container->getParameter('storage_api.test.url');
self::$client->setServerParameters(['HTTP_X-StorageApi-Token' => $sapiToken]);
$this->storageApi = new SapiClient(['token' => $sapiToken, 'url' => $sapiUrl, 'userAgent' => $this->componentName]);
if ($driver != null) {
$this->configuration = new Configuration($this->componentName . '-' . $driver, $this->storageApi, $driver);
} else {
$this->configuration = new Configuration($this->componentName, $this->storageApi);
}
// Cleanup
$sysBucketId = $this->configuration->getSysBucketId($this->writerId);
if ($this->storageApi->bucketExists($sysBucketId)) {
$accTables = $this->storageApi->listTables($sysBucketId);
foreach ($accTables as $table) {
$this->storageApi->dropTable($table['id']);
}
$this->storageApi->dropBucket($sysBucketId);
}
}
public function testConnection()
{
$testing = $this->container->getParameter('testing');
$tokenData = $this->storageApi->verifyToken();
/** @var Encryptor $encryptor */
$encryptor = $this->container->get('syrup.encryptor');
/** @var ObjectEncryptor $objectEncryptor */
$objectEncryptor = $this->container->get('syrup.object_encryptor');
/** @var Executor $executor */
$executor = $this->container->get('syrup.job_executor');
$executor->setStorageApi($this->storageApi);
$result = $executor->execute(new Job($objectEncryptor, ['id' => $this->storageApi->generateId(), 'runId' => $this->storageApi->generateId(), 'project' => ['id' => $tokenData['owner']['id'], 'name' => $tokenData['owner']['name']], 'token' => ['id' => $tokenData['id'], 'description' => $tokenData['description'], 'token' => $encryptor->encrypt($this->storageApi->getTokenString())], 'component' => $this->componentName, 'command' => 'test', 'params' => $testing['db']['mysql'], 'process' => ['host' => gethostname(), 'pid' => getmypid()], 'createdTime' => date('c')]));
$this->assertArrayHasKey('status', $result);
$this->assertEquals('ok', $result['status']);
}
/**
* @param Orchestration $entity
* @param \Keboola\StorageApi\Client $storageApi
* @return Orchestration
*/
public function fillOrchestrationEntity(Orchestration $entity, Client $storageApi)
{
/** @var $entity Orchestration */
$entity = parent::fillEntity($entity);
if (!$entity->getTokenId()) {
$entity->setTokenId($storageApi->createToken('manage', sprintf('Orchestrator %s', $entity->getName()), null, true));
}
// fill token data
$token = $storageApi->getToken($entity->getTokenId());
$token = new Token(new Client(array('token' => $token['token'], 'url' => $storageApi->getApiUrl())));
$entity->setToken($token)->setTokenId($token->getId())->setTokenDesc($token->getDescription())->setTokenOwnerName($token->getOwnerName())->setProjectId($token->getProjectId());
// notifications
$data = $this->getPostJsonData();
if (!empty($data['notifications'])) {
$data['notifications'] = array_map(function ($row) {
$notification = new Notification();
$notification->fromArray($row);
return $notification;
}, $data['notifications']);
$entity->setNotifications($data['notifications']);
}
// tasks
$data = $this->getPostJsonData();
if (!empty($data['tasks'])) {
$data['tasks'] = array_map(function ($row) {
$notification = new OrchestrationTask();
$notification->fromArray($row);
return $notification;
}, $data['tasks']);
foreach ($data['tasks'] as $task) {
$entity->addTask($task);
}
}
//@FIXME catch and throw exception
return $entity;
}
/**
* Get parent runId to the current runId (defined by SAPI client)
* @return string Parent part of hierarchical Id.
*/
public function getParentRunId()
{
$runId = $this->client->getRunId();
if (!empty($runId)) {
if (($pos = strrpos($runId, '.')) === false) {
// there is no parent
$parentRunId = $runId;
} else {
$parentRunId = substr($runId, 0, $pos);
}
} else {
// there is no runId
$parentRunId = '';
}
return $parentRunId;
}
protected function writeTable(CsvFile $csv, $outputTable, $incremental, $primaryKey)
{
try {
$tableNameArr = explode('.', $outputTable);
$bucketId = $tableNameArr[0] . "." . $tableNameArr[1];
$tableName = $tableNameArr[2];
} catch (ContextErrorException $e) {
throw new UserException("Wrong output table name.", $e);
}
if (!count($csv->getHeader())) {
throw new ApplicationException("Trying to upload an empty file");
}
try {
if (!$this->storageApi->bucketExists($bucketId)) {
$bucketArr = explode('.', $bucketId);
$this->storageApi->createBucket(str_replace('c-', '', $bucketArr[1]), SapiClient::STAGE_IN, 'DB Extractor data bucket');
}
if (!$this->storageApi->tableExists($outputTable)) {
$this->storageApi->createTableAsync($bucketId, $tableName, $csv, array('primaryKey' => $primaryKey));
} else {
// handle unexpected temporary errors like "unable to fork()"
$success = false;
$exception = null;
for ($i = 0; $i < 2 && !$success; $i++) {
try {
$this->storageApi->writeTableAsync($outputTable, $csv, array('incremental' => $incremental));
$success = true;
} catch (\Exception $e) {
$exception = $e;
$this->logger->warning("Error writing to SAPI", ['exception' => $exception]);
}
sleep(1);
}
if (!$success) {
throw $exception;
}
}
} catch (ClientException $e) {
if ($e->getCode() < 500) {
throw new UserException($e->getMessage(), $e);
} else {
throw new ApplicationException($e->getMessage(), $e);
}
}
$this->logger->info("Table " . $tableName . " imported to Storage API");
}
public function addQuery($accountId, $params, $rowId = null)
{
$table = $this->getAccountTable($accountId);
if ($rowId == null) {
// post request
$rowId = $this->storageApi->generateId();
}
// BC
if (!isset($params['name']) && isset($params['outputTable'])) {
$outputTableArr = explode('.', $params['outputTable']);
$params['name'] = array_pop($outputTableArr);
}
// outputTable can be derived from name
$outputTable = isset($params['outputTable']) && !empty($params['outputTable']) ? $params['outputTable'] : "in.c-" . $this->componentName . "-" . $accountId . "." . $params['name'];
$arrayToSave = ['id' => $rowId, 'name' => $params['name'], 'query' => $this->checkArray($params, 'query'), 'outputTable' => $outputTable, 'incremental' => $this->checkArray($params, 'incremental'), 'primaryKey' => $this->checkArray($params, 'primaryKey'), 'enabled' => isset($params['enabled']) ? intval($params['enabled']) : 1];
$table->setFromArray([$arrayToSave]);
$table->save();
return $arrayToSave;
}
