protected function setUp() { parent::setUp(); $container = $this->httpClient->getContainer(); $this->encryptor = $container->get('syrup.encryptor'); $this->restApi = $container->get('wr_google_drive.rest_api'); $this->testCsvPath = __DIR__ . '/data/test.csv'; $this->configuration = $container->get('wr_google_drive.configuration'); $this->configuration->setStorageApi($this->storageApiClient); try { $this->configuration->create(); } catch (\Exception $e) { // bucket exists } $this->initEnv(); $this->initApi($this->accessToken, $this->refreshToken); // Cleanup $sysBucketId = $this->configuration->getSysBucketId(); $accTables = $this->storageApiClient->listTables($sysBucketId); foreach ($accTables as $table) { $this->storageApiClient->dropTable($table['id']); } if ($this->storageApiClient->tableExists($this->tableId)) { $this->storageApiClient->dropTable($this->tableId); } if ($this->storageApiClient->tableExists($this->rsTableId)) { $this->storageApiClient->dropTable($this->rsTableId); } if (!$this->storageApiClient->bucketExists('in.c-wr-google-drive')) { $this->storageApiClient->createBucket('wr-google-drive', Client::STAGE_IN, 'Google Drive IN bucket'); } if (!$this->storageApiClient->bucketExists('in.c-wr-google-drive-rs')) { $this->storageApiClient->createBucket('wr-google-drive', Client::STAGE_IN, 'Google Drive IN bucket', 'redshift'); } $csvFile = new CsvFile($this->testCsvPath); $this->storageApiClient->createTable('in.c-wr-google-drive', 'test', $csvFile); $this->storageApiClient->createTable('in.c-wr-google-drive-rs', 'test', $csvFile); }
public function oauthCallbackAction() { $session = $this->get('session'); $token = $session->get('token'); $accountId = $session->get('account'); $referrer = $session->get('referrer'); $external = $session->get('external'); if ($token == null) { throw new UserException("Your session expired, please try again"); } $code = $this->get('request')->query->get('code'); if (empty($code)) { throw new UserException('Could not read from Google API, please try again'); } $googleApi = $this->getGoogleApi(); try { $storageApi = new StorageApi(array('token' => $token, 'userAgent' => $this->componentName)); /** @var ObjectEncryptor $encryptor */ $encryptor = $this->get('syrup.object_encryptor'); $configuration = new Configuration($this->componentName, $encryptor); $configuration->setStorageApi($storageApi); $tokens = $googleApi->authorize($code, $this->container->get('router')->generate('keboola_google_drive_writer_oauth_callback', array(), UrlGeneratorInterface::ABSOLUTE_URL)); $googleApi->setCredentials($tokens['access_token'], $tokens['refresh_token']); $userData = json_decode($googleApi->request('/oauth2/v2/userinfo')->getBody(), true); if ($external) { $this->container->get('session')->clear(); $referrer .= strstr($referrer, '?') ? '&' : '?'; $referrer .= 'access-token=' . urlencode($this->container->get('syrup.encryptor')->encrypt($tokens['access_token'])) . '&refresh-token=' . urlencode($this->container->get('syrup.encryptor')->encrypt($tokens['refresh_token'])) . '&email=' . $userData['email']; return new RedirectResponse($referrer); } else { $account = $configuration->getAccountBy('accountId', $accountId); if (null == $account) { throw new ConfigurationException("Account doesn't exist"); } $account->setGoogleId($userData['id'])->setGoogleName($userData['name'])->setEmail($userData['email'])->setAccessToken($tokens['access_token'])->setRefreshToken($tokens['refresh_token'])->save(); } $this->container->get('session')->clear(); if ($referrer) { return new RedirectResponse($referrer); } else { return new JsonResponse(array('status' => 'ok')); } } catch (\Exception $e) { throw new SyrupComponentException(500, 'Could not save API tokens', $e); } }
public function execute(Job $job) { $this->configuration->setStorageApi($this->storageApi); $accounts = $this->configuration->getAccounts(); $options = $job->getParams(); $status = []; if (isset($options['external'])) { // load files by tag from SAPI if (!isset($options['external']['account'])) { throw new UserException("Missing field 'account'"); } try { $this->configuration->create(); } catch (\Exception $e) { // create configuration if not exists } $account = new Account($this->configuration, uniqid('external')); $account->fromArray($options['external']['account']); $writer = $this->writerFactory->create($account); if (!isset($options['external']['query'])) { throw new UserException("Missing field 'query'"); } $queryString = $options['external']['query']; $queryString .= ' -tags:wr-google-drive-processed'; if (isset($options['external']['filterByRunId']) && $options['external']['filterByRunId']) { $parentRunId = $this->getParentRunId($job->getRunId()); if ($parentRunId) { $queryString .= ' +tags:runId-' . $parentRunId; } } $uploadedFiles = $this->storageApi->listFiles((new ListFilesOptions())->setQuery($queryString)); if (empty($uploadedFiles)) { throw new UserException("No file matches your query '" . $queryString . "'"); } foreach ($uploadedFiles as $uploadedFile) { $tmpFile = $this->temp->createTmpFile('wr-gd'); file_put_contents($tmpFile->getPathname(), fopen($uploadedFile['url'], 'r')); $file = new File(['id' => $uploadedFile['id'], 'title' => $uploadedFile['name'], 'targetFolder' => isset($options['external']['targetFolder']) ? $options['external']['targetFolder'] : null, 'type' => File::TYPE_FILE, 'pathname' => $tmpFile, 'size' => $uploadedFile['sizeBytes']]); $gdFiles = $writer->listFiles(['q' => "trashed=false and name='" . $uploadedFile['name'] . "'"]); if (!empty($gdFiles['files'])) { $lastGdFile = array_shift($gdFiles['files']); $file->setGoogleId($lastGdFile['id']); } $file = $writer->process($file); // tag file 'wr-google-drive-processed' $this->storageApi->addFileTag($uploadedFile['id'], 'wr-google-drive-processed'); } // delete temporary account $this->configuration->removeAccount($account->getAccountId()); } else { $fileFilter = null; if (isset($options['config'])) { if (!isset($accounts[$options['config']])) { throw new UserException("Config '" . $options['config'] . "' does not exist."); } $accounts = [$options['config'] => $accounts[$options['config']]]; if (isset($options['file'])) { /** @var Account $account */ $account = $accounts[$options['config']]; if (null == $account->getFile($options['file'])) { throw new UserException("File '" . $options['file'] . "' not found"); } $fileFilter = $options['file']; } } /** @var Account $account */ foreach ($accounts as $accountId => $account) { $writer = $this->writerFactory->create($account); $files = $account->getFiles(); /** @var File $file */ foreach ($files as $file) { if ($fileFilter != null && $file->getId() != $fileFilter) { continue; } $file->setPathname($this->temp->createTmpFile()->getPathname()); try { $tableExporter = new TableExporter($this->storageApi); $tableExporter->exportTable($file->getTableId(), $file->getPathname(), []); } catch (ClientException $e) { throw new UserException($e->getMessage(), $e, ['file' => $file->toArray()]); } $file = $writer->process($file); $status[$account->getAccountName()][$file->getTitle()] = 'ok'; } // updated changes to files $account->save(); } } return $status; }
public function preExecute(Request $request) { parent::preExecute($request); $this->configuration = $this->container->get('wr_google_drive.configuration'); $this->configuration->setStorageApi($this->storageApi); }