示例#1
0
 public function run($table)
 {
     $incremental = isset($table['incremental']) ? $table['incremental'] : false;
     $primaryKey = isset($table['primaryKey']) ? $table['primaryKey'] : null;
     $maxTries = isset($this->dbConfig['retries']) ? $this->dbConfig['retries'] + 1 : 1;
     if (is_numeric($maxTries) == false) {
         throw new UserException("Retries parameter is not a valid number, given:" . $maxTries);
     }
     $tries = 0;
     $isError = true;
     $dbException = null;
     while ($tries < $maxTries && $isError) {
         $isError = false;
         try {
             $this->export($table['query'], $table['outputTable'], $incremental, $primaryKey);
         } catch (DbException $e) {
             $this->logger->warning("DB error occured", array('exception' => $e));
             $isError = true;
             $dbException = $e;
         }
         sleep(pow(2, $tries) + rand(0, 1000) / 1000);
         $this->restartConnection();
         $tries++;
     }
     if ($isError) {
         $userException = new UserException("Query '{$table['query']}' failed after {$maxTries} attempt(s). Reason '{$dbException->getMessage()}'", $dbException);
         $userException->setData($dbException->getData());
         throw $userException;
     }
 }
 public function testKernelException()
 {
     $request = Request::create('/syrup/run', 'POST');
     $request->headers->set('X-StorageApi-Token', SYRUP_SAPI_TEST_TOKEN);
     $message = uniqid();
     $event = new GetResponseForExceptionEvent(self::$kernel, $request, HttpKernelInterface::MASTER_REQUEST, new UserException($message));
     $this->listener->onKernelException($event);
     $records = $this->testLogHandler->getRecords();
     $this->assertCount(1, $records);
     $record = array_pop($records);
     $this->assertArrayHasKey('priority', $record);
     $this->assertEquals('ERROR', $record['priority']);
     $this->assertArrayHasKey('exception', $record);
     $this->assertArrayHasKey('class', $record['exception']);
     $this->assertEquals('Keboola\\Syrup\\Exception\\UserException', $record['exception']['class']);
     $response = $event->getResponse();
     $this->assertEquals(400, $response->getStatusCode());
     $jsonResponse = json_decode($response->getContent(), true);
     $this->assertArrayHasKey('status', $jsonResponse);
     $this->assertEquals('error', $jsonResponse['status']);
     $this->assertArrayHasKey('code', $jsonResponse);
     $this->assertEquals(400, $jsonResponse['code']);
     $this->assertArrayHasKey('exceptionId', $jsonResponse);
     $this->assertArrayHasKey('runId', $jsonResponse);
     $message = uniqid();
     $event = new GetResponseForExceptionEvent(self::$kernel, $request, HttpKernelInterface::MASTER_REQUEST, new ClientException($message));
     $this->listener->onKernelException($event);
     $records = $this->testLogHandler->getRecords();
     $this->assertCount(2, $records);
     $record = array_pop($records);
     $this->assertArrayHasKey('priority', $record);
     $this->assertEquals('CRITICAL', $record['priority']);
     $this->assertArrayHasKey('exception', $record);
     $this->assertArrayHasKey('class', $record['exception']);
     $this->assertEquals('Keboola\\StorageApi\\ClientException', $record['exception']['class']);
     $response = $event->getResponse();
     $this->assertEquals(500, $response->getStatusCode());
     $jsonResponse = json_decode($response->getContent(), true);
     $this->assertArrayHasKey('status', $jsonResponse);
     $this->assertEquals('error', $jsonResponse['status']);
     $this->assertArrayHasKey('code', $jsonResponse);
     $this->assertEquals(500, $jsonResponse['code']);
     $this->assertArrayHasKey('exceptionId', $jsonResponse);
     $this->assertArrayHasKey('runId', $jsonResponse);
     $exception = new UserException(uniqid());
     $exception->setData(['d1' => uniqid(), 'd2' => uniqid()]);
     $event = new GetResponseForExceptionEvent(self::$kernel, $request, HttpKernelInterface::MASTER_REQUEST, $exception);
     $this->listener->onKernelException($event);
     $records = $this->testLogHandler->getRecords();
     $this->assertCount(3, $records);
     $record = array_pop($records);
     $this->assertArrayHasKey('context', $record);
     $this->assertArrayHasKey('data', $record['context']);
     $this->assertArrayHasKey('d1', $record['context']['data']);
     $this->assertArrayHasKey('d2', $record['context']['data']);
 }
示例#3
0
 public function run($options = null)
 {
     $this->dataManager = new DataManager($this->configuration, $this->temp);
     $accounts = $this->configuration->getAccounts();
     if (isset($options['account']) || isset($options['config'])) {
         $accountId = isset($options['account']) ? $options['account'] : $options['config'];
         if (!isset($accounts[$accountId])) {
             throw new ConfigurationException("Account '" . $accountId . "' does not exist.");
         }
         $accounts = array($accountId => $accounts[$accountId]);
     }
     if (isset($options['sheetId'])) {
         if (!isset($options['config']) && !isset($options['account'])) {
             throw new UserException("Missing parameter 'config'");
         }
         if (!isset($options['googleId'])) {
             throw new UserException("Missing parameter 'googleId'");
         }
     }
     $status = array();
     /** @var Account $account */
     foreach ($accounts as $accountId => $account) {
         $this->currAccountId = $accountId;
         $this->driveApi->getApi()->setCredentials($account->getAccessToken(), $account->getRefreshToken());
         $this->driveApi->getApi()->setRefreshTokenCallback(array($this, 'refreshTokenCallback'));
         $sheets = $account->getSheets();
         if (isset($options['sheetId'])) {
             $sheets = [$account->getSheet($options['googleId'], $options['sheetId'])];
         }
         /** @var Sheet $sheet */
         foreach ($sheets as $sheet) {
             $this->logger->info('Importing sheet ' . $sheet->getSheetTitle());
             try {
                 $meta = $this->driveApi->getFile($sheet->getGoogleId());
             } catch (RequestException $e) {
                 if ($e->getResponse()->getStatusCode() == 404) {
                     throw new UserException(sprintf("File '%s' not found in Google Drive", $sheet->getTitle()), $e);
                 } else {
                     $userException = new UserException("Google Drive Error: " . $e->getMessage(), $e);
                     $userException->setData(array('message' => $e->getMessage(), 'reason' => $e->getResponse()->getReasonPhrase(), 'account' => $accountId, 'sheet' => $sheet->toArray()));
                     throw $userException;
                 }
             }
             if (!isset($meta['exportLinks'])) {
                 $e = new ApplicationException("ExportLinks missing in file resource");
                 $e->setData(['fileMetadata' => $meta]);
                 throw $e;
             }
             if (isset($meta['exportLinks']['text/csv'])) {
                 $exportLink = $meta['exportLinks']['text/csv'] . '&gid=' . $sheet->getSheetId();
             } else {
                 $exportLink = str_replace('pdf', 'csv', $meta['exportLinks']['application/pdf']) . '&gid=' . $sheet->getSheetId();
             }
             try {
                 $data = $this->driveApi->export($exportLink);
                 if ($data->getSize() > 0) {
                     $this->dataManager->save($data, $sheet);
                 } else {
                     $this->logger->warning(sprintf("Sheet is empty. File: '%s', Sheet: '%s'.", $sheet->getTitle(), $sheet->getSheetTitle()));
                     $status[$accountId][$sheet->getSheetTitle()] = "file is empty";
                 }
             } catch (RequestException $e) {
                 $userException = new UserException("Error importing file - sheet: '" . $sheet->getTitle() . " - " . $sheet->getSheetTitle() . "'. ", $e);
                 $userException->setData(array('message' => $e->getMessage(), 'reason' => $e->getResponse()->getReasonPhrase(), 'body' => substr($e->getResponse()->getBody(), 0, 300), 'account' => $accountId, 'sheet' => $sheet->toArray()));
                 throw $userException;
             }
         }
     }
     return array("status" => "ok", "sheets" => $status);
 }
 /**
  * @param $id
  * @return JsonResponse
  */
 public function getAccountAction($id)
 {
     $account = $this->getConfiguration()->getAccountBy('accountId', $id);
     if ($account == null) {
         throw new UserException("Account '" . $id . "' not found");
     }
     if (isset($account->toArray()['items']) && !empty($account->toArray()['items'])) {
         // this will get addtitional info from Google GA API if needed
         if (!isset($account->toArray()['items'][0]['accountName']) || empty($account->toArray()['items'][0]['accountName'])) {
             try {
                 $profiles = $this->getApi($account)->getAllProfiles();
             } catch (RequestException $e) {
                 $userException = new UserException($e->getMessage(), $e);
                 $userException->setData(['code' => $e->getResponse()->getStatusCode(), 'message' => $e->getMessage(), 'reason' => $e->getResponse()->getReasonPhrase(), 'body' => $e->getResponse()->getBody()]);
                 throw $userException;
             }
             foreach ($account->getData() as $row) {
                 foreach ($profiles as $accName => $webProperties) {
                     foreach ($webProperties as $webPropertyName => $prfs) {
                         foreach ($prfs as $pr) {
                             if ($pr['id'] == $row['googleId']) {
                                 $account->addProfile(new Profile(['googleId' => $pr['id'], 'name' => $pr['name'], 'webPropertyId' => $pr['webPropertyId'], 'webPropertyName' => $webPropertyName, 'accountId' => $pr['accountId'], 'accountName' => $accName]));
                             }
                         }
                     }
                 }
             }
             $account->save();
         }
     }
     return $this->createJsonResponse($account->toArray());
 }