public function run($table)
{
$incremental = isset($table['incremental']) ? $table['incremental'] : false;
$primaryKey = isset($table['primaryKey']) ? $table['primaryKey'] : null;
$maxTries = isset($this->dbConfig['retries']) ? $this->dbConfig['retries'] + 1 : 1;
if (is_numeric($maxTries) == false) {
throw new UserException("Retries parameter is not a valid number, given:" . $maxTries);
}
$tries = 0;
$isError = true;
$dbException = null;
while ($tries < $maxTries && $isError) {
$isError = false;
try {
$this->export($table['query'], $table['outputTable'], $incremental, $primaryKey);
} catch (DbException $e) {
$this->logger->warning("DB error occured", array('exception' => $e));
$isError = true;
$dbException = $e;
}
sleep(pow(2, $tries) + rand(0, 1000) / 1000);
$this->restartConnection();
$tries++;
}
if ($isError) {
$userException = new UserException("Query '{$table['query']}' failed after {$maxTries} attempt(s). Reason '{$dbException->getMessage()}'", $dbException);
$userException->setData($dbException->getData());
throw $userException;
}
}
public function testKernelException()
{
$request = Request::create('/syrup/run', 'POST');
$request->headers->set('X-StorageApi-Token', SYRUP_SAPI_TEST_TOKEN);
$message = uniqid();
$event = new GetResponseForExceptionEvent(self::$kernel, $request, HttpKernelInterface::MASTER_REQUEST, new UserException($message));
$this->listener->onKernelException($event);
$records = $this->testLogHandler->getRecords();
$this->assertCount(1, $records);
$record = array_pop($records);
$this->assertArrayHasKey('priority', $record);
$this->assertEquals('ERROR', $record['priority']);
$this->assertArrayHasKey('exception', $record);
$this->assertArrayHasKey('class', $record['exception']);
$this->assertEquals('Keboola\\Syrup\\Exception\\UserException', $record['exception']['class']);
$response = $event->getResponse();
$this->assertEquals(400, $response->getStatusCode());
$jsonResponse = json_decode($response->getContent(), true);
$this->assertArrayHasKey('status', $jsonResponse);
$this->assertEquals('error', $jsonResponse['status']);
$this->assertArrayHasKey('code', $jsonResponse);
$this->assertEquals(400, $jsonResponse['code']);
$this->assertArrayHasKey('exceptionId', $jsonResponse);
$this->assertArrayHasKey('runId', $jsonResponse);
$message = uniqid();
$event = new GetResponseForExceptionEvent(self::$kernel, $request, HttpKernelInterface::MASTER_REQUEST, new ClientException($message));
$this->listener->onKernelException($event);
$records = $this->testLogHandler->getRecords();
$this->assertCount(2, $records);
$record = array_pop($records);
$this->assertArrayHasKey('priority', $record);
$this->assertEquals('CRITICAL', $record['priority']);
$this->assertArrayHasKey('exception', $record);
$this->assertArrayHasKey('class', $record['exception']);
$this->assertEquals('Keboola\\StorageApi\\ClientException', $record['exception']['class']);
$response = $event->getResponse();
$this->assertEquals(500, $response->getStatusCode());
$jsonResponse = json_decode($response->getContent(), true);
$this->assertArrayHasKey('status', $jsonResponse);
$this->assertEquals('error', $jsonResponse['status']);
$this->assertArrayHasKey('code', $jsonResponse);
$this->assertEquals(500, $jsonResponse['code']);
$this->assertArrayHasKey('exceptionId', $jsonResponse);
$this->assertArrayHasKey('runId', $jsonResponse);
$exception = new UserException(uniqid());
$exception->setData(['d1' => uniqid(), 'd2' => uniqid()]);
$event = new GetResponseForExceptionEvent(self::$kernel, $request, HttpKernelInterface::MASTER_REQUEST, $exception);
$this->listener->onKernelException($event);
$records = $this->testLogHandler->getRecords();
$this->assertCount(3, $records);
$record = array_pop($records);
$this->assertArrayHasKey('context', $record);
$this->assertArrayHasKey('data', $record['context']);
$this->assertArrayHasKey('d1', $record['context']['data']);
$this->assertArrayHasKey('d2', $record['context']['data']);
}
public function run($options = null)
{
$this->dataManager = new DataManager($this->configuration, $this->temp);
$accounts = $this->configuration->getAccounts();
if (isset($options['account']) || isset($options['config'])) {
$accountId = isset($options['account']) ? $options['account'] : $options['config'];
if (!isset($accounts[$accountId])) {
throw new ConfigurationException("Account '" . $accountId . "' does not exist.");
}
$accounts = array($accountId => $accounts[$accountId]);
}
if (isset($options['sheetId'])) {
if (!isset($options['config']) && !isset($options['account'])) {
throw new UserException("Missing parameter 'config'");
}
if (!isset($options['googleId'])) {
throw new UserException("Missing parameter 'googleId'");
}
}
$status = array();
/** @var Account $account */
foreach ($accounts as $accountId => $account) {
$this->currAccountId = $accountId;
$this->driveApi->getApi()->setCredentials($account->getAccessToken(), $account->getRefreshToken());
$this->driveApi->getApi()->setRefreshTokenCallback(array($this, 'refreshTokenCallback'));
$sheets = $account->getSheets();
if (isset($options['sheetId'])) {
$sheets = [$account->getSheet($options['googleId'], $options['sheetId'])];
}
/** @var Sheet $sheet */
foreach ($sheets as $sheet) {
$this->logger->info('Importing sheet ' . $sheet->getSheetTitle());
try {
$meta = $this->driveApi->getFile($sheet->getGoogleId());
} catch (RequestException $e) {
if ($e->getResponse()->getStatusCode() == 404) {
throw new UserException(sprintf("File '%s' not found in Google Drive", $sheet->getTitle()), $e);
} else {
$userException = new UserException("Google Drive Error: " . $e->getMessage(), $e);
$userException->setData(array('message' => $e->getMessage(), 'reason' => $e->getResponse()->getReasonPhrase(), 'account' => $accountId, 'sheet' => $sheet->toArray()));
throw $userException;
}
}
if (!isset($meta['exportLinks'])) {
$e = new ApplicationException("ExportLinks missing in file resource");
$e->setData(['fileMetadata' => $meta]);
throw $e;
}
if (isset($meta['exportLinks']['text/csv'])) {
$exportLink = $meta['exportLinks']['text/csv'] . '&gid=' . $sheet->getSheetId();
} else {
$exportLink = str_replace('pdf', 'csv', $meta['exportLinks']['application/pdf']) . '&gid=' . $sheet->getSheetId();
}
try {
$data = $this->driveApi->export($exportLink);
if ($data->getSize() > 0) {
$this->dataManager->save($data, $sheet);
} else {
$this->logger->warning(sprintf("Sheet is empty. File: '%s', Sheet: '%s'.", $sheet->getTitle(), $sheet->getSheetTitle()));
$status[$accountId][$sheet->getSheetTitle()] = "file is empty";
}
} catch (RequestException $e) {
$userException = new UserException("Error importing file - sheet: '" . $sheet->getTitle() . " - " . $sheet->getSheetTitle() . "'. ", $e);
$userException->setData(array('message' => $e->getMessage(), 'reason' => $e->getResponse()->getReasonPhrase(), 'body' => substr($e->getResponse()->getBody(), 0, 300), 'account' => $accountId, 'sheet' => $sheet->toArray()));
throw $userException;
}
}
}
return array("status" => "ok", "sheets" => $status);
}
/**
* @param $id
* @return JsonResponse
*/
public function getAccountAction($id)
{
$account = $this->getConfiguration()->getAccountBy('accountId', $id);
if ($account == null) {
throw new UserException("Account '" . $id . "' not found");
}
if (isset($account->toArray()['items']) && !empty($account->toArray()['items'])) {
// this will get addtitional info from Google GA API if needed
if (!isset($account->toArray()['items'][0]['accountName']) || empty($account->toArray()['items'][0]['accountName'])) {
try {
$profiles = $this->getApi($account)->getAllProfiles();
} catch (RequestException $e) {
$userException = new UserException($e->getMessage(), $e);
$userException->setData(['code' => $e->getResponse()->getStatusCode(), 'message' => $e->getMessage(), 'reason' => $e->getResponse()->getReasonPhrase(), 'body' => $e->getResponse()->getBody()]);
throw $userException;
}
foreach ($account->getData() as $row) {
foreach ($profiles as $accName => $webProperties) {
foreach ($webProperties as $webPropertyName => $prfs) {
foreach ($prfs as $pr) {
if ($pr['id'] == $row['googleId']) {
$account->addProfile(new Profile(['googleId' => $pr['id'], 'name' => $pr['name'], 'webPropertyId' => $pr['webPropertyId'], 'webPropertyName' => $webPropertyName, 'accountId' => $pr['accountId'], 'accountName' => $accName]));
}
}
}
}
}
$account->save();
}
}
return $this->createJsonResponse($account->toArray());
}