/**
* {@inheritdoc}
*/
public function getNextRequest(ClientInterface $client, JobConfig $jobConfig, $response, $data)
{
$nextUrl = Utils::getDataFromPath($this->urlParam, $response, '.');
if (empty($nextUrl)) {
return false;
}
// since validation - cannot be greater than now
$now = new \DateTime();
$sinceDateTime = \DateTime::createFromFormat('U', Url::fromString($nextUrl)->getQuery()->get('since'));
if ($sinceDateTime && $sinceDateTime > $now) {
return false;
}
$config = $jobConfig->getConfig();
if (!$this->includeParams) {
$config['params'] = [];
}
if (!$this->paramIsQuery) {
$config['endpoint'] = $nextUrl;
} else {
// Create an array from the query string
$responseQuery = Query::fromString(ltrim($nextUrl, '?'))->toArray();
$config['params'] = array_replace($config['params'], $responseQuery);
}
return $client->createRequest($config);
}
/**
* {@inheritdoc}
*/
public function getNextRequest(ClientInterface $client, JobConfig $jobConfig, $response, $data)
{
$nextUrl = Utils::getDataFromPath($this->urlParam, $response, '.');
if (empty($nextUrl)) {
return false;
}
// start_time validation
// https://developer.zendesk.com/rest_api/docs/core/incremental_export#incremental-ticket-export
$now = new \DateTime();
$startDateTime = \DateTime::createFromFormat('U', Url::fromString($nextUrl)->getQuery()->get('start_time'));
if ($startDateTime && $startDateTime > $now->modify(sprintf("-%d minutes", self::NEXT_PAGE_FILTER_MINUTES))) {
return false;
}
$config = $jobConfig->getConfig();
if (!$this->includeParams) {
$config['params'] = [];
}
if (!$this->paramIsQuery) {
$config['endpoint'] = $nextUrl;
} else {
// Create an array from the query string
$responseQuery = Query::fromString(ltrim($nextUrl, '?'))->toArray();
$config['params'] = array_replace($config['params'], $responseQuery);
}
return $client->createRequest($config);
}
protected function getFileUrl($response, $attempt = 0)
{
Logger::log("debug", "Retrieving report file URL (attempt = {$attempt}).", Utils::to_assoc($response));
switch ($response->status) {
case "REPORT_AVAILABLE":
Logger::log("debug", "Report {$this->config["reportId"]} is ready! URL: {$response->urls->apiUrl}", Utils::to_assoc($response));
if ($response->format != "CSV") {
throw new UserException("[415] The report {$this->config["reportId"]} export format is not CSV.");
}
$result = $response->urls->apiUrl;
break;
case "PROCESSING":
if ($attempt > 10) {
throw new UserException("The report export timed out.");
}
$attempt++;
$sleep = pow(2, $attempt);
Logger::log("debug", "Report {$this->config["reportId"]} is still not ready (status = {$response->status}). Waiting for {$sleep}s.");
sleep($sleep);
$result = $this->getFileUrl($this->download($this->client->createRequest("GET", "userprofiles/{$this->config["profileId"]}/reports/{$this->config["reportId"]}/files/{$response->id}")), $attempt);
break;
default:
throw new SyrupComponentException(500, "Unknown report status {$response->status}");
}
return $result;
}
/**
* Try to find the data array within $response.
*
* @param array|object $response
* @param array $config
* @return array
* @todo support array of dataFields
* - would return object with results, changing the class' API
* - parse would just have to loop through if it returns an object
* - and append $type with the dataField
* @deprecated Use response module
*/
public function process($response, JobConfig $jobConfig)
{
$config = $jobConfig->getConfig();
// If dataField doesn't say where the data is in a response, try to find it!
if (!empty($config['dataField'])) {
if (is_array($config['dataField'])) {
if (empty($config['dataField']['path'])) {
throw new UserException("'dataField.path' must be set!");
}
$path = $config['dataField']['path'];
} elseif (is_scalar($config['dataField'])) {
$path = $config['dataField'];
} else {
throw new UserException("'dataField' must be either a path string or an object with 'path' attribute.");
}
$data = Utils::getDataFromPath($path, $response, ".");
if (empty($data)) {
Logger::log('warning', "dataField '{$path}' contains no data!");
$data = [];
} elseif (!is_array($data)) {
// In case of a single object being returned
$data = [$data];
}
} elseif (is_array($response)) {
// Simplest case, the response is just the dataset
$data = $response;
} elseif (is_object($response)) {
// Find arrays in the response
$arrays = [];
foreach ($response as $key => $value) {
if (is_array($value)) {
$arrays[$key] = $value;
}
// TODO else {$this->metadata[$key] = json_encode($value);} ? return [$data,$metadata];
}
$arrayNames = array_keys($arrays);
if (count($arrays) == 1) {
$data = $arrays[$arrayNames[0]];
} elseif (count($arrays) == 0) {
Logger::log('warning', "No data array found in response! (endpoint: {$config['endpoint']})", ['response' => json_encode($response)]);
$data = [];
} else {
$e = new UserException("More than one array found in response! Use 'dataField' parameter to specify a key to the data array. (endpoint: {$config['endpoint']}, arrays in response root: " . join(", ", $arrayNames) . ")");
$e->setData(['response' => json_encode($response), 'arrays found' => $arrayNames]);
throw $e;
}
} else {
$e = new UserException('Unknown response from API.');
$e->setData(['response' => json_encode($response)]);
throw $e;
}
return $data;
}
/**
* @param array|\stdClass $functions
* @param array $params ['attr' => $attributesArray, ...]
* @param Builder $builder
* @return array
*/
public static function build($functions, array $params = [], Builder $builder = null)
{
if (is_null($builder)) {
$builder = new Builder();
}
$functions = (array) Utils::arrayToObject($functions);
try {
array_walk($functions, function (&$value, $key) use($params, $builder) {
$value = !is_object($value) ? $value : $builder->run($value, $params);
});
} catch (UserScriptException $e) {
throw new UserException('User script error: ' . $e->getMessage());
}
return $functions;
}
public function __construct(ScrollerInterface $scroller, array $config)
{
if (!empty($config['forceStop'])) {
if (!empty($config['forceStop']['pages'])) {
$this->pageLimit = $config['forceStop']['pages'];
}
if (!empty($config['forceStop']['time'])) {
$this->timeLimit = is_int($config['forceStop']['time']) ? $config['forceStop']['time'] : strtotime($config['forceStop']['time'], 0);
}
if (!empty($config['forceStop']['volume'])) {
$this->volumeLimit = Utils::return_bytes($config['forceStop']['volume']);
}
}
parent::__construct($scroller, $config);
$this->reset();
}
/**
* @return array
*/
public function process($response, JobConfig $jobConfig)
{
if (empty($jobConfig->getConfig()['parseObject'])) {
return $response;
}
$config = $jobConfig->getConfig()['parseObject'];
if (!is_object($response)) {
if (empty($response)) {
return [];
}
throw new UserException("Data in response is not an object, while one was expected!");
}
$path = empty($config['path']) ? "." : $config['path'];
$key = empty($config['keyColumn']) ? "rowId" : $config['keyColumn'];
return $this->convertObjectWithKeys(Utils::getDataFromPath($path, $response, '.'), $key);
}
/**
*
* Creates key value pairs for `values` property up to 2 leves of nesting
*
* @param $path
* @param $data
* @return array
* @throws \Keboola\Utils\Exception\NoDataFoundException
*/
protected function flatten($path, $data)
{
foreach (Utils::getDataFromPath($path, $data, '.') as $metric) {
if ($metric->values && is_array($metric->values)) {
$parsedMetrics = [];
foreach ($metric->values as $value) {
$end_time = '';
if (isset($value->end_time)) {
$end_time = $value->end_time;
}
if (!isset($value->end_time) && $metric->period != 'lifetime') {
continue;
}
// scalar value or empty value
if (!isset($value->value) || is_scalar($value->value)) {
if (!isset($value->value)) {
$val = 0;
} else {
$val = $value->value;
}
$parsedMetrics[] = (object) ["id" => $metric->id, "key1" => "", "key2" => "", "end_time" => $end_time, "value" => $val];
continue;
}
if (is_object($value->value)) {
foreach ((array) $value->value as $key1 => $value1) {
if (is_object($value1)) {
foreach ((array) $value1 as $key2 => $value2) {
$parsedMetrics[] = (object) ["id" => $metric->id, "key1" => $key1, "key2" => $key2, "end_time" => $end_time, "value" => $value2];
}
} else {
$parsedMetrics[] = (object) ["id" => $metric->id, "key1" => $key1, "key2" => "", "end_time" => $end_time, "value" => $value1];
}
}
continue;
}
}
$metric->values = $parsedMetrics;
}
$result[] = $metric;
}
if ($path != '.') {
$data->{$path} = $result;
} else {
$data = $result;
}
return $data;
}
/**
* {@inheritdoc}
*/
public function getNextRequest(ClientInterface $client, JobConfig $jobConfig, $response, $data)
{
$nextParam = Utils::getDataFromPath($this->responseParam, $response, '.');
if (empty($nextParam)) {
return false;
} else {
$config = $jobConfig->getConfig();
if (!$this->includeParams) {
$config['params'] = [];
}
if (!is_null($this->scrollRequest)) {
$config = $this->createScrollRequest($config, $this->scrollRequest);
}
$config['params'][$this->queryParam] = $nextParam;
return $client->createRequest($config);
}
}
public function __construct(array $authorization)
{
if (empty($authorization['oauth_api']['credentials'])) {
throw new UserException("OAuth API credentials not supplied in config");
}
$oauthApiDetails = $authorization['oauth_api']['credentials'];
foreach (['#data', 'appKey', '#appSecret'] as $key) {
if (empty($oauthApiDetails[$key])) {
throw new UserException("Missing '{$key}' for OAuth 1.0 authorization");
}
}
$data = Utils::json_decode($oauthApiDetails['#data']);
$this->token = $data->oauth_token;
$this->tokenSecret = $data->oauth_token_secret;
$this->consumerKey = $oauthApiDetails['appKey'];
$this->consumerSecret = $oauthApiDetails['#appSecret'];
}
/**
* @param RestClient $client
*/
public function authenticateClient(RestClient $client)
{
$sub = new UrlSignature();
// Create array of objects instead of arrays from YML
$q = (array) Utils::arrayToObject($this->query);
$sub->setSignatureGenerator(function (array $requestInfo = []) use($q) {
$params = array_merge($requestInfo, ['attr' => $this->attrs]);
$query = [];
try {
foreach ($q as $key => $value) {
$query[$key] = is_scalar($value) ? $value : $this->builder->run($value, $params);
}
} catch (UserScriptException $e) {
throw new UserException("Error in query authentication script: " . $e->getMessage());
}
return $query;
});
$client->getClient()->getEmitter()->attach($sub);
}
public function __construct($authorization, array $api)
{
if (empty($authorization['oauth_api']['credentials'])) {
throw new UserException("OAuth API credentials not supplied in config");
}
$oauthApiDetails = $authorization['oauth_api']['credentials'];
foreach (['#data', 'appKey', '#appSecret'] as $key) {
if (empty($oauthApiDetails[$key])) {
throw new UserException("Missing '{$key}' for OAuth 2.0 authorization");
}
}
try {
$oAuthData = Utils::json_decode($oauthApiDetails['#data'], true);
} catch (JsonDecodeException $e) {
throw new UserException("The OAuth data is not a valid JSON");
}
$consumerData = ['client_id' => $oauthApiDetails['appKey'], 'client_secret' => $oauthApiDetails['#appSecret']];
$this->params = ['consumer' => $consumerData, 'user' => $oAuthData];
$this->auth = $api['authentication'];
}
/**
* {@inheritdoc}
*/
public function getNextRequest(ClientInterface $client, JobConfig $jobConfig, $response, $data)
{
$nextUrl = Utils::getDataFromPath($this->urlParam, $response, '.');
if (empty($nextUrl)) {
return false;
} else {
$config = $jobConfig->getConfig();
if (!$this->includeParams) {
$config['params'] = [];
}
if (!$this->paramIsQuery) {
$config['endpoint'] = $nextUrl;
} else {
// Create an array from the query string
$responseQuery = Query::fromString(ltrim($nextUrl, '?'))->toArray();
$config['params'] = array_replace($config['params'], $responseQuery);
}
return $client->createRequest($config);
}
}
/**
* @param string|int $limit
*/
public function setMemoryLimit($limit)
{
$this->memoryLimit = Utils::return_bytes($limit);
}
/**
* @param array|object $definitions
*/
protected function addGenerator($subscriber, $definitions, $authorization)
{
// Create array of objects instead of arrays from YML
$q = (array) Utils::arrayToObject($definitions);
$subscriber->setSignatureGenerator(function (array $requestInfo = []) use($q, $authorization) {
$params = array_merge($requestInfo, ['authorization' => $authorization]);
$result = [];
try {
foreach ($q as $key => $value) {
$result[$key] = is_scalar($value) ? $value : $this->builder->run($value, $params);
}
} catch (UserScriptException $e) {
throw new UserException("Error in OAuth authentication script: " . $e->getMessage());
}
return $result;
});
}
/**
* Analyze row of input data & create $this->struct
*
* @param mixed $row
* @param string $type
* @return void
*/
protected function analyzeRow($row, $type)
{
// Current row's structure
$struct = [];
$rowType = $this->getType($row);
// If the row is scalar, make it a {"data" => $value} object
if (is_scalar($row)) {
$struct[Parser::DATA_COLUMN] = $this->getType($row);
} elseif (is_object($row)) {
// process each property of the object
foreach ($row as $key => $field) {
$fieldType = $this->getType($field);
if ($fieldType == "object") {
// Only assign the type if the object isn't empty
if (Utils::isEmptyObject($field)) {
continue;
}
$this->analyzeRow($field, $type . "." . $key);
} elseif ($fieldType == "array") {
$arrayType = $this->analyze($field, $type . "." . $key);
if (false !== $arrayType) {
$fieldType = 'arrayOf' . $arrayType;
} else {
$fieldType = 'NULL';
}
}
$struct[$key] = $fieldType;
}
} elseif ($this->nestedArrayAsJson && is_array($row)) {
$this->log->log("WARNING", "Unsupported array nesting in '{$type}'! Converting to JSON string.", ['row' => $row]);
$rowType = $struct[Parser::DATA_COLUMN] = 'string';
} elseif (is_null($row)) {
// do nothing
} else {
throw new JsonParserException("Unsupported data row in '{$type}'!", ['row' => $row]);
}
$this->getStruct()->add($type, $struct);
return $rowType;
}
/**
* Compare a value from within an object
* using the $columnName, $operator and $value
* @param \stdClass $object
* @return bool
* @throws FilterException
* @api
*/
public function compareObject(\stdClass $object)
{
$value = Utils::getDataFromPath($this->columnName, $object, ".");
if (empty($this->multiFilter)) {
return $this->compare($value);
} else {
if ($this->multiOperator == "&") {
foreach ($this->multiFilter as $filter) {
if (!$filter->compareObject($object)) {
return false;
}
}
return true;
} elseif ($this->multiOperator == "|") {
foreach ($this->multiFilter as $filter) {
if ($filter->compareObject($object)) {
return true;
}
}
return false;
} else {
throw new FilterException("MultiFilter is set but MultiOperator is not recognized.");
}
}
}
/**
* @param object $response
* @return int
*/
protected function getExpiry(\stdclass $response)
{
if (!isset($this->auth['expires'])) {
return null;
} elseif (is_numeric($this->auth['expires'])) {
return time() + (int) $this->auth['expires'];
} elseif (is_array($this->auth['expires'])) {
if (empty($this->auth['expires']['response'])) {
throw new UserException("'authentication.expires' must be either an integer or an array with 'response' key containing a path in the response");
}
$rExpiry = Utils::getDataFromPath($this->auth['expires']['response'], $response, '.');
$expiry = is_int($rExpiry) ? $rExpiry : strtotime($rExpiry);
if (!empty($this->auth['expires']['relative'])) {
$expiry += time();
}
if ($expiry < time()) {
throw new UserException("Login authentication returned expiry time before current time: '{$rExpiry}'");
}
return $expiry;
}
}
/**
* @param string $field
* @param array $parentResults
* @param int $level
* @return mixed
*/
protected function getPlaceholderValue($field, $parentResults, $level, $placeholder)
{
try {
if (!array_key_exists($level, $parentResults)) {
$maxLevel = empty($parentResults) ? 0 : max(array_keys($parentResults)) + 1;
throw new UserException("Level " . ++$level . " not found in parent results! Maximum level: " . $maxLevel);
}
return Utils::getDataFromPath($field, $parentResults[$level], ".", false);
} catch (NoDataFoundException $e) {
throw new UserException("No value found for {$placeholder} in parent result. (level: " . ++$level . ")", 0, null, ['parents' => $parentResults]);
}
}
/**
* {@inheritdoc}
*/
public function getNextRequest(ClientInterface $client, JobConfig $jobConfig, $response, $data)
{
if (empty($data)) {
$this->reset();
return false;
} else {
$cursor = 0;
foreach ($data as $item) {
$cursorVal = Utils::getDataFromPath($this->idKey, $item, '.');
if (is_null($this->max) || $cursorVal > $this->max) {
$this->max = $cursorVal;
}
if (is_null($this->min) || $cursorVal < $this->min) {
$this->min = $cursorVal;
}
$cursor = $this->reverse ? $this->min : $this->max;
}
if (0 !== $this->increment) {
if (!is_numeric($cursor)) {
throw new UserException("Trying to increment a pointer that is not numeric.");
}
$cursor += $this->increment;
}
$jobConfig->setParam($this->param, $cursor);
return $client->createRequest($jobConfig->getConfig());
}
}
protected static function getRetryDelay($retries, AbstractTransferEvent $event, $headerName)
{
if (is_null($event->getResponse()) || !$event->getResponse()->hasHeader($headerName)) {
return RetrySubscriber::exponentialDelay($retries, $event);
}
$retryAfter = $event->getResponse()->getHeader($headerName);
if (is_numeric($retryAfter)) {
if ($retryAfter < time() - strtotime('1 day', 0)) {
return $retryAfter;
} else {
return $retryAfter - time();
}
}
if (Utils::isValidDateTimeString($retryAfter, DATE_RFC1123)) {
$date = \DateTime::createFromFormat(DATE_RFC1123, $retryAfter);
return $date->getTimestamp() - time();
}
return RetrySubscriber::exponentialDelay($retries, $event);
}
/**
* @expectedException \Keboola\Juicer\Exception\UserException
* @expectedExceptionMessage User script error: date() expects at least 1 parameter, 0 given
*/
public function testBuildParamsException()
{
$cfg = new JobConfig(1, ['params' => Utils::json_decode('{
"filters": {
"function": "date"
}
}')]);
$job = $this->getJob($cfg);
$job->setAttributes(['das.attribute' => "something interesting"]);
$job->setMetadata(['time' => ['previousStart' => 0, 'currentStart' => time()]]);
$job->setBuilder(new Builder());
self::callMethod($job, 'buildParams', [$cfg]);
}
/**
* Parse the data
* @param array|object $data shall be the response body
* @param string $type is a WSDL data type (has to be obtained from the WSDL definition)
* @param string $path a path to the results list(the array containing each record) within the response
* @param string $parent used internally for naming child arrays/columns
* @param string $parentId used internally to link child objects to parent
*/
public function parse($data, $type, $path = null, $parent = null, $parentId = null)
{
if (!empty($path)) {
$data = Utils::getDataFromPath($path, $data);
}
$fileName = $type;
if (empty($this->csvFiles[$fileName])) {
$header = array_keys($this->struct[$type]);
if ($parentId) {
array_push($header, "WSDL_parentId");
}
$this->csvFiles[$fileName] = Table::create($fileName, $header, $this->getTemp());
}
$handle = $this->csvFiles[$fileName];
$struct = $this->struct[$type];
foreach (Utils::to_assoc($data) as $record) {
$row = [];
foreach ($struct as $key => $valueType) {
if (empty($record[$key])) {
$row[$key] = null;
} elseif (in_array($valueType, $this->stdTypes)) {
$row[$key] = (string) $record[$key];
} elseif (array_key_exists($valueType, $this->struct)) {
// Walk through the data type and parse children
foreach ($this->struct[$valueType] as $attr => $attrType) {
$childId = $type . "_" . $attrType . "_" . (!empty($row["id"]) ? $row["id"] : uniqid());
$row[$key] = $childId;
$childPath = "{$key}/{$attr}";
$this->parse($record, $attrType, $childPath, $type, $childId);
}
} else {
$row[$key] = null;
}
}
// FIXME set this in the data before actually caling the fn
if ($parentId) {
$row["WSDL_parentId"] = $parentId;
}
$handle->writeRow($row);
}
}
public function testNoStrictScalarChange()
{
$parser = $this->getParser();
$data = Utils::json_decode('[
{"field": 128},
{"field": "string"},
{"field": true}
]');
$parser->process($data, 'threepack');
self::assertEquals(['"field"' . PHP_EOL, '"128"' . PHP_EOL, '"string"' . PHP_EOL, '"1"' . PHP_EOL], file($parser->getCsvFiles()['threepack']->getPathname()));
}
protected function loadJson($fileName)
{
$testFilesPath = $this->getDataDir() . $fileName . ".json";
$file = file_get_contents($testFilesPath);
return Utils::json_decode($file);
}