/**
* {@inheritdoc}
*/
public function getNextRequest(ClientInterface $client, JobConfig $jobConfig, $response, $data)
{
$nextUrl = Utils::getDataFromPath($this->urlParam, $response, '.');
if (empty($nextUrl)) {
return false;
}
// since validation - cannot be greater than now
$now = new \DateTime();
$sinceDateTime = \DateTime::createFromFormat('U', Url::fromString($nextUrl)->getQuery()->get('since'));
if ($sinceDateTime && $sinceDateTime > $now) {
return false;
}
$config = $jobConfig->getConfig();
if (!$this->includeParams) {
$config['params'] = [];
}
if (!$this->paramIsQuery) {
$config['endpoint'] = $nextUrl;
} else {
// Create an array from the query string
$responseQuery = Query::fromString(ltrim($nextUrl, '?'))->toArray();
$config['params'] = array_replace($config['params'], $responseQuery);
}
return $client->createRequest($config);
}
/**
* {@inheritdoc}
*/
public function getNextRequest(ClientInterface $client, JobConfig $jobConfig, $response, $data)
{
$nextUrl = Utils::getDataFromPath($this->urlParam, $response, '.');
if (empty($nextUrl)) {
return false;
}
// start_time validation
// https://developer.zendesk.com/rest_api/docs/core/incremental_export#incremental-ticket-export
$now = new \DateTime();
$startDateTime = \DateTime::createFromFormat('U', Url::fromString($nextUrl)->getQuery()->get('start_time'));
if ($startDateTime && $startDateTime > $now->modify(sprintf("-%d minutes", self::NEXT_PAGE_FILTER_MINUTES))) {
return false;
}
$config = $jobConfig->getConfig();
if (!$this->includeParams) {
$config['params'] = [];
}
if (!$this->paramIsQuery) {
$config['endpoint'] = $nextUrl;
} else {
// Create an array from the query string
$responseQuery = Query::fromString(ltrim($nextUrl, '?'))->toArray();
$config['params'] = array_replace($config['params'], $responseQuery);
}
return $client->createRequest($config);
}
/**
* Try to find the data array within $response.
*
* @param array|object $response
* @param array $config
* @return array
* @todo support array of dataFields
* - would return object with results, changing the class' API
* - parse would just have to loop through if it returns an object
* - and append $type with the dataField
* @deprecated Use response module
*/
public function process($response, JobConfig $jobConfig)
{
$config = $jobConfig->getConfig();
// If dataField doesn't say where the data is in a response, try to find it!
if (!empty($config['dataField'])) {
if (is_array($config['dataField'])) {
if (empty($config['dataField']['path'])) {
throw new UserException("'dataField.path' must be set!");
}
$path = $config['dataField']['path'];
} elseif (is_scalar($config['dataField'])) {
$path = $config['dataField'];
} else {
throw new UserException("'dataField' must be either a path string or an object with 'path' attribute.");
}
$data = Utils::getDataFromPath($path, $response, ".");
if (empty($data)) {
Logger::log('warning', "dataField '{$path}' contains no data!");
$data = [];
} elseif (!is_array($data)) {
// In case of a single object being returned
$data = [$data];
}
} elseif (is_array($response)) {
// Simplest case, the response is just the dataset
$data = $response;
} elseif (is_object($response)) {
// Find arrays in the response
$arrays = [];
foreach ($response as $key => $value) {
if (is_array($value)) {
$arrays[$key] = $value;
}
// TODO else {$this->metadata[$key] = json_encode($value);} ? return [$data,$metadata];
}
$arrayNames = array_keys($arrays);
if (count($arrays) == 1) {
$data = $arrays[$arrayNames[0]];
} elseif (count($arrays) == 0) {
Logger::log('warning', "No data array found in response! (endpoint: {$config['endpoint']})", ['response' => json_encode($response)]);
$data = [];
} else {
$e = new UserException("More than one array found in response! Use 'dataField' parameter to specify a key to the data array. (endpoint: {$config['endpoint']}, arrays in response root: " . join(", ", $arrayNames) . ")");
$e->setData(['response' => json_encode($response), 'arrays found' => $arrayNames]);
throw $e;
}
} else {
$e = new UserException('Unknown response from API.');
$e->setData(['response' => json_encode($response)]);
throw $e;
}
return $data;
}
/**
* @return array
*/
public function process($response, JobConfig $jobConfig)
{
if (empty($jobConfig->getConfig()['parseObject'])) {
return $response;
}
$config = $jobConfig->getConfig()['parseObject'];
if (!is_object($response)) {
if (empty($response)) {
return [];
}
throw new UserException("Data in response is not an object, while one was expected!");
}
$path = empty($config['path']) ? "." : $config['path'];
$key = empty($config['keyColumn']) ? "rowId" : $config['keyColumn'];
return $this->convertObjectWithKeys(Utils::getDataFromPath($path, $response, '.'), $key);
}
/**
*
* Creates key value pairs for `values` property up to 2 leves of nesting
*
* @param $path
* @param $data
* @return array
* @throws \Keboola\Utils\Exception\NoDataFoundException
*/
protected function flatten($path, $data)
{
foreach (Utils::getDataFromPath($path, $data, '.') as $metric) {
if ($metric->values && is_array($metric->values)) {
$parsedMetrics = [];
foreach ($metric->values as $value) {
$end_time = '';
if (isset($value->end_time)) {
$end_time = $value->end_time;
}
if (!isset($value->end_time) && $metric->period != 'lifetime') {
continue;
}
// scalar value or empty value
if (!isset($value->value) || is_scalar($value->value)) {
if (!isset($value->value)) {
$val = 0;
} else {
$val = $value->value;
}
$parsedMetrics[] = (object) ["id" => $metric->id, "key1" => "", "key2" => "", "end_time" => $end_time, "value" => $val];
continue;
}
if (is_object($value->value)) {
foreach ((array) $value->value as $key1 => $value1) {
if (is_object($value1)) {
foreach ((array) $value1 as $key2 => $value2) {
$parsedMetrics[] = (object) ["id" => $metric->id, "key1" => $key1, "key2" => $key2, "end_time" => $end_time, "value" => $value2];
}
} else {
$parsedMetrics[] = (object) ["id" => $metric->id, "key1" => $key1, "key2" => "", "end_time" => $end_time, "value" => $value1];
}
}
continue;
}
}
$metric->values = $parsedMetrics;
}
$result[] = $metric;
}
if ($path != '.') {
$data->{$path} = $result;
} else {
$data = $result;
}
return $data;
}
/**
* {@inheritdoc}
*/
public function getNextRequest(ClientInterface $client, JobConfig $jobConfig, $response, $data)
{
$nextParam = Utils::getDataFromPath($this->responseParam, $response, '.');
if (empty($nextParam)) {
return false;
} else {
$config = $jobConfig->getConfig();
if (!$this->includeParams) {
$config['params'] = [];
}
if (!is_null($this->scrollRequest)) {
$config = $this->createScrollRequest($config, $this->scrollRequest);
}
$config['params'][$this->queryParam] = $nextParam;
return $client->createRequest($config);
}
}
/**
* {@inheritdoc}
*/
public function getNextRequest(ClientInterface $client, JobConfig $jobConfig, $response, $data)
{
$nextUrl = Utils::getDataFromPath($this->urlParam, $response, '.');
if (empty($nextUrl)) {
return false;
} else {
$config = $jobConfig->getConfig();
if (!$this->includeParams) {
$config['params'] = [];
}
if (!$this->paramIsQuery) {
$config['endpoint'] = $nextUrl;
} else {
// Create an array from the query string
$responseQuery = Query::fromString(ltrim($nextUrl, '?'))->toArray();
$config['params'] = array_replace($config['params'], $responseQuery);
}
return $client->createRequest($config);
}
}
/**
* Compare a value from within an object
* using the $columnName, $operator and $value
* @param \stdClass $object
* @return bool
* @throws FilterException
* @api
*/
public function compareObject(\stdClass $object)
{
$value = Utils::getDataFromPath($this->columnName, $object, ".");
if (empty($this->multiFilter)) {
return $this->compare($value);
} else {
if ($this->multiOperator == "&") {
foreach ($this->multiFilter as $filter) {
if (!$filter->compareObject($object)) {
return false;
}
}
return true;
} elseif ($this->multiOperator == "|") {
foreach ($this->multiFilter as $filter) {
if ($filter->compareObject($object)) {
return true;
}
}
return false;
} else {
throw new FilterException("MultiFilter is set but MultiOperator is not recognized.");
}
}
}
/**
* @param object $response
* @return int
*/
protected function getExpiry(\stdclass $response)
{
if (!isset($this->auth['expires'])) {
return null;
} elseif (is_numeric($this->auth['expires'])) {
return time() + (int) $this->auth['expires'];
} elseif (is_array($this->auth['expires'])) {
if (empty($this->auth['expires']['response'])) {
throw new UserException("'authentication.expires' must be either an integer or an array with 'response' key containing a path in the response");
}
$rExpiry = Utils::getDataFromPath($this->auth['expires']['response'], $response, '.');
$expiry = is_int($rExpiry) ? $rExpiry : strtotime($rExpiry);
if (!empty($this->auth['expires']['relative'])) {
$expiry += time();
}
if ($expiry < time()) {
throw new UserException("Login authentication returned expiry time before current time: '{$rExpiry}'");
}
return $expiry;
}
}
/**
* @param string $field
* @param array $parentResults
* @param int $level
* @return mixed
*/
protected function getPlaceholderValue($field, $parentResults, $level, $placeholder)
{
try {
if (!array_key_exists($level, $parentResults)) {
$maxLevel = empty($parentResults) ? 0 : max(array_keys($parentResults)) + 1;
throw new UserException("Level " . ++$level . " not found in parent results! Maximum level: " . $maxLevel);
}
return Utils::getDataFromPath($field, $parentResults[$level], ".", false);
} catch (NoDataFoundException $e) {
throw new UserException("No value found for {$placeholder} in parent result. (level: " . ++$level . ")", 0, null, ['parents' => $parentResults]);
}
}
/**
* Parse the data
* @param array|object $data shall be the response body
* @param string $type is a WSDL data type (has to be obtained from the WSDL definition)
* @param string $path a path to the results list(the array containing each record) within the response
* @param string $parent used internally for naming child arrays/columns
* @param string $parentId used internally to link child objects to parent
*/
public function parse($data, $type, $path = null, $parent = null, $parentId = null)
{
if (!empty($path)) {
$data = Utils::getDataFromPath($path, $data);
}
$fileName = $type;
if (empty($this->csvFiles[$fileName])) {
$header = array_keys($this->struct[$type]);
if ($parentId) {
array_push($header, "WSDL_parentId");
}
$this->csvFiles[$fileName] = Table::create($fileName, $header, $this->getTemp());
}
$handle = $this->csvFiles[$fileName];
$struct = $this->struct[$type];
foreach (Utils::to_assoc($data) as $record) {
$row = [];
foreach ($struct as $key => $valueType) {
if (empty($record[$key])) {
$row[$key] = null;
} elseif (in_array($valueType, $this->stdTypes)) {
$row[$key] = (string) $record[$key];
} elseif (array_key_exists($valueType, $this->struct)) {
// Walk through the data type and parse children
foreach ($this->struct[$valueType] as $attr => $attrType) {
$childId = $type . "_" . $attrType . "_" . (!empty($row["id"]) ? $row["id"] : uniqid());
$row[$key] = $childId;
$childPath = "{$key}/{$attr}";
$this->parse($record, $attrType, $childPath, $type, $childId);
}
} else {
$row[$key] = null;
}
}
// FIXME set this in the data before actually caling the fn
if ($parentId) {
$row["WSDL_parentId"] = $parentId;
}
$handle->writeRow($row);
}
}
/**
* {@inheritdoc}
*/
public function getNextRequest(ClientInterface $client, JobConfig $jobConfig, $response, $data)
{
if (empty($data)) {
$this->reset();
return false;
} else {
$cursor = 0;
foreach ($data as $item) {
$cursorVal = Utils::getDataFromPath($this->idKey, $item, '.');
if (is_null($this->max) || $cursorVal > $this->max) {
$this->max = $cursorVal;
}
if (is_null($this->min) || $cursorVal < $this->min) {
$this->min = $cursorVal;
}
$cursor = $this->reverse ? $this->min : $this->max;
}
if (0 !== $this->increment) {
if (!is_numeric($cursor)) {
throw new UserException("Trying to increment a pointer that is not numeric.");
}
$cursor += $this->increment;
}
$jobConfig->setParam($this->param, $cursor);
return $client->createRequest($jobConfig->getConfig());
}
}
