/**
* {@inheritdoc}
*/
public function getNextRequest(ClientInterface $client, JobConfig $jobConfig, $response, $data)
{
$nextUrl = Utils::getDataFromPath($this->urlParam, $response, '.');
if (empty($nextUrl)) {
return false;
}
// since validation - cannot be greater than now
$now = new \DateTime();
$sinceDateTime = \DateTime::createFromFormat('U', Url::fromString($nextUrl)->getQuery()->get('since'));
if ($sinceDateTime && $sinceDateTime > $now) {
return false;
}
$config = $jobConfig->getConfig();
if (!$this->includeParams) {
$config['params'] = [];
}
if (!$this->paramIsQuery) {
$config['endpoint'] = $nextUrl;
} else {
// Create an array from the query string
$responseQuery = Query::fromString(ltrim($nextUrl, '?'))->toArray();
$config['params'] = array_replace($config['params'], $responseQuery);
}
return $client->createRequest($config);
}
/**
* {@inheritdoc}
*/
public function getNextRequest(ClientInterface $client, JobConfig $jobConfig, $response, $data)
{
$nextUrl = Utils::getDataFromPath($this->urlParam, $response, '.');
if (empty($nextUrl)) {
return false;
}
// start_time validation
// https://developer.zendesk.com/rest_api/docs/core/incremental_export#incremental-ticket-export
$now = new \DateTime();
$startDateTime = \DateTime::createFromFormat('U', Url::fromString($nextUrl)->getQuery()->get('start_time'));
if ($startDateTime && $startDateTime > $now->modify(sprintf("-%d minutes", self::NEXT_PAGE_FILTER_MINUTES))) {
return false;
}
$config = $jobConfig->getConfig();
if (!$this->includeParams) {
$config['params'] = [];
}
if (!$this->paramIsQuery) {
$config['endpoint'] = $nextUrl;
} else {
// Create an array from the query string
$responseQuery = Query::fromString(ltrim($nextUrl, '?'))->toArray();
$config['params'] = array_replace($config['params'], $responseQuery);
}
return $client->createRequest($config);
}
/**
* @return array
*/
public function process($response, JobConfig $jobConfig)
{
if (empty($jobConfig->getConfig()['parseObject'])) {
return $response;
}
$config = $jobConfig->getConfig()['parseObject'];
if (!is_object($response)) {
if (empty($response)) {
return [];
}
throw new UserException("Data in response is not an object, while one was expected!");
}
$path = empty($config['path']) ? "." : $config['path'];
$key = empty($config['keyColumn']) ? "rowId" : $config['keyColumn'];
return $this->convertObjectWithKeys(Utils::getDataFromPath($path, $response, '.'), $key);
}
/**
* @param JobConfig $jobConfig
* @return ScrollerInterface
*/
public function getScrollerForJob(JobConfig $jobConfig)
{
if (empty($jobConfig->getConfig()['scroller'])) {
if (empty($this->defaultScroller)) {
return new NoScroller();
}
if (!array_key_exists($this->defaultScroller, $this->scrollers)) {
throw new UserException("Default scroller '{$this->defaultScroller}' does not exist");
}
return $this->scrollers[$this->defaultScroller];
}
$scrollerId = $jobConfig->getConfig()['scroller'];
if (empty($this->scrollers[$scrollerId])) {
throw new UserException("Scroller '{$scrollerId}' not set in API definitions. Scrollers defined: " . join(', ', array_keys($this->scrollers)));
}
return $this->scrollers[$scrollerId];
}
/**
* Try to find the data array within $response.
*
* @param array|object $response
* @param array $config
* @return array
* @todo support array of dataFields
* - would return object with results, changing the class' API
* - parse would just have to loop through if it returns an object
* - and append $type with the dataField
* @deprecated Use response module
*/
public function process($response, JobConfig $jobConfig)
{
$config = $jobConfig->getConfig();
// If dataField doesn't say where the data is in a response, try to find it!
if (!empty($config['dataField'])) {
if (is_array($config['dataField'])) {
if (empty($config['dataField']['path'])) {
throw new UserException("'dataField.path' must be set!");
}
$path = $config['dataField']['path'];
} elseif (is_scalar($config['dataField'])) {
$path = $config['dataField'];
} else {
throw new UserException("'dataField' must be either a path string or an object with 'path' attribute.");
}
$data = Utils::getDataFromPath($path, $response, ".");
if (empty($data)) {
Logger::log('warning', "dataField '{$path}' contains no data!");
$data = [];
} elseif (!is_array($data)) {
// In case of a single object being returned
$data = [$data];
}
} elseif (is_array($response)) {
// Simplest case, the response is just the dataset
$data = $response;
} elseif (is_object($response)) {
// Find arrays in the response
$arrays = [];
foreach ($response as $key => $value) {
if (is_array($value)) {
$arrays[$key] = $value;
}
// TODO else {$this->metadata[$key] = json_encode($value);} ? return [$data,$metadata];
}
$arrayNames = array_keys($arrays);
if (count($arrays) == 1) {
$data = $arrays[$arrayNames[0]];
} elseif (count($arrays) == 0) {
Logger::log('warning', "No data array found in response! (endpoint: {$config['endpoint']})", ['response' => json_encode($response)]);
$data = [];
} else {
$e = new UserException("More than one array found in response! Use 'dataField' parameter to specify a key to the data array. (endpoint: {$config['endpoint']}, arrays in response root: " . join(", ", $arrayNames) . ")");
$e->setData(['response' => json_encode($response), 'arrays found' => $arrayNames]);
throw $e;
}
} else {
$e = new UserException('Unknown response from API.');
$e->setData(['response' => json_encode($response)]);
throw $e;
}
return $data;
}
/**
* @param $response
* @param JobConfig $jobConfig
* @return array
*/
public function process($response, JobConfig $jobConfig)
{
$config = $jobConfig->getConfig();
if (!isset($config['parser']['method'])) {
return $response;
}
$path = empty($config['dataField']) ? "." : $config['dataField'];
if ($config['parser']['method'] == 'facebook.insights') {
return $this->flatten($path, $response);
}
return $response;
}
/**
* {@inheritdoc}
*/
public function getNextRequest(ClientInterface $client, JobConfig $jobConfig, $response, $data)
{
$nextParam = Utils::getDataFromPath($this->responseParam, $response, '.');
if (empty($nextParam)) {
return false;
} else {
$config = $jobConfig->getConfig();
if (!$this->includeParams) {
$config['params'] = [];
}
if (!is_null($this->scrollRequest)) {
$config = $this->createScrollRequest($config, $this->scrollRequest);
}
$config['params'][$this->queryParam] = $nextParam;
return $client->createRequest($config);
}
}
/**
* {@inheritdoc}
*/
public function getNextRequest(ClientInterface $client, JobConfig $jobConfig, $response, $data)
{
$nextUrl = Utils::getDataFromPath($this->urlParam, $response, '.');
if (empty($nextUrl)) {
return false;
} else {
$config = $jobConfig->getConfig();
if (!$this->includeParams) {
$config['params'] = [];
}
if (!$this->paramIsQuery) {
$config['endpoint'] = $nextUrl;
} else {
// Create an array from the query string
$responseQuery = Query::fromString(ltrim($nextUrl, '?'))->toArray();
$config['params'] = array_replace($config['params'], $responseQuery);
}
return $client->createRequest($config);
}
}
/**
* {@inheritdoc}
*/
public function getFirstRequest(ClientInterface $client, JobConfig $jobConfig)
{
return $client->createRequest($jobConfig->getConfig());
}
/**
* Create a child job with current client and parser
* @param JobConfig $config
* @return static
*/
protected function createChild(JobConfig $config, array $parentResults)
{
// Clone the config to prevent overwriting the placeholder(s) in endpoint
$job = new static(clone $config, $this->client, $this->parser);
$params = [];
$placeholders = !empty($config->getConfig()['placeholders']) ? $config->getConfig()['placeholders'] : [];
if (empty($placeholders)) {
Logger::log("WARNING", "No 'placeholders' set for '" . $config->getConfig()['endpoint'] . "'");
}
foreach ($placeholders as $placeholder => $field) {
$params[$placeholder] = $this->getPlaceholder($placeholder, $field, $parentResults);
}
// Add parent params as well (for 'tagging' child-parent data)
// Same placeholder in deeper nesting replaces parent value
if (!empty($this->parentParams)) {
$params = array_replace($this->parentParams, $params);
}
$job->setParams($params);
$job->setParentResults($parentResults);
return $job;
}
/**
* {@inheritdoc}
*/
public function getNextRequest(ClientInterface $client, JobConfig $jobConfig, $response, $data)
{
if (empty($data)) {
$this->reset();
return false;
} else {
$cursor = 0;
foreach ($data as $item) {
$cursorVal = Utils::getDataFromPath($this->idKey, $item, '.');
if (is_null($this->max) || $cursorVal > $this->max) {
$this->max = $cursorVal;
}
if (is_null($this->min) || $cursorVal < $this->min) {
$this->min = $cursorVal;
}
$cursor = $this->reverse ? $this->min : $this->max;
}
if (0 !== $this->increment) {
if (!is_numeric($cursor)) {
throw new UserException("Trying to increment a pointer that is not numeric.");
}
$cursor += $this->increment;
}
$jobConfig->setParam($this->param, $cursor);
return $client->createRequest($jobConfig->getConfig());
}
}
/**
* Returns a config with scroller params
* @param JobConfig $jobConfig
* @return array
*/
protected function getParams(JobConfig $jobConfig)
{
$config = $jobConfig->getConfig();
$scrollParams = [$this->limitParam => $this->getLimit($jobConfig), $this->offsetParam => $this->pointer];
$config['params'] = array_replace($jobConfig->getParams(), $scrollParams);
return $config;
}
/**
* @param JobConfig $config
*/
public static function create(JobConfig $config)
{
$filters = empty($config->getConfig()['responseFilter']) ? [] : (is_array($config->getConfig()['responseFilter']) ? $config->getConfig()['responseFilter'] : [$config->getConfig()['responseFilter']]);
$delimiter = empty($config->getConfig()['responseFilterDelimiter']) ? self::DEFAULT_DELIMITER : $config->getConfig()['responseFilterDelimiter'];
return new self($filters, $delimiter);
}
/**
* @return string
*/
protected function getDataType()
{
$config = $this->config->getConfig();
$type = !empty($config['dataType']) ? $config['dataType'] : $config['endpoint'];
return $type;
}
