/**
* {@inheritdoc}
*/
public function getNextRequest(ClientInterface $client, JobConfig $jobConfig, $response, $data)
{
$nextUrl = Utils::getDataFromPath($this->urlParam, $response, '.');
if (empty($nextUrl)) {
return false;
}
// since validation - cannot be greater than now
$now = new \DateTime();
$sinceDateTime = \DateTime::createFromFormat('U', Url::fromString($nextUrl)->getQuery()->get('since'));
if ($sinceDateTime && $sinceDateTime > $now) {
return false;
}
$config = $jobConfig->getConfig();
if (!$this->includeParams) {
$config['params'] = [];
}
if (!$this->paramIsQuery) {
$config['endpoint'] = $nextUrl;
} else {
// Create an array from the query string
$responseQuery = Query::fromString(ltrim($nextUrl, '?'))->toArray();
$config['params'] = array_replace($config['params'], $responseQuery);
}
return $client->createRequest($config);
}
/**
* {@inheritdoc}
*/
public function getNextRequest(ClientInterface $client, JobConfig $jobConfig, $response, $data)
{
$nextUrl = Utils::getDataFromPath($this->urlParam, $response, '.');
if (empty($nextUrl)) {
return false;
}
// start_time validation
// https://developer.zendesk.com/rest_api/docs/core/incremental_export#incremental-ticket-export
$now = new \DateTime();
$startDateTime = \DateTime::createFromFormat('U', Url::fromString($nextUrl)->getQuery()->get('start_time'));
if ($startDateTime && $startDateTime > $now->modify(sprintf("-%d minutes", self::NEXT_PAGE_FILTER_MINUTES))) {
return false;
}
$config = $jobConfig->getConfig();
if (!$this->includeParams) {
$config['params'] = [];
}
if (!$this->paramIsQuery) {
$config['endpoint'] = $nextUrl;
} else {
// Create an array from the query string
$responseQuery = Query::fromString(ltrim($nextUrl, '?'))->toArray();
$config['params'] = array_replace($config['params'], $responseQuery);
}
return $client->createRequest($config);
}
/**
* @param $response
* @param JobConfig $jobConfig
* @return array
*/
public function process($response, JobConfig $jobConfig)
{
$config = $jobConfig->getConfig();
if (!isset($config['parser']['method'])) {
return $response;
}
$path = empty($config['dataField']) ? "." : $config['dataField'];
if ($config['parser']['method'] == 'facebook.insights') {
return $this->flatten($path, $response);
}
return $response;
}
/**
* Try to find the data array within $response.
*
* @param array|object $response
* @param array $config
* @return array
* @todo support array of dataFields
* - would return object with results, changing the class' API
* - parse would just have to loop through if it returns an object
* - and append $type with the dataField
* @deprecated Use response module
*/
public function process($response, JobConfig $jobConfig)
{
$config = $jobConfig->getConfig();
// If dataField doesn't say where the data is in a response, try to find it!
if (!empty($config['dataField'])) {
if (is_array($config['dataField'])) {
if (empty($config['dataField']['path'])) {
throw new UserException("'dataField.path' must be set!");
}
$path = $config['dataField']['path'];
} elseif (is_scalar($config['dataField'])) {
$path = $config['dataField'];
} else {
throw new UserException("'dataField' must be either a path string or an object with 'path' attribute.");
}
$data = Utils::getDataFromPath($path, $response, ".");
if (empty($data)) {
Logger::log('warning', "dataField '{$path}' contains no data!");
$data = [];
} elseif (!is_array($data)) {
// In case of a single object being returned
$data = [$data];
}
} elseif (is_array($response)) {
// Simplest case, the response is just the dataset
$data = $response;
} elseif (is_object($response)) {
// Find arrays in the response
$arrays = [];
foreach ($response as $key => $value) {
if (is_array($value)) {
$arrays[$key] = $value;
}
// TODO else {$this->metadata[$key] = json_encode($value);} ? return [$data,$metadata];
}
$arrayNames = array_keys($arrays);
if (count($arrays) == 1) {
$data = $arrays[$arrayNames[0]];
} elseif (count($arrays) == 0) {
Logger::log('warning', "No data array found in response! (endpoint: {$config['endpoint']})", ['response' => json_encode($response)]);
$data = [];
} else {
$e = new UserException("More than one array found in response! Use 'dataField' parameter to specify a key to the data array. (endpoint: {$config['endpoint']}, arrays in response root: " . join(", ", $arrayNames) . ")");
$e->setData(['response' => json_encode($response), 'arrays found' => $arrayNames]);
throw $e;
}
} else {
$e = new UserException('Unknown response from API.');
$e->setData(['response' => json_encode($response)]);
throw $e;
}
return $data;
}
/**
* @return array
*/
public function process($response, JobConfig $jobConfig)
{
if (empty($jobConfig->getConfig()['parseObject'])) {
return $response;
}
$config = $jobConfig->getConfig()['parseObject'];
if (!is_object($response)) {
if (empty($response)) {
return [];
}
throw new UserException("Data in response is not an object, while one was expected!");
}
$path = empty($config['path']) ? "." : $config['path'];
$key = empty($config['keyColumn']) ? "rowId" : $config['keyColumn'];
return $this->convertObjectWithKeys(Utils::getDataFromPath($path, $response, '.'), $key);
}
/**
* @expectedException \Keboola\Juicer\Exception\UserException
* @expectedExceptionMessage More than one array found in response! Use 'dataField' parameter to specify a key to the data array. (endpoint: a, arrays in response root: results, otherArray)
*/
public function testMultipleArraysException()
{
$cfg = JobConfig::create(['endpoint' => 'a']);
$module = new FindResponseArray();
$response = (object) ['results' => [(object) ['id' => 1], (object) ['id' => 2]], 'otherArray' => ['a', 'b']];
$data = $module->process($response, $cfg);
}
public function testMissingData()
{
$jobConfig = JobConfig::create(['endpoint' => 'ep', 'responseFilter' => 'out.arr[].in']);
$filter = Filter::create($jobConfig);
$data = [(object) ['id' => 1, 'out' => (object) ['arr' => [(object) ['in' => 'string'], (object) ['uh' => 'no "in" here!'], (object) ['in' => ['str', 'ing']]]]]];
self::assertEquals([(object) ['id' => 1, 'out' => (object) ['arr' => [(object) ['in' => 'string'], (object) ['uh' => 'no "in" here!'], (object) ['in' => '["str","ing"]']]]]], $filter->run($data));
}
/**
* {@inheritdoc}
*/
public function getNextRequest(ClientInterface $client, JobConfig $jobConfig, $response, $data)
{
if (empty($data)) {
$this->reset();
return false;
} else {
$cursor = 0;
foreach ($data as $item) {
$cursorVal = Utils::getDataFromPath($this->idKey, $item, '.');
if (is_null($this->max) || $cursorVal > $this->max) {
$this->max = $cursorVal;
}
if (is_null($this->min) || $cursorVal < $this->min) {
$this->min = $cursorVal;
}
$cursor = $this->reverse ? $this->min : $this->max;
}
if (0 !== $this->increment) {
if (!is_numeric($cursor)) {
throw new UserException("Trying to increment a pointer that is not numeric.");
}
$cursor += $this->increment;
}
$jobConfig->setParam($this->param, $cursor);
return $client->createRequest($jobConfig->getConfig());
}
}
/**
* @param JobConfig $jobConfig
* @return ScrollerInterface
*/
public function getScrollerForJob(JobConfig $jobConfig)
{
if (empty($jobConfig->getConfig()['scroller'])) {
if (empty($this->defaultScroller)) {
return new NoScroller();
}
if (!array_key_exists($this->defaultScroller, $this->scrollers)) {
throw new UserException("Default scroller '{$this->defaultScroller}' does not exist");
}
return $this->scrollers[$this->defaultScroller];
}
$scrollerId = $jobConfig->getConfig()['scroller'];
if (empty($this->scrollers[$scrollerId])) {
throw new UserException("Scroller '{$scrollerId}' not set in API definitions. Scrollers defined: " . join(', ', array_keys($this->scrollers)));
}
return $this->scrollers[$scrollerId];
}
public function testProcess()
{
$cfg = JobConfig::create(['endpoint' => 'a', 'parseObject' => ['keyColumn' => 'id', 'path' => 'results']]);
$module = new QualtricsPreParser();
$response = (object) ['results' => (object) [1 => 'first', 2 => 'second'], 'otherArray' => ['a', 'b']];
$data = $module->process($response, $cfg);
self::assertEquals([(object) ['data' => 'first', 'id' => 1], (object) ['data' => 'second', 'id' => 2]], $data);
}
/**
* {@inheritdoc}
*/
public function getNextRequest(ClientInterface $client, JobConfig $jobConfig, $response, $data)
{
$nextParam = Utils::getDataFromPath($this->responseParam, $response, '.');
if (empty($nextParam)) {
return false;
} else {
$config = $jobConfig->getConfig();
if (!$this->includeParams) {
$config['params'] = [];
}
if (!is_null($this->scrollRequest)) {
$config = $this->createScrollRequest($config, $this->scrollRequest);
}
$config['params'][$this->queryParam] = $nextParam;
return $client->createRequest($config);
}
}
public function testCreateRequest()
{
$arr = ['first' => 1, 'second' => 'two'];
$jobConfig = JobConfig::create(['endpoint' => 'ep', 'params' => $arr]);
$client = new RestClient(new Client());
$request = $client->createRequest($jobConfig->getConfig());
$expected = new RestRequest('ep', $arr);
self::assertEquals($expected, $request);
}
public function testGetConfig()
{
$configuration = new Configuration('./Tests/data/recursive', 'test', new Temp('test'));
$config = $configuration->getConfig();
$yml = Yaml::parse(file_get_contents('./Tests/data/recursive/config.yml'));
$jobs = $config->getJobs();
self::assertEquals(JobConfig::create($yml['parameters']['config']['jobs'][0]), reset($jobs));
self::assertEquals($yml['parameters']['config']['outputBucket'], $config->getAttribute('outputBucket'));
}
/**
* {@inheritdoc}
*/
public function getNextRequest(ClientInterface $client, JobConfig $jobConfig, $response, $data)
{
$nextUrl = Utils::getDataFromPath($this->urlParam, $response, '.');
if (empty($nextUrl)) {
return false;
} else {
$config = $jobConfig->getConfig();
if (!$this->includeParams) {
$config['params'] = [];
}
if (!$this->paramIsQuery) {
$config['endpoint'] = $nextUrl;
} else {
// Create an array from the query string
$responseQuery = Query::fromString(ltrim($nextUrl, '?'))->toArray();
$config['params'] = array_replace($config['params'], $responseQuery);
}
return $client->createRequest($config);
}
}
public function testProcessInsightsLifetime()
{
$cfg = JobConfig::create(['endpoint' => 'insights', 'dataField' => 'data', 'parser' => ['method' => 'facebook.insights']]);
$module = new \Keboola\ExGenericModule\FacebookPreParser();
$jsonResponse = <<<JSON
{
"data": [
{
"name": "page_fan_adds_unique",
"period": "lifetime",
"values": [
{
"value": 1
},
{
"value": 2
}
],
"title": "Daily New Likes",
"id": "177057932317550/insights/page_fan_adds_unique"
}
],
"paging": {
"previous": "prev",
"next": "next"
}
}
JSON;
$response = json_decode($jsonResponse);
$data = $module->process($response, $cfg);
$this->assertEquals((object) ['data' => [(object) ['name' => 'page_fan_adds_unique', 'period' => 'lifetime', 'values' => [(object) ['id' => '177057932317550/insights/page_fan_adds_unique', 'key1' => '', 'key2' => '', 'end_time' => '', 'value' => 1], (object) ['id' => '177057932317550/insights/page_fan_adds_unique', 'key1' => '', 'key2' => '', 'end_time' => '', 'value' => 2]], 'title' => 'Daily New Likes', 'id' => '177057932317550/insights/page_fan_adds_unique']], 'paging' => (object) ['previous' => 'prev', 'next' => 'next']], $data);
}
/**
* @param JobConfig $config
* @return array
*/
protected function buildParams(JobConfig $config)
{
$params = UserFunction::build($config->getParams(), ['attr' => $this->attributes, 'time' => $this->metadata['time']], $this->stringBuilder);
$config->setParams($params);
return $params;
}
/**
* @param JobConfig $config
*/
public static function create(JobConfig $config)
{
$filters = empty($config->getConfig()['responseFilter']) ? [] : (is_array($config->getConfig()['responseFilter']) ? $config->getConfig()['responseFilter'] : [$config->getConfig()['responseFilter']]);
$delimiter = empty($config->getConfig()['responseFilterDelimiter']) ? self::DEFAULT_DELIMITER : $config->getConfig()['responseFilterDelimiter'];
return new self($filters, $delimiter);
}
/**
* {@inheritdoc}
*/
public function getFirstRequest(ClientInterface $client, JobConfig $jobConfig)
{
return $client->createRequest($jobConfig->getConfig());
}
/**
* Create a child job with current client and parser
* @param JobConfig $config
* @return static
*/
protected function createChild(JobConfig $config, array $parentResults)
{
// Clone the config to prevent overwriting the placeholder(s) in endpoint
$job = new static(clone $config, $this->client, $this->parser);
$params = [];
$placeholders = !empty($config->getConfig()['placeholders']) ? $config->getConfig()['placeholders'] : [];
if (empty($placeholders)) {
Logger::log("WARNING", "No 'placeholders' set for '" . $config->getConfig()['endpoint'] . "'");
}
foreach ($placeholders as $placeholder => $field) {
$params[$placeholder] = $this->getPlaceholder($placeholder, $field, $parentResults);
}
// Add parent params as well (for 'tagging' child-parent data)
// Same placeholder in deeper nesting replaces parent value
if (!empty($this->parentParams)) {
$params = array_replace($this->parentParams, $params);
}
$job->setParams($params);
$job->setParentResults($parentResults);
return $job;
}
public function testMergeResults()
{
Logger::setLogger($this->getLogger('testMergeResults', true));
$configFirst = JobConfig::create(['endpoint' => '1st', 'dataType' => 'first']);
$configTags = JobConfig::create(['endpoint' => '2nd', 'dataType' => 'tags']);
$config = new Config('ex', 'test', []);
$config->setAttributes(['mappings' => ['first' => ['id' => ['type' => 'column', 'mapping' => ['destination' => 'item_id']], 'tags' => ['type' => 'table', 'destination' => 'tags', 'tableMapping' => ['user' => ['mapping' => ['destination' => 'user', 'primaryKey' => true]], 'tag' => ['mapping' => ['destination' => 'tag', 'primaryKey' => true]]], 'parentKey' => ['disable' => true]]], 'tags' => ['user' => ['mapping' => ['destination' => 'user', 'primaryKey' => true]], 'tag' => ['mapping' => ['destination' => 'tag', 'primaryKey' => true]]]]]);
$firstData = json_decode('[
{
"id": 1,
"arr": [1,2,3]
},
{
"id": 2,
"arr": ["a","b","c"],
"tags": [
{
"user": "******",
"tag": "tag1"
},
{
"user": "******",
"tag": "tag2"
}
]
}
]');
$secondData = json_decode('[
{
"user": "******",
"tag": "tag3"
},
{
"user": "******",
"tag": "tag4"
}
]');
$parser = JsonMap::create($config);
$parser->process($firstData, $configFirst->getDataType());
$parser->process($secondData, $configTags->getDataType());
self::assertEquals(['"user","tag"' . PHP_EOL, '"asd","tag1"' . PHP_EOL, '"asd","tag2"' . PHP_EOL, '"asd","tag3"' . PHP_EOL, '"asd","tag4"' . PHP_EOL], file($parser->getResults()['tags']));
}
/**
* @param JobConfig $jobConfig
* @return int
*/
protected function getLimit(JobConfig $jobConfig)
{
$params = $jobConfig->getParams();
return empty($params[$this->limitParam]) ? $this->limit : $params[$this->limitParam];
}
/**
* @param object $job
* @return JobConfig
*/
protected function createJob($job)
{
if (!is_array($job)) {
throw new UserException("Invalid format for job configuration.", 0, null, ['job' => $job]);
}
return JobConfig::create($job);
}
/**
* @return string
*/
protected function getDataType()
{
$config = $this->config->getConfig();
$type = !empty($config['dataType']) ? $config['dataType'] : $config['endpoint'];
return $type;
}
/**
* @param JobConfig $job
*/
public function addChildJob(self $job)
{
$this->childJobs[$job->getJobId()] = $job;
}
public function testGetDataTypeFromEndpoint()
{
$jobConfig = JobConfig::create(['endpoint' => 'resources/res.json']);
$job = $this->getMockForAbstractClass('Keboola\\Juicer\\Extractor\\Job', [$jobConfig, RestClient::create(), new Json(Parser::create($this->getLogger('job', true)))]);
$this->assertEquals($jobConfig->getEndpoint(), $this->callMethod($job, 'getDataType', []));
}