/**
* @expectedException \Keboola\Juicer\Exception\UserException
* @expectedExceptionMessage Scroller 'nonExistentScroller' not set in API definitions. Scrollers defined: param, cursor, page
*/
public function testUndefinedScrollerException()
{
$config = $this->getScrollerConfig();
$scroller = new MultipleScroller($config);
$noScrollerConfig = new JobConfig('none', ['endpoint' => 'data', 'scroller' => 'nonExistentScroller']);
$req = $scroller->getFirstRequest(RestClient::create(), $noScrollerConfig);
}
public function run(Config $config)
{
$client = RestClient::create(['base_url' => $this->baseUrl, 'defaults' => ['headers' => UserFunction::build($this->headers, ['attr' => $config->getAttributes()])]], JuicerRest::convertRetry($this->retryConfig));
if (!empty($this->defaultRequestOptions)) {
$client->setDefaultRequestOptions($this->defaultRequestOptions);
}
$this->auth->authenticateClient($client);
// Verbose Logging of all requests
$client->getClient()->getEmitter()->attach(new LogRequest());
if ($this->cache) {
CacheSubscriber::attach($client->getClient(), ['storage' => $this->cache, 'validate' => false, 'can_cache' => function (RequestInterface $requestInterface) {
return true;
}]);
}
$this->initParser($config);
$builder = new Builder();
foreach ($config->getJobs() as $jobConfig) {
$this->runJob($jobConfig, $client, $config, $builder);
}
if ($this->parser instanceof Json) {
// FIXME fallback from JsonMap
$this->metadata = array_replace_recursive($this->metadata, $this->parser->getMetadata());
}
// return $this->parser->getResults();
}
public function testGetFirstRequest()
{
$client = RestClient::create();
$config = $this->getConfig();
$scroller = new ResponseParamScroller(['responseParam' => '_scroll_id', 'queryParam' => 'scroll_id', 'includeParams' => false, 'scrollRequest' => ['endpoint' => '_search/scroll', 'method' => 'POST', 'params' => ['scroll' => '1m']]]);
$expected = $client->createRequest($config->getConfig());
self::assertEquals($expected, $scroller->getFirstRequest($client, $config));
}
public function authenticateClient(RestClient $client)
{
if (empty($this->auth['loginRequest'])) {
throw new UserException("'loginRequest' is not configured for Login authentication");
}
$loginRequest = $this->getAuthRequest($this->auth['loginRequest']);
$sub = new LoginSubscriber();
// @return [query, headers]
$sub->setLoginMethod(function () use($client, $loginRequest, $sub) {
// Need to bypass the subscriber for the login call
$client->getClient()->getEmitter()->detach($sub);
$response = $client->download($loginRequest);
$client->getClient()->getEmitter()->attach($sub);
return ['query' => $this->getResults($response, 'query'), 'headers' => $this->getResults($response, 'headers'), 'expires' => $this->getExpiry($response)];
});
$client->getClient()->getEmitter()->attach($sub);
}
public function testGetFirstRequest()
{
$client = RestClient::create();
$config = new JobConfig('test', ['endpoint' => 'test', 'params' => ['a' => 1, 'b' => 2]]);
$scroller = new NoScroller();
$req = $scroller->getFirstRequest($client, $config);
$expected = $client->createRequest($config->getConfig());
self::assertEquals($expected, $req);
}
public function testGetNextRequestNested()
{
$client = RestClient::create();
$config = new JobConfig('test', ['endpoint' => 'test']);
$scroller = new CursorScroller(['idKey' => 'id.int', 'param' => 'since_id']);
$response = [(object) ['id' => (object) ['int' => 3]]];
$next = $scroller->getNextRequest($client, $config, $response, $response);
$expected = $client->createRequest(['endpoint' => 'test', 'params' => ['since_id' => 3]]);
self::assertEquals($expected, $next);
}
public function testGetNextRequestQueryParams()
{
$client = RestClient::create();
$config = $this->getConfig();
$response = (object) ['data' => [], 'scroll' => '?page=2&b=v'];
$scroller = new ZendeskResponseUrlScroller(['urlKey' => 'scroll', 'paramIsQuery' => true, 'includeParams' => true]);
$nextRequest = $scroller->getNextRequest($client, $config, $response, $response->data);
$expected = $client->createRequest(['endpoint' => 'test', 'params' => ['page' => 2, 'a' => 1, 'b' => 'v']]);
self::assertEquals($expected, $nextRequest);
}
/**
* @param RestClient $client
*/
public function authenticateClient(RestClient $client)
{
$sub = new UrlSignature();
// Create array of objects instead of arrays from YML
$q = (array) Utils::arrayToObject($this->query);
$sub->setSignatureGenerator(function (array $requestInfo = []) use($q) {
$params = array_merge($requestInfo, ['attr' => $this->attrs]);
$query = [];
try {
foreach ($q as $key => $value) {
$query[$key] = is_scalar($value) ? $value : $this->builder->run($value, $params);
}
} catch (UserScriptException $e) {
throw new UserException("Error in query authentication script: " . $e->getMessage());
}
return $query;
});
$client->getClient()->getEmitter()->attach($sub);
}
public function testGetNextRequestHasMore()
{
$client = RestClient::create();
$jobConfig = new JobConfig('test', ['endpoint' => 'test']);
$config = ['nextPageFlag' => ['field' => 'hasMore', 'stopOn' => false]];
$scroller = new OffsetScroller(['limit' => 10]);
$decorated = new HasMoreScrollerDecorator($scroller, $config);
self::assertInstanceOf('Keboola\\Juicer\\Pagination\\OffsetScroller', $decorated->getScroller());
$next = $decorated->getNextRequest($client, $jobConfig, (object) ['hasMore' => true], array_fill(0, 10, ['k' => 'v']));
self::assertInstanceOf('Keboola\\Juicer\\Client\\RestRequest', $next);
$noNext = $decorated->getNextRequest($client, $jobConfig, (object) ['hasMore' => false], array_fill(0, 10, ['k' => 'v']));
self::assertFalse($noNext);
}
public function testOffsetFromJob()
{
$client = RestClient::create();
$config = new JobConfig('test', ['endpoint' => 'test', 'params' => ['startAt' => 3]]);
$limit = 10;
$scroller = new OffsetScroller(['limit' => $limit, 'offsetFromJob' => true, 'offsetParam' => 'startAt']);
$first = $scroller->getFirstRequest($client, $config);
self::assertEquals($config->getParams()['startAt'], $first->getParams()['startAt']);
$response = new \stdClass();
$response->data = array_fill(0, 10, (object) ['key' => 'value']);
$second = $scroller->getNextRequest($client, $config, $response, $response->data);
self::assertEquals($config->getParams()['startAt'] + $limit, $second->getParams()['startAt']);
}
public function testTimeLimit()
{
$client = RestClient::create();
$jobConfig = new JobConfig('test', ['endpoint' => 'test']);
$scroller = new PageScroller([]);
$decorator = new ForceStopScrollerDecorator($scroller, ['forceStop' => ['time' => 3]]);
$response = ['a'];
$i = 0;
while ($request = $decorator->getNextRequest($client, $jobConfig, [$response], $response)) {
self::assertInstanceOf('Keboola\\Juicer\\Client\\RestRequest', $request);
$i++;
sleep(1);
}
self::assertFalse($decorator->getNextRequest($client, $jobConfig, $response, $response));
// Assert 3 pages were true
self::assertEquals(3, $i);
}
public function testDefaultRequestOptions()
{
$defaultOptions = ['method' => 'POST', 'params' => ['defA' => 'defValA', 'defB' => 'defValB']];
$client = RestClient::create();
$client->setDefaultRequestOptions($defaultOptions);
$requestOptions = ['endpoint' => 'ep', 'params' => ['defB' => 'overrideB']];
$request = $client->createRequest($requestOptions);
self::assertEquals($defaultOptions['method'], $request->getMethod());
self::assertEquals($requestOptions['endpoint'], $request->getEndpoint());
self::assertEquals(array_replace($defaultOptions['params'], $requestOptions['params']), $request->getParams());
}
public function testGetNextRequestQueryParams()
{
$pagingNext = (new DateTime())->modify('-30 days');
$client = RestClient::create();
$config = $this->getConfig();
$response = (object) ['data' => [], 'scroll' => '?b=v&since=' . $pagingNext->getTimestamp()];
$scroller = new FacebookResponseUrlScroller(['urlKey' => 'scroll', 'paramIsQuery' => true, 'includeParams' => true]);
$nextRequest = $scroller->getNextRequest($client, $config, $response, $response->data);
$expected = $client->createRequest(['endpoint' => 'test', 'params' => ['since' => $pagingNext->getTimestamp(), 'a' => 1, 'b' => 'v']]);
self::assertEquals($expected, $nextRequest);
}
/**
* @param RestClient $client
* @todo Add a possibility to add the option before each request, to allow refresh/signature here?
*/
public function authenticateClient(RestClient $client)
{
$client->getClient()->setDefaultOption('auth', [$this->username, $this->password]);
}
/**
* @param RestClient $client
*/
public function authenticateClient(RestClient $client)
{
$sub = new Oauth1(['consumer_key' => $this->consumerKey, 'consumer_secret' => $this->consumerSecret, 'token' => $this->token, 'token_secret' => $this->tokenSecret]);
$client->getClient()->getEmitter()->attach($sub);
$client->getClient()->setDefaultOption('auth', 'oauth');
}
/**
* @param RestClient $client
*/
public function authenticateClient(RestClient $client)
{
$subscribers = [['subscriber' => new UrlSignature(), 'definitions' => $this->query], ['subscriber' => new HeaderSignature(), 'definitions' => $this->headers]];
$authorization = ['clientId' => $this->clientId, 'nonce' => substr(sha1(uniqid(microtime(), true)), 0, 16), 'timestamp' => time()];
if (!is_scalar($this->data)) {
$authorization = array_merge($authorization, Utils::flattenArray(Utils::objectToArray($this->data), 'data.'));
} else {
$authorization['data'] = $this->data;
}
foreach ($subscribers as $subscriber) {
if (empty($subscriber['definitions'])) {
continue;
}
$this->addGenerator($subscriber['subscriber'], $subscriber['definitions'], $authorization);
$client->getClient()->getEmitter()->attach($subscriber['subscriber']);
}
}
public function testSetState()
{
$client = RestClient::create();
$config = new JobConfig('test', ['endpoint' => 'test']);
$scroller = new PageScroller(['pageParam' => 'p']);
$first = $scroller->getFirstRequest($client, $config);
$second = $scroller->getNextRequest($client, $config, new \stdClass(), ['item']);
$state = $scroller->getState();
$newScroller = new PageScroller([]);
$newScroller->setState($state);
$third = $newScroller->getNextRequest($client, $config, new \stdClass(), ['item']);
self::assertEquals(3, $third->getParams()['p']);
}
public function testGetDataTypeFromEndpoint()
{
$jobConfig = JobConfig::create(['endpoint' => 'resources/res.json']);
$job = $this->getMockForAbstractClass('Keboola\\Juicer\\Extractor\\Job', [$jobConfig, RestClient::create(), new Json(Parser::create($this->getLogger('job', true)))]);
$this->assertEquals($jobConfig->getEndpoint(), $this->callMethod($job, 'getDataType', []));
}
/**
* I'm not too sure this is optimal!
* If it looks stupid, but works, it ain't stupid!
*/
public function getJob($dir = 'recursive')
{
$temp = new Temp('recursion');
$configuration = new Configuration(__DIR__ . '/../data/' . $dir, 'test', $temp);
$jobConfig = array_values($configuration->getConfig()->getJobs())[0];
$parser = Json::create($configuration->getConfig(), $this->getLogger('test', true), $temp);
$client = RestClient::create();
$history = new History();
$client->getClient()->getEmitter()->attach($history);
$job = $this->getMockForAbstractClass('Keboola\\Juicer\\Extractor\\RecursiveJob', [$jobConfig, $client, $parser]);
return [$job, $client, $parser, $history, $jobConfig];
}
/**
* @param JobConfig $config
* @return GenericExtractorJob
*/
protected function getJob(JobConfig $config)
{
return new GenericExtractorJob($config, RestClient::create(['base_url' => 'http://example.com/api/']), Json::create(new Config('ex-generic-test', 'test', []), $this->getLogger(), new Temp()));
}