public function run(Config $config) { $client = RestClient::create(['base_url' => $this->baseUrl, 'defaults' => ['headers' => UserFunction::build($this->headers, ['attr' => $config->getAttributes()])]], JuicerRest::convertRetry($this->retryConfig)); if (!empty($this->defaultRequestOptions)) { $client->setDefaultRequestOptions($this->defaultRequestOptions); } $this->auth->authenticateClient($client); // Verbose Logging of all requests $client->getClient()->getEmitter()->attach(new LogRequest()); if ($this->cache) { CacheSubscriber::attach($client->getClient(), ['storage' => $this->cache, 'validate' => false, 'can_cache' => function (RequestInterface $requestInterface) { return true; }]); } $this->initParser($config); $builder = new Builder(); foreach ($config->getJobs() as $jobConfig) { $this->runJob($jobConfig, $client, $config, $builder); } if ($this->parser instanceof Json) { // FIXME fallback from JsonMap $this->metadata = array_replace_recursive($this->metadata, $this->parser->getMetadata()); } // return $this->parser->getResults(); }
/** * @expectedException \Keboola\Juicer\Exception\UserException * @expectedExceptionMessage Scroller 'nonExistentScroller' not set in API definitions. Scrollers defined: param, cursor, page */ public function testUndefinedScrollerException() { $config = $this->getScrollerConfig(); $scroller = new MultipleScroller($config); $noScrollerConfig = new JobConfig('none', ['endpoint' => 'data', 'scroller' => 'nonExistentScroller']); $req = $scroller->getFirstRequest(RestClient::create(), $noScrollerConfig); }
public function testGetFirstRequest() { $client = RestClient::create(); $config = $this->getConfig(); $scroller = new ResponseParamScroller(['responseParam' => '_scroll_id', 'queryParam' => 'scroll_id', 'includeParams' => false, 'scrollRequest' => ['endpoint' => '_search/scroll', 'method' => 'POST', 'params' => ['scroll' => '1m']]]); $expected = $client->createRequest($config->getConfig()); self::assertEquals($expected, $scroller->getFirstRequest($client, $config)); }
public function testGetFirstRequest() { $client = RestClient::create(); $config = new JobConfig('test', ['endpoint' => 'test', 'params' => ['a' => 1, 'b' => 2]]); $scroller = new NoScroller(); $req = $scroller->getFirstRequest($client, $config); $expected = $client->createRequest($config->getConfig()); self::assertEquals($expected, $req); }
public function testGetNextRequestQueryParams() { $client = RestClient::create(); $config = $this->getConfig(); $response = (object) ['data' => [], 'scroll' => '?page=2&b=v']; $scroller = new ZendeskResponseUrlScroller(['urlKey' => 'scroll', 'paramIsQuery' => true, 'includeParams' => true]); $nextRequest = $scroller->getNextRequest($client, $config, $response, $response->data); $expected = $client->createRequest(['endpoint' => 'test', 'params' => ['page' => 2, 'a' => 1, 'b' => 'v']]); self::assertEquals($expected, $nextRequest); }
public function testGetNextRequestNested() { $client = RestClient::create(); $config = new JobConfig('test', ['endpoint' => 'test']); $scroller = new CursorScroller(['idKey' => 'id.int', 'param' => 'since_id']); $response = [(object) ['id' => (object) ['int' => 3]]]; $next = $scroller->getNextRequest($client, $config, $response, $response); $expected = $client->createRequest(['endpoint' => 'test', 'params' => ['since_id' => 3]]); self::assertEquals($expected, $next); }
public function testGetNextRequestHasMore() { $client = RestClient::create(); $jobConfig = new JobConfig('test', ['endpoint' => 'test']); $config = ['nextPageFlag' => ['field' => 'hasMore', 'stopOn' => false]]; $scroller = new OffsetScroller(['limit' => 10]); $decorated = new HasMoreScrollerDecorator($scroller, $config); self::assertInstanceOf('Keboola\\Juicer\\Pagination\\OffsetScroller', $decorated->getScroller()); $next = $decorated->getNextRequest($client, $jobConfig, (object) ['hasMore' => true], array_fill(0, 10, ['k' => 'v'])); self::assertInstanceOf('Keboola\\Juicer\\Client\\RestRequest', $next); $noNext = $decorated->getNextRequest($client, $jobConfig, (object) ['hasMore' => false], array_fill(0, 10, ['k' => 'v'])); self::assertFalse($noNext); }
public function testOffsetFromJob() { $client = RestClient::create(); $config = new JobConfig('test', ['endpoint' => 'test', 'params' => ['startAt' => 3]]); $limit = 10; $scroller = new OffsetScroller(['limit' => $limit, 'offsetFromJob' => true, 'offsetParam' => 'startAt']); $first = $scroller->getFirstRequest($client, $config); self::assertEquals($config->getParams()['startAt'], $first->getParams()['startAt']); $response = new \stdClass(); $response->data = array_fill(0, 10, (object) ['key' => 'value']); $second = $scroller->getNextRequest($client, $config, $response, $response->data); self::assertEquals($config->getParams()['startAt'] + $limit, $second->getParams()['startAt']); }
public function testTimeLimit() { $client = RestClient::create(); $jobConfig = new JobConfig('test', ['endpoint' => 'test']); $scroller = new PageScroller([]); $decorator = new ForceStopScrollerDecorator($scroller, ['forceStop' => ['time' => 3]]); $response = ['a']; $i = 0; while ($request = $decorator->getNextRequest($client, $jobConfig, [$response], $response)) { self::assertInstanceOf('Keboola\\Juicer\\Client\\RestRequest', $request); $i++; sleep(1); } self::assertFalse($decorator->getNextRequest($client, $jobConfig, $response, $response)); // Assert 3 pages were true self::assertEquals(3, $i); }
public function testGetNextRequestQueryParams() { $pagingNext = (new DateTime())->modify('-30 days'); $client = RestClient::create(); $config = $this->getConfig(); $response = (object) ['data' => [], 'scroll' => '?b=v&since=' . $pagingNext->getTimestamp()]; $scroller = new FacebookResponseUrlScroller(['urlKey' => 'scroll', 'paramIsQuery' => true, 'includeParams' => true]); $nextRequest = $scroller->getNextRequest($client, $config, $response, $response->data); $expected = $client->createRequest(['endpoint' => 'test', 'params' => ['since' => $pagingNext->getTimestamp(), 'a' => 1, 'b' => 'v']]); self::assertEquals($expected, $nextRequest); }
/** * I'm not too sure this is optimal! * If it looks stupid, but works, it ain't stupid! */ public function getJob($dir = 'recursive') { $temp = new Temp('recursion'); $configuration = new Configuration(__DIR__ . '/../data/' . $dir, 'test', $temp); $jobConfig = array_values($configuration->getConfig()->getJobs())[0]; $parser = Json::create($configuration->getConfig(), $this->getLogger('test', true), $temp); $client = RestClient::create(); $history = new History(); $client->getClient()->getEmitter()->attach($history); $job = $this->getMockForAbstractClass('Keboola\\Juicer\\Extractor\\RecursiveJob', [$jobConfig, $client, $parser]); return [$job, $client, $parser, $history, $jobConfig]; }
/** * @param JobConfig $config * @return GenericExtractorJob */ protected function getJob(JobConfig $config) { return new GenericExtractorJob($config, RestClient::create(['base_url' => 'http://example.com/api/']), Json::create(new Config('ex-generic-test', 'test', []), $this->getLogger(), new Temp())); }
public function testGetDataTypeFromEndpoint() { $jobConfig = JobConfig::create(['endpoint' => 'resources/res.json']); $job = $this->getMockForAbstractClass('Keboola\\Juicer\\Extractor\\Job', [$jobConfig, RestClient::create(), new Json(Parser::create($this->getLogger('job', true)))]); $this->assertEquals($jobConfig->getEndpoint(), $this->callMethod($job, 'getDataType', [])); }
public function testSetState() { $client = RestClient::create(); $config = new JobConfig('test', ['endpoint' => 'test']); $scroller = new PageScroller(['pageParam' => 'p']); $first = $scroller->getFirstRequest($client, $config); $second = $scroller->getNextRequest($client, $config, new \stdClass(), ['item']); $state = $scroller->getState(); $newScroller = new PageScroller([]); $newScroller->setState($state); $third = $newScroller->getNextRequest($client, $config, new \stdClass(), ['item']); self::assertEquals(3, $third->getParams()['p']); }
public function testDefaultRequestOptions() { $defaultOptions = ['method' => 'POST', 'params' => ['defA' => 'defValA', 'defB' => 'defValB']]; $client = RestClient::create(); $client->setDefaultRequestOptions($defaultOptions); $requestOptions = ['endpoint' => 'ep', 'params' => ['defB' => 'overrideB']]; $request = $client->createRequest($requestOptions); self::assertEquals($defaultOptions['method'], $request->getMethod()); self::assertEquals($requestOptions['endpoint'], $request->getEndpoint()); self::assertEquals(array_replace($defaultOptions['params'], $requestOptions['params']), $request->getParams()); }