/**
* @param array $params Values to override in the config
* @return Config
* @todo separate the loading of YML and pass it as an argument
*/
public function getConfig(array $params = null)
{
try {
$configYml = $this->getYaml('/config.yml', 'parameters', 'config');
} catch (NoDataException $e) {
throw new UserException($e->getMessage(), 0, $e);
}
if (!is_null($params)) {
$configYml = array_replace($configYml, $params);
}
$configName = empty($configYml['id']) ? '' : $configYml['id'];
$runtimeParams = [];
// TODO get runtime params from console
if (empty($configYml['jobs'])) {
throw new UserException("No 'jobs' specified in the config!");
}
$jobs = $configYml['jobs'];
$jobConfigs = [];
foreach ($jobs as $job) {
$jobConfig = $this->createJob($job);
$jobConfigs[$jobConfig->getJobId()] = $jobConfig;
}
unset($configYml['jobs']);
// weird
$config = new Config($this->appName, $configName, $runtimeParams);
$config->setJobs($jobConfigs);
$config->setAttributes($configYml);
return $config;
}
public function testCreateAuthQuery()
{
$config = new Config('testApp', 'testCfg', []);
$config->setAttributes(['key' => 'val']);
$api = ['authentication' => ['type' => 'query', 'query' => ['param' => ['attr' => 'key']]]];
$queryAuth = Api::createAuth($api, $config, []);
self::assertEquals($api['authentication']['query'], self::getProperty($queryAuth, 'query'));
self::assertEquals($config->getAttributes(), self::getProperty($queryAuth, 'attrs'));
self::assertInstanceOf('\\Keboola\\GenericExtractor\\Authentication\\Query', $queryAuth);
}
/**
* @param Config $config
*/
public function loadConfig(Config $config)
{
$attrs = $config->getAttributes();
$configHeaders = empty($attrs['http']['headers']) ? [] : $attrs['http']['headers'];
if (!empty($this->requiredHeaders)) {
foreach ($this->requiredHeaders as $rHeader) {
if (empty($configHeaders[$rHeader])) {
throw new UserException("Missing required header {$rHeader} in config.http.headers!");
}
}
}
$this->configHeaders = $configHeaders;
}
/**
* @param Config $config // not used anywhere in real aps (yet? - analyze)
* @param Logger $logger
* @param Temp $temp
* @param array $metadata
* @return static
*/
public static function create(Config $config, Monolog $logger, Temp $temp, array $metadata = [])
{
// TODO move this if to $this->validateStruct altogether
if (!empty($metadata['json_parser.struct']) && is_array($metadata['json_parser.struct'])) {
if (empty($metadata['json_parser.structVersion']) || $metadata['json_parser.structVersion'] != Struct::STRUCT_VERSION) {
// temporary
$metadata['json_parser.struct'] = self::updateStruct($metadata['json_parser.struct']);
}
$struct = $metadata['json_parser.struct'];
} else {
$struct = [];
}
$rowsToAnalyze = null != $config && !empty($config->getRuntimeParams()["analyze"]) ? $config->getRuntimeParams()["analyze"] : -1;
$parser = JsonParser::create($logger, $struct, $rowsToAnalyze);
$parser->setTemp($temp);
return new static($parser);
}
/**
* @param Config $config
* @return static
*/
public static function create(Config $config, ParserInterface $fallbackParser = null)
{
if (empty($config->getAttribute('mappings'))) {
throw new UserException("Cannot initialize JSON Mapper with no mapping");
}
$mappers = [];
foreach ($config->getAttribute('mappings') as $type => $mapping) {
if (empty($mapping)) {
throw new UserException("Empty mapping for '{$type}' in config.");
}
$mappers[$type] = new Mapper($mapping, $type);
}
foreach ($config->getJobs() as $job) {
$type = $job->getDataType();
if (empty($mappers[$type])) {
if (is_null($fallbackParser)) {
throw new UserException("Missing mapping for '{$type}' in config.");
}
}
}
$parser = new static($mappers);
$parser->setFallbackParser($fallbackParser);
return $parser;
}
/**
* @param JobConfig $jobConfig
* @param RestClient $client
* @param Config $config
* @param Builder $builder
*/
protected function runJob($jobConfig, $client, $config, $builder)
{
// FIXME this is rather duplicated in RecursiveJob::createChild()
$job = new GenericExtractorJob($jobConfig, $client, $this->parser);
$job->setScroller($this->scroller);
$job->setAttributes($config->getAttributes());
$job->setMetadata($this->metadata);
$job->setBuilder($builder);
$job->setResponseModules($this->modules['response']);
if (!empty($config->getAttribute('userData'))) {
$job->setUserParentId(is_scalar($config->getAttribute('userData')) ? ['userData' => $config->getAttribute('userData')] : $config->getAttribute('userData'));
}
$job->run();
}
public function testMergeResults()
{
Logger::setLogger($this->getLogger('testMergeResults', true));
$configFirst = JobConfig::create(['endpoint' => '1st', 'dataType' => 'first']);
$configTags = JobConfig::create(['endpoint' => '2nd', 'dataType' => 'tags']);
$config = new Config('ex', 'test', []);
$config->setAttributes(['mappings' => ['first' => ['id' => ['type' => 'column', 'mapping' => ['destination' => 'item_id']], 'tags' => ['type' => 'table', 'destination' => 'tags', 'tableMapping' => ['user' => ['mapping' => ['destination' => 'user', 'primaryKey' => true]], 'tag' => ['mapping' => ['destination' => 'tag', 'primaryKey' => true]]], 'parentKey' => ['disable' => true]]], 'tags' => ['user' => ['mapping' => ['destination' => 'user', 'primaryKey' => true]], 'tag' => ['mapping' => ['destination' => 'tag', 'primaryKey' => true]]]]]);
$firstData = json_decode('[
{
"id": 1,
"arr": [1,2,3]
},
{
"id": 2,
"arr": ["a","b","c"],
"tags": [
{
"user": "******",
"tag": "tag1"
},
{
"user": "******",
"tag": "tag2"
}
]
}
]');
$secondData = json_decode('[
{
"user": "******",
"tag": "tag3"
},
{
"user": "******",
"tag": "tag4"
}
]');
$parser = JsonMap::create($config);
$parser->process($firstData, $configFirst->getDataType());
$parser->process($secondData, $configTags->getDataType());
self::assertEquals(['"user","tag"' . PHP_EOL, '"asd","tag1"' . PHP_EOL, '"asd","tag2"' . PHP_EOL, '"asd","tag3"' . PHP_EOL, '"asd","tag4"' . PHP_EOL], file($parser->getResults()['tags']));
}