コード例 #1
0
ファイル: Configuration.php プロジェクト: keboola/juicer
 /**
  * @param array $params Values to override in the config
  * @return Config
  * @todo separate the loading of YML and pass it as an argument
  */
 public function getConfig(array $params = null)
 {
     try {
         $configYml = $this->getYaml('/config.yml', 'parameters', 'config');
     } catch (NoDataException $e) {
         throw new UserException($e->getMessage(), 0, $e);
     }
     if (!is_null($params)) {
         $configYml = array_replace($configYml, $params);
     }
     $configName = empty($configYml['id']) ? '' : $configYml['id'];
     $runtimeParams = [];
     // TODO get runtime params from console
     if (empty($configYml['jobs'])) {
         throw new UserException("No 'jobs' specified in the config!");
     }
     $jobs = $configYml['jobs'];
     $jobConfigs = [];
     foreach ($jobs as $job) {
         $jobConfig = $this->createJob($job);
         $jobConfigs[$jobConfig->getJobId()] = $jobConfig;
     }
     unset($configYml['jobs']);
     // weird
     $config = new Config($this->appName, $configName, $runtimeParams);
     $config->setJobs($jobConfigs);
     $config->setAttributes($configYml);
     return $config;
 }
コード例 #2
0
ファイル: ApiTest.php プロジェクト: keboola/generic-extractor
 public function testCreateAuthQuery()
 {
     $config = new Config('testApp', 'testCfg', []);
     $config->setAttributes(['key' => 'val']);
     $api = ['authentication' => ['type' => 'query', 'query' => ['param' => ['attr' => 'key']]]];
     $queryAuth = Api::createAuth($api, $config, []);
     self::assertEquals($api['authentication']['query'], self::getProperty($queryAuth, 'query'));
     self::assertEquals($config->getAttributes(), self::getProperty($queryAuth, 'attrs'));
     self::assertInstanceOf('\\Keboola\\GenericExtractor\\Authentication\\Query', $queryAuth);
 }
コード例 #3
0
ファイル: Headers.php プロジェクト: keboola/generic-extractor
 /**
  * @param Config $config
  */
 public function loadConfig(Config $config)
 {
     $attrs = $config->getAttributes();
     $configHeaders = empty($attrs['http']['headers']) ? [] : $attrs['http']['headers'];
     if (!empty($this->requiredHeaders)) {
         foreach ($this->requiredHeaders as $rHeader) {
             if (empty($configHeaders[$rHeader])) {
                 throw new UserException("Missing required header {$rHeader} in config.http.headers!");
             }
         }
     }
     $this->configHeaders = $configHeaders;
 }
コード例 #4
0
ファイル: Json.php プロジェクト: keboola/juicer
 /**
  * @param Config $config // not used anywhere in real aps (yet? - analyze)
  * @param Logger $logger
  * @param Temp $temp
  * @param array $metadata
  * @return static
  */
 public static function create(Config $config, Monolog $logger, Temp $temp, array $metadata = [])
 {
     // TODO move this if to $this->validateStruct altogether
     if (!empty($metadata['json_parser.struct']) && is_array($metadata['json_parser.struct'])) {
         if (empty($metadata['json_parser.structVersion']) || $metadata['json_parser.structVersion'] != Struct::STRUCT_VERSION) {
             // temporary
             $metadata['json_parser.struct'] = self::updateStruct($metadata['json_parser.struct']);
         }
         $struct = $metadata['json_parser.struct'];
     } else {
         $struct = [];
     }
     $rowsToAnalyze = null != $config && !empty($config->getRuntimeParams()["analyze"]) ? $config->getRuntimeParams()["analyze"] : -1;
     $parser = JsonParser::create($logger, $struct, $rowsToAnalyze);
     $parser->setTemp($temp);
     return new static($parser);
 }
コード例 #5
0
ファイル: JsonMap.php プロジェクト: keboola/juicer
 /**
  * @param Config $config
  * @return static
  */
 public static function create(Config $config, ParserInterface $fallbackParser = null)
 {
     if (empty($config->getAttribute('mappings'))) {
         throw new UserException("Cannot initialize JSON Mapper with no mapping");
     }
     $mappers = [];
     foreach ($config->getAttribute('mappings') as $type => $mapping) {
         if (empty($mapping)) {
             throw new UserException("Empty mapping for '{$type}' in config.");
         }
         $mappers[$type] = new Mapper($mapping, $type);
     }
     foreach ($config->getJobs() as $job) {
         $type = $job->getDataType();
         if (empty($mappers[$type])) {
             if (is_null($fallbackParser)) {
                 throw new UserException("Missing mapping for '{$type}' in config.");
             }
         }
     }
     $parser = new static($mappers);
     $parser->setFallbackParser($fallbackParser);
     return $parser;
 }
コード例 #6
0
 /**
  * @param JobConfig $jobConfig
  * @param RestClient $client
  * @param Config $config
  * @param Builder $builder
  */
 protected function runJob($jobConfig, $client, $config, $builder)
 {
     // FIXME this is rather duplicated in RecursiveJob::createChild()
     $job = new GenericExtractorJob($jobConfig, $client, $this->parser);
     $job->setScroller($this->scroller);
     $job->setAttributes($config->getAttributes());
     $job->setMetadata($this->metadata);
     $job->setBuilder($builder);
     $job->setResponseModules($this->modules['response']);
     if (!empty($config->getAttribute('userData'))) {
         $job->setUserParentId(is_scalar($config->getAttribute('userData')) ? ['userData' => $config->getAttribute('userData')] : $config->getAttribute('userData'));
     }
     $job->run();
 }
コード例 #7
0
ファイル: JsonMapTest.php プロジェクト: keboola/juicer
 public function testMergeResults()
 {
     Logger::setLogger($this->getLogger('testMergeResults', true));
     $configFirst = JobConfig::create(['endpoint' => '1st', 'dataType' => 'first']);
     $configTags = JobConfig::create(['endpoint' => '2nd', 'dataType' => 'tags']);
     $config = new Config('ex', 'test', []);
     $config->setAttributes(['mappings' => ['first' => ['id' => ['type' => 'column', 'mapping' => ['destination' => 'item_id']], 'tags' => ['type' => 'table', 'destination' => 'tags', 'tableMapping' => ['user' => ['mapping' => ['destination' => 'user', 'primaryKey' => true]], 'tag' => ['mapping' => ['destination' => 'tag', 'primaryKey' => true]]], 'parentKey' => ['disable' => true]]], 'tags' => ['user' => ['mapping' => ['destination' => 'user', 'primaryKey' => true]], 'tag' => ['mapping' => ['destination' => 'tag', 'primaryKey' => true]]]]]);
     $firstData = json_decode('[
         {
             "id": 1,
             "arr": [1,2,3]
         },
         {
             "id": 2,
             "arr": ["a","b","c"],
             "tags": [
                 {
                     "user": "******",
                     "tag": "tag1"
                 },
                 {
                     "user": "******",
                     "tag": "tag2"
                 }
             ]
         }
     ]');
     $secondData = json_decode('[
         {
             "user": "******",
             "tag": "tag3"
         },
         {
             "user": "******",
             "tag": "tag4"
         }
     ]');
     $parser = JsonMap::create($config);
     $parser->process($firstData, $configFirst->getDataType());
     $parser->process($secondData, $configTags->getDataType());
     self::assertEquals(['"user","tag"' . PHP_EOL, '"asd","tag1"' . PHP_EOL, '"asd","tag2"' . PHP_EOL, '"asd","tag3"' . PHP_EOL, '"asd","tag4"' . PHP_EOL], file($parser->getResults()['tags']));
 }