/**
* No change to JSON parser structure should happen when nothing is parsed!
*/
public function testRunMetadataUpdate()
{
$logger = $this->getLogger('test', true);
Logger::setLogger($logger);
$meta = ['json_parser.struct' => ['tickets.via' => ['channel' => 'scalar', 'source' => 'object']], 'time' => ['previousStart' => 123]];
$cfg = new Config('testApp', 'testCfg', []);
$api = Api::create(['baseUrl' => 'http://example.com'], $cfg);
$ex = new GenericExtractor(new Temp());
$ex->setLogger($logger);
$ex->setApi($api);
$ex->setMetadata($meta);
$ex->run($cfg);
$after = $ex->getMetadata();
self::assertEquals($meta['json_parser.struct'], $after['json_parser.struct']);
self::assertArrayHasKey('time', $after);
}
public function run()
{
$temp = new Temp(APP_NAME);
Logger::initLogger(APP_NAME);
$arguments = getopt("d::", ["data::"]);
if (!isset($arguments["data"])) {
throw new UserException('Data folder not set.');
}
$configuration = new Configuration($arguments['data'], APP_NAME, $temp);
$configs = $configuration->getMultipleConfigs();
$metadata = $configuration->getConfigMetadata() ?: [];
$metadata['time']['previousStart'] = empty($metadata['time']['previousStart']) ? 0 : $metadata['time']['previousStart'];
$metadata['time']['currentStart'] = time();
$modules = $this->loadModules($configuration);
$authorization = $configuration->getAuthorization();
$cacheStorage = $this->initCacheStorage($configuration);
$results = [];
foreach ($configs as $config) {
// Reinitialize logger depending on debug status
if ($config->getAttribute('debug')) {
Logger::initLogger(APP_NAME, true);
} else {
Logger::initLogger(APP_NAME);
}
$api = $configuration->getApi($config, $authorization);
if (!empty($config->getAttribute('outputBucket'))) {
$outputBucket = $config->getAttribute('outputBucket');
} elseif (!empty($config->getConfigName())) {
$outputBucket = 'ex-api-' . $api->getName() . "-" . $config->getConfigName();
} else {
$outputBucket = "__kbc_default";
}
$extractor = new GenericExtractor($temp);
$extractor->setLogger(Logger::getLogger());
if ($cacheStorage) {
$extractor->enableCache($cacheStorage);
}
if (!empty($results[$outputBucket])) {
$extractor->setParser($results[$outputBucket]['parser']);
}
$extractor->setApi($api);
$extractor->setMetadata($metadata);
$extractor->setModules($modules);
$extractor->run($config);
$metadata = $extractor->getMetadata();
$results[$outputBucket]['parser'] = $extractor->getParser();
$results[$outputBucket]['incremental'] = $config->getAttribute('incrementalOutput');
}
foreach ($results as $bucket => $result) {
Logger::log('debug', "Processing results for {$bucket}.");
$configuration->storeResults($result['parser']->getResults(), $bucket == "__kbc_default" ? null : $bucket, true, $result['incremental']);
// move files and flatten file structure
$folderFinder = new Finder();
$fs = new Filesystem();
$folders = $folderFinder->directories()->in($arguments['data'] . "/out/tables")->depth(0);
foreach ($folders as $folder) {
//$files = $finder->files()->in($folder->getPathname())->depth(0);
$filesFinder = new Finder();
$files = $filesFinder->files()->in($folder->getPathname())->depth(0);
foreach ($files as $file) {
$destination = $arguments['data'] . "/out/tables/" . basename($folder->getPathname()) . "." . basename($file->getPathname());
// maybe move will be better?
$fs->copy($file->getPathname(), $destination);
$fs->remove($file);
}
}
$fs->remove($folders);
}
$metadata['time']['previousStart'] = $metadata['time']['currentStart'];
unset($metadata['time']['currentStart']);
$configuration->saveConfigMetadata($metadata);
}