public function run()
{
$temp = new Temp(APP_NAME);
Logger::initLogger(APP_NAME);
$arguments = getopt("d::", ["data::"]);
if (!isset($arguments["data"])) {
throw new UserException('Data folder not set.');
}
$configuration = new Configuration($arguments['data'], APP_NAME, $temp);
$configs = $configuration->getMultipleConfigs();
$metadata = $configuration->getConfigMetadata() ?: [];
$metadata['time']['previousStart'] = empty($metadata['time']['previousStart']) ? 0 : $metadata['time']['previousStart'];
$metadata['time']['currentStart'] = time();
$modules = $this->loadModules($configuration);
$authorization = $configuration->getAuthorization();
$cacheStorage = $this->initCacheStorage($configuration);
$results = [];
foreach ($configs as $config) {
// Reinitialize logger depending on debug status
if ($config->getAttribute('debug')) {
Logger::initLogger(APP_NAME, true);
} else {
Logger::initLogger(APP_NAME);
}
$api = $configuration->getApi($config, $authorization);
if (!empty($config->getAttribute('outputBucket'))) {
$outputBucket = $config->getAttribute('outputBucket');
} elseif (!empty($config->getConfigName())) {
$outputBucket = 'ex-api-' . $api->getName() . "-" . $config->getConfigName();
} else {
$outputBucket = "__kbc_default";
}
$extractor = new GenericExtractor($temp);
$extractor->setLogger(Logger::getLogger());
if ($cacheStorage) {
$extractor->enableCache($cacheStorage);
}
if (!empty($results[$outputBucket])) {
$extractor->setParser($results[$outputBucket]['parser']);
}
$extractor->setApi($api);
$extractor->setMetadata($metadata);
$extractor->setModules($modules);
$extractor->run($config);
$metadata = $extractor->getMetadata();
$results[$outputBucket]['parser'] = $extractor->getParser();
$results[$outputBucket]['incremental'] = $config->getAttribute('incrementalOutput');
}
foreach ($results as $bucket => $result) {
Logger::log('debug', "Processing results for {$bucket}.");
$configuration->storeResults($result['parser']->getResults(), $bucket == "__kbc_default" ? null : $bucket, true, $result['incremental']);
// move files and flatten file structure
$folderFinder = new Finder();
$fs = new Filesystem();
$folders = $folderFinder->directories()->in($arguments['data'] . "/out/tables")->depth(0);
foreach ($folders as $folder) {
//$files = $finder->files()->in($folder->getPathname())->depth(0);
$filesFinder = new Finder();
$files = $filesFinder->files()->in($folder->getPathname())->depth(0);
foreach ($files as $file) {
$destination = $arguments['data'] . "/out/tables/" . basename($folder->getPathname()) . "." . basename($file->getPathname());
// maybe move will be better?
$fs->copy($file->getPathname(), $destination);
$fs->remove($file);
}
}
$fs->remove($folders);
}
$metadata['time']['previousStart'] = $metadata['time']['currentStart'];
unset($metadata['time']['currentStart']);
$configuration->saveConfigMetadata($metadata);
}
<?php
use Keboola\Juicer\Common\Logger;
use Keboola\Juicer\Exception\ApplicationException;
use Keboola\Juicer\Exception\UserException;
use Keboola\MetricsWriter\Application;
use Symfony\Component\Yaml\Yaml;
require_once dirname(__FILE__) . "/bootstrap.php";
Logger::initLogger(APP_NAME);
try {
$arguments = getopt("d::", ["data::"]);
if (!isset($arguments["data"])) {
throw new UserException('Data folder not set.');
}
$config = Yaml::parse(file_get_contents($arguments["data"] . "/config.yml"));
$config['dataFolder'] = $arguments['data'];
$app = new Application($config);
$app->run();
} catch (UserException $e) {
Logger::log('error', $e->getMessage(), (array) $e->getData());
exit(1);
} catch (ApplicationException $e) {
Logger::log('error', $e->getMessage(), (array) $e->getData());
exit($e->getCode() > 1 ? $e->getCode() : 2);
} catch (\Exception $e) {
Logger::log('error', $e->getMessage(), ['errFile' => $e->getFile(), 'errLine' => $e->getLine(), 'trace' => $e->getTrace()]);
exit(2);
}
Logger::log('info', "Writer finished successfully.");
exit(0);
public function setUp()
{
Logger::initLogger('ex-generic_test');
}
