public function run() { $temp = new Temp(APP_NAME); Logger::initLogger(APP_NAME); $arguments = getopt("d::", ["data::"]); if (!isset($arguments["data"])) { throw new UserException('Data folder not set.'); } $configuration = new Configuration($arguments['data'], APP_NAME, $temp); $configs = $configuration->getMultipleConfigs(); $metadata = $configuration->getConfigMetadata() ?: []; $metadata['time']['previousStart'] = empty($metadata['time']['previousStart']) ? 0 : $metadata['time']['previousStart']; $metadata['time']['currentStart'] = time(); $modules = $this->loadModules($configuration); $authorization = $configuration->getAuthorization(); $cacheStorage = $this->initCacheStorage($configuration); $results = []; foreach ($configs as $config) { // Reinitialize logger depending on debug status if ($config->getAttribute('debug')) { Logger::initLogger(APP_NAME, true); } else { Logger::initLogger(APP_NAME); } $api = $configuration->getApi($config, $authorization); if (!empty($config->getAttribute('outputBucket'))) { $outputBucket = $config->getAttribute('outputBucket'); } elseif (!empty($config->getConfigName())) { $outputBucket = 'ex-api-' . $api->getName() . "-" . $config->getConfigName(); } else { $outputBucket = "__kbc_default"; } $extractor = new GenericExtractor($temp); $extractor->setLogger(Logger::getLogger()); if ($cacheStorage) { $extractor->enableCache($cacheStorage); } if (!empty($results[$outputBucket])) { $extractor->setParser($results[$outputBucket]['parser']); } $extractor->setApi($api); $extractor->setMetadata($metadata); $extractor->setModules($modules); $extractor->run($config); $metadata = $extractor->getMetadata(); $results[$outputBucket]['parser'] = $extractor->getParser(); $results[$outputBucket]['incremental'] = $config->getAttribute('incrementalOutput'); } foreach ($results as $bucket => $result) { Logger::log('debug', "Processing results for {$bucket}."); $configuration->storeResults($result['parser']->getResults(), $bucket == "__kbc_default" ? null : $bucket, true, $result['incremental']); // move files and flatten file structure $folderFinder = new Finder(); $fs = new Filesystem(); $folders = $folderFinder->directories()->in($arguments['data'] . "/out/tables")->depth(0); foreach ($folders as $folder) { //$files = $finder->files()->in($folder->getPathname())->depth(0); $filesFinder = new Finder(); $files = $filesFinder->files()->in($folder->getPathname())->depth(0); foreach ($files as $file) { $destination = $arguments['data'] . "/out/tables/" . basename($folder->getPathname()) . "." . basename($file->getPathname()); // maybe move will be better? $fs->copy($file->getPathname(), $destination); $fs->remove($file); } } $fs->remove($folders); } $metadata['time']['previousStart'] = $metadata['time']['currentStart']; unset($metadata['time']['currentStart']); $configuration->saveConfigMetadata($metadata); }
<?php use Keboola\Juicer\Common\Logger; use Keboola\Juicer\Exception\ApplicationException; use Keboola\Juicer\Exception\UserException; use Keboola\MetricsWriter\Application; use Symfony\Component\Yaml\Yaml; require_once dirname(__FILE__) . "/bootstrap.php"; Logger::initLogger(APP_NAME); try { $arguments = getopt("d::", ["data::"]); if (!isset($arguments["data"])) { throw new UserException('Data folder not set.'); } $config = Yaml::parse(file_get_contents($arguments["data"] . "/config.yml")); $config['dataFolder'] = $arguments['data']; $app = new Application($config); $app->run(); } catch (UserException $e) { Logger::log('error', $e->getMessage(), (array) $e->getData()); exit(1); } catch (ApplicationException $e) { Logger::log('error', $e->getMessage(), (array) $e->getData()); exit($e->getCode() > 1 ? $e->getCode() : 2); } catch (\Exception $e) { Logger::log('error', $e->getMessage(), ['errFile' => $e->getFile(), 'errLine' => $e->getLine(), 'trace' => $e->getTrace()]); exit(2); } Logger::log('info', "Writer finished successfully."); exit(0);
public function setUp() { Logger::initLogger('ex-generic_test'); }