/**
* Load files from a given directory.
* @param $data_folder string | The folder name.
* @return array
*/
public function load($data_folder = '')
{
$loader = new DataLoader();
$this->data = $loader->load($data_folder, '/config.json');
if (!$this->validator->validate($this->data)) {
throw new \Exception('Config.json fail.');
}
}
/**
* Load files from a given directory.
* @param $data_folder string | The folder name.
* @return array
*/
public function load($data_folder = '', $pages_data = '')
{
$this->data_path = '/' . $pages_data;
$files = $this->read_dir($data_folder . $this->data_path);
$pattern = '/(.+).json$/';
$data_loader = new DataLoader();
foreach ($files as $file) {
if (preg_match($pattern, $file, $match)) {
$f = $data_loader->load($data_folder . $this->data_path, $file);
$f['file'] = $file;
$this->files[] = $f;
}
}
}
$db->exec('ALTER TABLE curve DROP CONSTRAINT curve_datasetid_fkey');
$db->exec('ALTER TABLE curve DROP CONSTRAINT curve_identifier');
$db->exec('SET CONSTRAINTS ALL DEFERRED');
foreach ($dataFiles as $dataFile) {
// Get the file locally
$downloadFile = $ftpRoot . '/' . $dataFile;
$tgzFile = $downloadDir . DIRECTORY_SEPARATOR . $dataFile;
$txtFile = $extractedDir . DIRECTORY_SEPARATOR . str_replace('.tar.gz', '.txt', $dataFile);
try {
if (!downloadUrl($downloadFile, $tgzFile)) {
echo ' already downloaded.' . PHP_EOL;
}
extractTarGz($tgzFile, $extractedDir);
// Should create $txtFile
echo 'Parsing and loading data file into database...';
$dataLoader->load($txtFile, null, true);
echo '...done.' . PHP_EOL;
} catch (Exception $e) {
echo PHP_EOL . 'Failed loading data file [' . $dataFile . '].' . PHP_EOL . $e->getMessage() . PHP_EOL;
} finally {
if (file_exists($tgzFile)) {
unlink($tgzFile);
}
if (file_exists($txtFile)) {
unlink($txtFile);
}
}
}
$db->exec('ALTER TABLE curve ADD CONSTRAINT ' . 'curve_identifier UNIQUE (datasetid, latitude, longitude)');
$db->exec('ALTER TABLE curve ADD CONSTRAINT ' . 'curve_datasetid_fkey FOREIGN KEY (datasetid) REFERENCES dataset (id)');
$db->exec('ALTER TABLE curve ADD PRIMARY KEY (id)');
<?php
//
// This file provides a command line wrapper around the DataLoader to allow
// a user to load individual data files from the local file system without
// needing to re-install the entire application.
//
$oldDir = getcwd();
chdir(dirname(__FILE__));
if (count($argv) < 2) {
echo 'Usage: php ' . $argv[0] . ' <datafile>[ <datafile>...]' . PHP_EOL;
exit(-1);
}
include_once 'connect.admin.db.php';
include_once 'DataLoader.class.php';
$errors = 0;
$dataLoader = new DataLoader($db);
foreach (array_slice($argv, 1) as $dataFile) {
try {
echo $dataFile . PHP_EOL;
$dataLoader->load($dataFile, null, true);
} catch (Exception $ex) {
$errors += 1;
echo $ex->getMessage() . PHP_EOL;
}
}
echo 'Completed with ' . $errors . ' errors.' . PHP_EOL;
chdir($oldDir);
exit(0);
