use EasyCSV\Writer;
require_once dirname(dirname(__DIR__)) . '/inc/common.php';
$organization = 'doj-gov';
/**
* Create results dir for logs
*/
$results_dir = CKANMNGR_RESULTS_DIR . date('/Ymd_') . strtoupper($organization) . '_REMOVING_DUPLICATES';
if (!is_dir($results_dir)) {
mkdir($results_dir);
}
echo $organization . '_export.json' . PHP_EOL;
if (!is_file($results_dir . '/' . $organization . '_export.json')) {
$ProdCkanManager = new CkanManager(CKAN_API_URL);
$ProdCkanManager->resultsDir = $results_dir;
$prod = $ProdCkanManager->exportFiltered('organization:' . $organization . ' AND dataset_type:dataset', '', ['title', 'title_simple', 'name', 'url', 'identifier', 'guid', 'metadata_created', 'metadata_modified', 'extras_modified', 'topics', 'categories', 'tagging']);
file_put_contents($results_dir . '/' . $organization . '_export.json', json_encode($prod, JSON_PRETTY_PRINT));
$prod_csv = new Writer($results_dir . '/' . $organization . '_export.csv');
$headers = array_keys($prod[array_keys($prod)[0]]);
$prod_csv->writeRow($headers);
$prod_csv->writeFromArray($prod);
echo PHP_EOL . 'datasets from prod: ' . sizeof($prod) . PHP_EOL . PHP_EOL;
} else {
$prod = json_decode(file_get_contents($results_dir . '/' . $organization . '_export.json'), true);
echo PHP_EOL . 'datasets from prod: ' . sizeof($prod) . PHP_EOL . PHP_EOL;
}
if (!is_file($results_dir . '/' . $organization . '_export_sorted.json')) {
$prod_sorted = [];
foreach ($prod as $dataset_array) {
$index = Dataset::simplifyTitle($dataset_array['title_simple'] . '_' . $dataset_array['identifier']);
if (!isset($prod_sorted[$index])) {
