use EasyCSV\Writer; require_once dirname(dirname(__DIR__)) . '/inc/common.php'; $organization = 'doj-gov'; /** * Create results dir for logs */ $results_dir = CKANMNGR_RESULTS_DIR . date('/Ymd_') . strtoupper($organization) . '_REMOVING_DUPLICATES'; if (!is_dir($results_dir)) { mkdir($results_dir); } echo $organization . '_export.json' . PHP_EOL; if (!is_file($results_dir . '/' . $organization . '_export.json')) { $ProdCkanManager = new CkanManager(CKAN_API_URL); $ProdCkanManager->resultsDir = $results_dir; $prod = $ProdCkanManager->exportFiltered('organization:' . $organization . ' AND dataset_type:dataset', '', ['title', 'title_simple', 'name', 'url', 'identifier', 'guid', 'metadata_created', 'metadata_modified', 'extras_modified', 'topics', 'categories', 'tagging']); file_put_contents($results_dir . '/' . $organization . '_export.json', json_encode($prod, JSON_PRETTY_PRINT)); $prod_csv = new Writer($results_dir . '/' . $organization . '_export.csv'); $headers = array_keys($prod[array_keys($prod)[0]]); $prod_csv->writeRow($headers); $prod_csv->writeFromArray($prod); echo PHP_EOL . 'datasets from prod: ' . sizeof($prod) . PHP_EOL . PHP_EOL; } else { $prod = json_decode(file_get_contents($results_dir . '/' . $organization . '_export.json'), true); echo PHP_EOL . 'datasets from prod: ' . sizeof($prod) . PHP_EOL . PHP_EOL; } if (!is_file($results_dir . '/' . $organization . '_export_sorted.json')) { $prod_sorted = []; foreach ($prod as $dataset_array) { $index = Dataset::simplifyTitle($dataset_array['title_simple'] . '_' . $dataset_array['identifier']); if (!isset($prod_sorted[$index])) {