/** * {@inheritdoc} */ public function writerTest() { $csv = new Writer($this->path, 'w'); foreach ($this->generateRawData() as $row) { $csv->writeRow($row); } }
foreach ($cmp2 as $name => $dataset) { $title = $dataset['title_simple']; $cmp2_by_title[$title] = isset($cmp2_by_title[$title]) ? $cmp2_by_title[$title] : []; $cmp2_by_title[$title][] = $dataset; $guid = trim($dataset['guid']); if ($guid) { $cmp2_by_guid[$guid] = isset($cmp2_by_guid[$guid]) ? $cmp2_by_guid[$guid] : []; $cmp2_by_guid[$guid][] = $dataset; } } echo 'comparison.csv' . PHP_EOL; is_file($results_dir . '/comparison.csv') && unlink($results_dir . '/comparison.csv'); $csv = new Writer($results_dir . '/comparison.csv'); $cmp1_header = "DMS"; $cmp2_header = "NON-DMS"; $csv->writeRow([$cmp1_header . ' Title', $cmp1_header . ' URL', $cmp1_header . ' GUID', $cmp1_header . ' Topics', $cmp1_header . ' Categories', 'Matched', $cmp2_header . ' Title', $cmp2_header . ' URL', $cmp2_header . ' GUID', 'URL Match', 'GUID Match']); foreach ($cmp1 as $name => $cmp1_dataset) { if (isset($cmp2_by_guid[$cmp1_dataset['guid']])) { foreach ($cmp2_by_guid[$cmp1_dataset['guid']] as $cmp2_dataset) { $csv->writeRow([$cmp1_dataset['title'], $cmp1_dataset['url'], $cmp1_dataset['guid'], $cmp1_dataset['topics'], $cmp1_dataset['categories'], true, $cmp2_dataset['title'], $cmp2_dataset['url'], $cmp2_dataset['guid'], (bool) ($cmp1_dataset['name'] && $cmp1_dataset['name'] == $cmp2_dataset['name']), true]); } continue; } if (isset($cmp2_by_title[$cmp1_dataset['title_simple']])) { foreach ($cmp2_by_title[$cmp1_dataset['title_simple']] as $cmp2_dataset) { $csv->writeRow([$cmp1_dataset['title'], $cmp1_dataset['url'], $cmp1_dataset['guid'], $cmp1_dataset['topics'], $cmp1_dataset['categories'], true, $cmp2_dataset['title'], $cmp2_dataset['url'], $cmp2_dataset['guid'], true, (bool) ($cmp1_dataset['guid'] && $cmp1_dataset['guid'] == $cmp2_dataset['guid'])]); } continue; } $csv->writeRow([$cmp1_dataset['title'], $cmp1_dataset['url'], $cmp1_dataset['guid'], $cmp1_dataset['topics'], $cmp1_dataset['categories'], false, '', '', '', false, false]); }
/** * @param $category * @param CkanManager $CkanManagerProduction */ public function checkGroupAgainstProd($category, self $CkanManagerProduction) { $csv = new Writer($this->resultsDir . '/' . $category . date('_Ymd-His') . '.csv'); $csv->writeRow(['Staging dataset name', 'Staging Source', 'Prod exists', 'Prod has ' . $category, 'Prod Source']); $ckan_query = '((groups:' . $category . ') + dataset_type:dataset)'; $start = 0; $per_page = 20; while (true) { $packages = $this->tryPackageSearch($ckan_query, '', $per_page, $start); if (!$packages) { echo "{$start} / {$per_page} :: finish" . PHP_EOL; break; } foreach ($packages as $package) { if (is_array($package['extras']) && sizeof($package['extras']) && strpos(json_encode($package['extras']), '"dms"')) { $resource_type = 'DMS'; // echo "DMS ".$package['name'].PHP_EOL; } elseif (is_array($package['extras']) && sizeof($package['extras']) && strpos(json_encode($package['extras']), '"value":"geospatial"')) { $resource_type = 'GEO'; // echo "GEO ".$package['name'].PHP_EOL; } elseif (is_array($package['extras']) && sizeof($package['extras']) && strpos(json_encode($package['extras']), 'source_datajson_identifier')) { $resource_type = 'JSON'; // echo "JSON ".$package['name'].PHP_EOL; } else { $resource_type = 'OTHER'; echo json_encode($package['extras']) . PHP_EOL; echo "UNKNOWN: " . $package['name'] . PHP_EOL; } $prod_package = $CkanManagerProduction->tryPackageShow($package['name']); $exists = $prod_package ? 'EXISTS' : 'NOT FOUND'; $prod_category_found = ''; $prod_resource_type = ''; if ($prod_package) { $prod_category_found = 'FALSE'; if (isset($prod_package['groups']) && sizeof($prod_package['groups']) && strpos(json_encode($prod_package['groups']), $category)) { $prod_category_found = 'HAS'; } if (is_array($prod_package['extras']) && sizeof($prod_package['extras']) && strpos(json_encode($prod_package['extras']), '"dms"')) { $prod_resource_type = 'DMS'; // echo "DMS ".$prod_package['name'].PHP_EOL; } elseif (is_array($prod_package['extras']) && sizeof($prod_package['extras']) && strpos(json_encode($prod_package['extras']), '"value":"geospatial"')) { $prod_resource_type = 'GEO'; // echo "GEO ".$prod_package['name'].PHP_EOL; } elseif (is_array($prod_package['extras']) && sizeof($prod_package['extras']) && strpos(json_encode($prod_package['extras']), 'source_datajson_identifier')) { $prod_resource_type = 'JSON'; // echo "JSON ".$prod_package['name'].PHP_EOL; } else { $prod_resource_type = 'OTHER'; echo json_encode($prod_package['extras']) . PHP_EOL; echo "UNKNOWN on PROD: " . $prod_package['name'] . PHP_EOL; } } $csv->writeRow([$package['name'], $resource_type, $exists, $prod_category_found, $prod_resource_type]); } $start += $per_page; } }
// skip headers if (in_array(trim(strtolower($row['0'])), ['link', 'dataset', 'url', 'data.gov url'])) { continue; } if ($start > 0) { $start--; continue; } // no anchors please list($dataset, ) = explode('#', basename(trim($row['0']))); // echo $dataset.PHP_EOL; if (!$dataset) { continue; } // double trouble check if (strpos($row['0'], '://')) { if (!strpos($row['0'], '/dataset/')) { file_put_contents($results_dir . '/' . $basename . '_export.log.csv', $row['0'] . ',WRONG URL' . PHP_EOL, FILE_APPEND | LOCK_EX); continue; } } $lines = $CkanManager->exportPackage($dataset); foreach ($lines as $line) { $tags_csv->writeRow($line); } } } //$brief = $CkanManager->exportShort('extras_harvest_source_title:Test ISO WAF AND (dataset_type:dataset)'); //$csv->writeFromArray($brief); // show running time on finish timer();
foreach ($json_backup_noaa as $name => $dataset) { $title = $dataset['title_simple']; $json_backup_noaa_by_title[$title] = isset($json_backup_noaa_by_title[$title]) ? $json_backup_noaa_by_title[$title] : []; $json_backup_noaa_by_title[$title][] = $dataset; $guid = trim($dataset['guid']); if ($guid) { $json_backup_noaa_by_guid[$guid] = isset($json_backup_noaa_by_guid[$guid]) ? $json_backup_noaa_by_guid[$guid] : []; $json_backup_noaa_by_guid[$guid][] = $dataset; } } echo 'prod_vs_json_backup.csv' . PHP_EOL; is_file($results_dir . '/prod_vs_json_backup_noaa_geospatial.csv') && unlink($results_dir . '/prod_vs_json_backup_noaa_geospatial.csv'); $csv = new Writer($results_dir . '/prod_vs_json_backup_noaa_geospatial.csv'); $csv->writeRow(['UAT Title', 'UAT URL', 'UAT GUID', 'UAT Topics', 'UAT Categories', 'Matched', 'JSON Title', 'JSON URL', 'JSON GUID', 'URL Match', 'GUID Match']); $csv_uat_tagging = new Writer($results_dir . '/uat_tagging.csv'); $csv_uat_tagging->writeRow(['url', 'group', 'tags']); foreach ($uat_noaa as $name => $uat_dataset) { if (isset($json_backup_noaa_by_guid[$uat_dataset['guid']])) { foreach ($json_backup_noaa_by_guid[$uat_dataset['guid']] as $json_backup_dataset) { $csv->writeRow([$uat_dataset['title'], $uat_dataset['url'], $uat_dataset['guid'], $uat_dataset['topics'], $uat_dataset['categories'], true, $json_backup_dataset['title'], $json_backup_dataset['url'], $json_backup_dataset['guid'], (bool) ($uat_dataset['name'] == $json_backup_dataset['name']), true]); if (isset($json_backup_tags[$json_backup_dataset['title_simple']])) { foreach ($json_backup_tags[$json_backup_dataset['title_simple']] as $group => $tags) { $csv_uat_tagging->writeRow([$uat_dataset['url'], $group, join(';', $tags)]); } } } continue; } if (isset($json_backup_noaa_by_title[$uat_dataset['title_simple']])) { foreach ($json_backup_noaa_by_title[$uat_dataset['title_simple']] as $json_backup_dataset) { $csv->writeRow([$uat_dataset['title'], $uat_dataset['url'], $uat_dataset['guid'], $uat_dataset['topics'], $uat_dataset['categories'], true, $json_backup_dataset['title'], $json_backup_dataset['url'], $json_backup_dataset['guid'], true, (bool) ($uat_dataset['guid'] == $json_backup_dataset['guid'])]);
// We don't want the header (use curl_getinfo()) curl_setopt($curl_ch, CURLOPT_HEADER, false); // Track the handle's request string curl_setopt($curl_ch, CURLINFO_HEADER_OUT, true); // Attempt to retrieve the modification date of the remote document. curl_setopt($curl_ch, CURLOPT_FILETIME, true); // Initialize cURL headers foreach (glob(CKANMNGR_DATA_DIR . '/redirects_*.csv') as $csv_file) { $status = PHP_EOL . PHP_EOL . basename($csv_file) . PHP_EOL . PHP_EOL; echo $status; $basename = str_replace('.csv', '', basename($csv_file)); // fix wrong END-OF-LINE file_put_contents($csv_file, preg_replace('/[\\r\\n]+/', "\n", file_get_contents($csv_file))); $csv_source = new EasyCSV\Reader($csv_file, 'r+', false); $csv_destination = new EasyCSV\Writer($results_dir . '/' . $basename . '_log.csv'); $csv_destination->writeRow(['from', 'to', 'status', 'real_redirect']); $i = 0; while (true) { if (!($i++ % 100)) { echo $i . PHP_EOL; } $row = $csv_source->getRow(); if (!$row) { break; } // skip headers if (in_array(trim(strtolower($row[0])), ['socrata code', 'from', 'source url'])) { // $csv_destination->writeRow($row); continue; } $socrata_url = $row[0];
$d['status'] = 'deleted'; } array_push($statistics, $d); } } $delete_csv = new Writer($results_dir . '/delete_' . $organization . '.csv'); $delete_csv->writeRow(['url']); $delete_csv->writeFromArray($delete); $delete_full_csv = new Writer($results_dir . '/' . $organization . '_delete_full.csv'); $headers = array_keys($delete_full[0]); $delete_full_csv->writeRow($headers); $delete_full_csv->writeFromArray($delete_full); $stats_csv = new Writer($results_dir . '/' . $organization . '_statistics.csv'); $headers = array_keys($statistics[0]); $stats_csv->writeRow($headers); $stats_csv->writeFromArray($statistics); $survivors_csv = new Writer($results_dir . '/' . $organization . '_survivors.csv'); $headers = array_keys($survivors[array_keys($survivors)[0]]); $survivors_csv->writeRow($headers); $survivors_csv->writeFromArray($survivors); $stitle = ''; foreach ($delete_full as $dataset) { if ($dataset['title_simple'] !== $stitle) { $stitle = $dataset['title_simple']; // echo PHP_EOL; } // echo printf('%20s %20s',$dataset['title_simple'],$dataset['name']).PHP_EOL; } } // show running time on finish timer();
foreach ($json_backup_epa as $name => $dataset) { $title = $dataset['title_simple']; $json_backup_epa_by_title[$title] = isset($json_backup_epa_by_title[$title]) ? $json_backup_epa_by_title[$title] : []; $json_backup_epa_by_title[$title][] = $dataset; $guid = trim($dataset['guid']); if ($guid) { $json_backup_epa_by_guid[$guid] = isset($json_backup_epa_by_guid[$guid]) ? $json_backup_epa_by_guid[$guid] : []; $json_backup_epa_by_guid[$guid][] = $dataset; } } echo 'prod_vs_json_backup.csv' . PHP_EOL; is_file($results_dir . '/prod_vs_json_backup_epa_geospatial.csv') && unlink($results_dir . '/prod_vs_json_backup_epa_geospatial.csv'); $csv = new Writer($results_dir . '/prod_vs_json_backup_epa_geospatial.csv'); $csv->writeRow(['Prod Title', 'Prod URL', 'Prod GUID', 'Prod Topics', 'Prod Categories', 'Matched', 'JSON Title', 'JSON URL', 'JSON GUID', 'URL Match', 'Title Match', 'GUID Match']); $csv_prod_tagging = new Writer($results_dir . '/prod_tagging.csv'); $csv_prod_tagging->writeRow(['url', 'group', 'tags', 'old_url', 'new_title', 'old_title', 'match_by']); foreach ($prod_epa as $name => $prod_dataset) { if (isset($json_backup_epa_by_guid[$prod_dataset['guid']])) { foreach ($json_backup_epa_by_guid[$prod_dataset['guid']] as $json_backup_dataset) { $csv->writeRow([$prod_dataset['title'], $prod_dataset['url'], $prod_dataset['guid'], $prod_dataset['topics'], $prod_dataset['categories'], true, $json_backup_dataset['title'], $json_backup_dataset['url'], $json_backup_dataset['guid'], (bool) ($prod_dataset['name'] && $prod_dataset['name'] == $json_backup_dataset['name']), (bool) ($prod_dataset['title_simple'] && $prod_dataset['title_simple'] == $json_backup_dataset['title_simple']), true]); if (isset($json_backup_tags[$json_backup_dataset['title_simple']])) { foreach ($json_backup_tags[$json_backup_dataset['title_simple']] as $group => $tags) { $csv_prod_tagging->writeRow([$prod_dataset['url'], $group, join(';', $tags), $json_backup_dataset['name'], $prod_dataset['title_simple'], $json_backup_dataset['title_simple'], 'guid: ' . $prod_dataset['guid']]); } } } continue; } if (isset($json_backup_epa_by_title[$prod_dataset['title_simple']])) { foreach ($json_backup_epa_by_title[$prod_dataset['title_simple']] as $json_backup_dataset) { $csv->writeRow([$prod_dataset['title'], $prod_dataset['url'], $prod_dataset['guid'], $prod_dataset['topics'], $prod_dataset['categories'], true, $json_backup_dataset['title'], $json_backup_dataset['url'], $json_backup_dataset['guid'], (bool) ($prod_dataset['name'] && $prod_dataset['name'] == $json_backup_dataset['name']), true, (bool) ($prod_dataset['guid'] == $json_backup_dataset['guid'])]);
$uat = new Writer($results_dir . '/uat.csv'); $uat->writeRow(['title', 'title_simple', 'name', 'url', 'topics', 'categories']); $UatCkanManager = new CkanManager(CKAN_UAT_API_URL); $UatCkanManager->resultsDir = $results_dir; $uat_nuclear = $UatCkanManager->exportBrief('extras_harvest_source_title:NRC data.json', '', 'http://uat-catalog-fe-data.reisys.com/dataset/'); $uat->writeFromArray($uat_nuclear); } else { $uat = new Reader($results_dir . '/uat.csv'); $uat_nuclear = $uat->getAll(); } $uat_nuclear_by_title = []; foreach ($uat_nuclear as $name => $dataset) { $title = $dataset['title_simple']; $uat_nuclear_by_title[$title] = isset($uat_nuclear_by_title[$title]) ? $uat_nuclear_by_title[$title] : []; $uat_nuclear_by_title[$title][] = $dataset; } echo 'prod_vs_uat.csv' . PHP_EOL; is_file($results_dir . '/prod_vs_uat_nuclear_geospatial.csv') && unlink($results_dir . '/prod_vs_uat_nuclear_geospatial.csv'); $csv = new Writer($results_dir . '/prod_vs_uat_nuclear_geospatial.csv'); $csv->writeRow(['Prod Title', 'Prod URL', 'Prod Topics', 'Prod Categories', 'Matched', 'UAT Title', 'UAT URL']); foreach ($prod_nuclear as $name => $prod_dataset) { if (isset($uat_nuclear_by_title[$prod_dataset['title_simple']])) { foreach ($uat_nuclear_by_title[$prod_dataset['title_simple']] as $uat_dataset) { $csv->writeRow([$prod_dataset['title'], $prod_dataset['url'], $prod_dataset['topics'], $prod_dataset['categories'], true, $uat_dataset['title'], $uat_dataset['url']]); } continue; } $csv->writeRow([$prod_dataset['title'], $prod_dataset['url'], $prod_dataset['topics'], $prod_dataset['categories'], false, '', '']); } // show running time on finish timer();
mkdir($results_dir); /** * Production */ $CkanManager = new CkanManager(CKAN_API_URL); /** * Staging */ //$CkanManager = new CkanManager(CKAN_STAGING_API_URL); foreach (glob(CKANMNGR_DATA_DIR . '/find_*.csv') as $csv_file) { $status = PHP_EOL . PHP_EOL . basename($csv_file) . PHP_EOL . PHP_EOL; echo $status; $basename = str_replace('.csv', '', basename($csv_file)); $csv_source = new EasyCSV\Reader($csv_file, 'r+', false); $csv_destination = new EasyCSV\Writer($results_dir . '/' . $basename . '_results.csv'); $csv_destination->writeRow(['url', 'exact match', 'title', 'found by title']); $i = 0; while (true) { if (!($i++ % 10)) { echo $i . PHP_EOL; } $row = $csv_source->getRow(); if (!$row) { break; } // skip headers if (in_array(trim(strtolower($row[0])), ['url', 'from', 'source url'])) { continue; } $title = $row[0]; /**
} $prod_epa_by_title = $prod_epa_by_guid = []; foreach ($prod_epa as $name => $dataset) { $title = $dataset['title_simple']; $prod_epa_by_title[$title] = isset($prod_epa_by_title[$title]) ? $prod_epa_by_title[$title] : []; $prod_epa_by_title[$title][] = $dataset; $guid = trim($dataset['guid']); if ($guid) { $prod_epa_by_guid[$guid] = isset($prod_epa_by_guid[$guid]) ? $prod_epa_by_guid[$guid] : []; $prod_epa_by_guid[$guid][] = $dataset; } } echo 'json_vs_prod.csv' . PHP_EOL; is_file($results_dir . '/json_vs_prod_epa.csv') && unlink($results_dir . '/json_vs_prod_epa.csv'); $csv = new Writer($results_dir . '/json_vs_prod_epa.csv'); $csv->writeRow(['Backup Title', 'Backup URL', 'Backup GUID', 'Backup Topics', 'Backup Categories', 'Matched', 'Prod Title', 'Prod URL', 'Prod GUID', 'URL Match', 'GUID Match']); foreach ($json_backup_epa as $name => $backup_dataset) { if (isset($prod_epa_by_guid[$backup_dataset['guid']])) { foreach ($prod_epa_by_guid[$backup_dataset['guid']] as $prod_dataset) { $csv->writeRow([$backup_dataset['title'], $backup_dataset['url'], $backup_dataset['guid'], $backup_dataset['topics'], $backup_dataset['categories'], true, $prod_dataset['title'], $prod_dataset['url'], $prod_dataset['guid'], (bool) ($backup_dataset['name'] == $prod_dataset['name']), true]); } continue; } if (isset($prod_epa_by_title[$backup_dataset['title_simple']])) { foreach ($prod_epa_by_title[$backup_dataset['title_simple']] as $prod_dataset) { $csv->writeRow([$backup_dataset['title'], $backup_dataset['url'], $backup_dataset['guid'], $backup_dataset['topics'], $backup_dataset['categories'], true, $prod_dataset['title'], $prod_dataset['url'], $prod_dataset['guid'], true, (bool) ($backup_dataset['guid'] == $prod_dataset['guid'])]); } continue; } $csv->writeRow([$backup_dataset['title'], $backup_dataset['url'], $backup_dataset['guid'], $backup_dataset['topics'], $backup_dataset['categories'], false, '', '', '', false, false]); }
// We don't want the header (use curl_getinfo()) curl_setopt($curl_ch, CURLOPT_HEADER, false); // Track the handle's request string curl_setopt($curl_ch, CURLINFO_HEADER_OUT, true); // Attempt to retrieve the modification date of the remote document. curl_setopt($curl_ch, CURLOPT_FILETIME, true); // Initialize cURL headers foreach (glob(CKANMNGR_DATA_DIR . '/redirects_ckan.csv') as $csv_file) { $status = PHP_EOL . PHP_EOL . basename($csv_file) . PHP_EOL . PHP_EOL; echo $status; $basename = str_replace('.csv', '', basename($csv_file)); // fix wrong END-OF-LINE file_put_contents($csv_file, preg_replace('/[\\r\\n]+/', "\n", file_get_contents($csv_file))); $csv_source = new EasyCSV\Reader($csv_file, 'r+', false); $csv_destination = new EasyCSV\Writer($results_dir . '/' . $basename . '_log.csv'); $csv_destination->writeRow(['url_xyz', 'url', 'url status', 'api_rest_xyz status', 'api_rest_xyz url']); $i = 0; while (true) { if (!($i++ % 10)) { echo $i . PHP_EOL; } $row = $csv_source->getRow(); if (!$row) { break; } // skip headers if (in_array(trim(strtolower($row[0])), ['socrata code', 'from', 'source url'])) { // $csv_destination->writeRow($row); continue; } $dataset_url_xyz = $row[0];
// $categories = explode(';', trim($row['2'])); // $categories = array_map('trim', $categories); // } // no anchors please $dataset = get_dataset_basename($row['0']); if (!$dataset) { continue; } // echo "\tOriginal: ".$dataset . PHP_EOL; // $CkanManager->assignGroupsAndCategoriesToDatasets( // [$dataset], // trim($row['1']), // $categories, // $basename // ); $output->writeRow([$dataset, trim($row['1']), $categories]); echo join(' , ', [$dataset, trim($row['1']), $categories]) . PHP_EOL; if (isset($brothers[$dataset])) { foreach ($brothers[$dataset] as $brother) { if (!strlen(trim($brother))) { continue; } $brother = get_dataset_basename($brother); if (!$brother) { continue; } $output->writeRow([$brother, trim($row['1']), $categories]); echo join(' , ', [$brother, trim($row['1']), $categories]) . PHP_EOL; // echo "\tUat (s):" . PHP_EOL; // $CkanManager->assignGroupsAndCategoriesToDatasets( // [$brother],
// We don't want the header (use curl_getinfo()) curl_setopt($curl_ch, CURLOPT_HEADER, false); // Track the handle's request string curl_setopt($curl_ch, CURLINFO_HEADER_OUT, true); // Attempt to retrieve the modification date of the remote document. curl_setopt($curl_ch, CURLOPT_FILETIME, true); // Initialize cURL headers foreach (glob(CKANMNGR_DATA_DIR . '/check_*.csv') as $csv_file) { $status = PHP_EOL . PHP_EOL . basename($csv_file) . PHP_EOL . PHP_EOL; echo $status; $basename = str_replace('.csv', '', basename($csv_file)); // fix wrong END-OF-LINE file_put_contents($csv_file, preg_replace('/[\\r\\n]+/', "\n", file_get_contents($csv_file))); $csv_source = new EasyCSV\Reader($csv_file, 'r+', false); $csv_destination = new EasyCSV\Writer($results_dir . '/' . $basename . '_log.csv'); $csv_destination->writeRow(['dataset', 'status', 'aapi found']); $i = 0; while (true) { if (!($i++ % 100)) { echo $i . PHP_EOL; } $row = $csv_source->getRow(); if (!$row) { break; } // skip headers if (in_array(trim(strtolower($row[0])), ['data.gov url'])) { continue; } $url = strtolower($row[0]); if (!strpos($url, '/dataset/')) {
// 'url', // 'identifier', // 'org title', // 'org name', // 'topics', // 'categories', //]); $CkanManager->resultsDir = $results_dir; //$brief = $CkanManager->exportShort('extras_license:"https\://creativecommons.org/publicdomain/zero/1.0/" AND (dataset_type:dataset)'); //$brief = $CkanManager->exportShort('','((collection_package_id:* OR *:*) AND license_id:"cc-by-sa" AND license:"https\://creativecommons.org/publicdomain/zero/1.0/") AND (dataset_type:dataset)'); //$brief = $CkanManager->exportShort('%28%28collection_package_id:*%20OR%20*:*%29+AND+license_id:"cc-by-sa"+AND+license:"https://creativecommons.org/publicdomain/zero/1.0/"%29'); //$brief = $CkanManager->exportShort('organization:wake-county AND (dataset_type:dataset)'); //$brief = $CkanManager->exportShort('organization:gsa-gov AND harvest_source_title:Open* AND (dataset_type:dataset)', //$brief = $CkanManager->exportShort('organization:doe-gov AND (dataset_type:dataset)'); //$brief = $CkanManager->exportShort('organization:dhs-gov AND (harvest_source_title:DHS*) AND (dataset_type:dataset)'); //$brief = $CkanManager->exportShort('organization:epa-gov AND (harvest_source_title:*Gateway) AND (dataset_type:dataset)'); //$brief = $CkanManager->exportShort('organization:epa-gov AND (metadata_type:geospatial) AND (dataset_type:dataset)'); //$brief = $CkanManager->exportShort('organization:nasa-gov AND (harvest_source_title:NASA*) AND (dataset_type:dataset)'); $brief = $CkanManager->exportShort('organization:ntsb-gov AND (dataset_type:dataset)'); //$brief = $CkanManager->exportShort('organization:noaa-gov AND metadata_type:geospatial AND (dataset_type:dataset) AND groups:*'); //$brief = $CkanManager->exportShort('metadata-source:dms AND (dataset_type:dataset)'); //$brief = $CkanManager->exportShort('organization:doj-gov AND (dataset_type:dataset)'); // 'http://uat-catalog-fe-data.reisys.com/dataset/'); //$brief = $CkanManager->exportShort('(extra_harvest_source_title:Open+*) AND (dataset_type:dataset)'); //$brief = $CkanManager->exportShort('organization:gsa-gov AND (dataset_type:dataset)'); //$brief = $CkanManager->exportShort('extras_harvest_source_title:Test ISO WAF AND (dataset_type:dataset)'); $headers = array_keys($brief[array_keys($brief)[0]]); $csv->writeRow($headers); $csv->writeFromArray($brief); // show running time on finish timer();
$new = new Writer($results_dir . '/new.csv'); $new->writeRow(['title', 'title_simple', 'name', 'url', 'topics', 'categories']); $UatCkanManager = new CkanManager(CKAN_API_URL); $UatCkanManager->resultsDir = $results_dir; $new_commerce = $UatCkanManager->exportBrief('extras_harvest_source_title:Commerce Non Spatial Data.json Harvest Source'); $new->writeFromArray($new_commerce); } else { $new = new Reader($results_dir . '/new.csv'); $new_commerce = $new->getAll(); } $new_commerce_by_title = []; foreach ($new_commerce as $name => $dataset) { $title = $dataset['title_simple']; $new_commerce_by_title[$title] = isset($new_commerce_by_title[$title]) ? $new_commerce_by_title[$title] : []; $new_commerce_by_title[$title][] = $dataset; } echo 'prod_vs_new.csv' . PHP_EOL; is_file($results_dir . '/prod_vs_prod_commerce.csv') && unlink($results_dir . '/prod_vs_prod_commerce.csv'); $csv = new Writer($results_dir . '/prod_vs_prod_commerce.csv'); $csv->writeRow(['Prod Title', 'Prod URL', 'Prod Topics', 'Prod Categories', 'Matched', 'NEW Title', 'NEW URL', 'URL Match']); foreach ($prod_commerce as $name => $prod_dataset) { if (isset($new_commerce_by_title[$prod_dataset['title_simple']])) { foreach ($new_commerce_by_title[$prod_dataset['title_simple']] as $new_dataset) { $csv->writeRow([$prod_dataset['title'], $prod_dataset['url'], $prod_dataset['topics'], $prod_dataset['categories'], true, $new_dataset['title'], $new_dataset['url'], true]); } continue; } $csv->writeRow([$prod_dataset['title'], $prod_dataset['url'], $prod_dataset['topics'], $prod_dataset['categories'], false, '', '', false]); } // show running time on finish timer();
} $qa_epa_by_title = $qa_epa_by_guid = []; foreach ($qa_epa as $name => $dataset) { $title = $dataset['title_simple']; $qa_epa_by_title[$title] = isset($qa_epa_by_title[$title]) ? $qa_epa_by_title[$title] : []; $qa_epa_by_title[$title][] = $dataset; $guid = trim($dataset['guid']); if ($guid) { $qa_epa_by_guid[$guid] = isset($qa_epa_by_guid[$guid]) ? $qa_epa_by_guid[$guid] : []; $qa_epa_by_guid[$guid][] = $dataset; } } echo 'prod_vs_qa.csv' . PHP_EOL; is_file($results_dir . '/prod_vs_qa_epa.csv') && unlink($results_dir . '/prod_vs_qa_epa.csv'); $csv = new Writer($results_dir . '/prod_vs_qa_epa.csv'); $csv->writeRow(['Prod Title', 'Prod URL', 'Prod GUID', 'Prod Topics', 'Prod Categories', 'Matched', 'QA Title', 'QA URL', 'QA GUID', 'URL Match', 'GUID Match']); foreach ($prod_epa as $name => $prod_dataset) { if (isset($qa_epa_by_guid[$prod_dataset['guid']])) { foreach ($qa_epa_by_guid[$prod_dataset['guid']] as $qa_dataset) { $csv->writeRow([$prod_dataset['title'], $prod_dataset['url'], $prod_dataset['guid'], $prod_dataset['topics'], $prod_dataset['categories'], true, $qa_dataset['title'], $qa_dataset['url'], $qa_dataset['guid'], (bool) ($prod_dataset['name'] == $qa_dataset['name']), true]); } continue; } if (isset($qa_epa_by_title[$prod_dataset['title_simple']])) { foreach ($qa_epa_by_title[$prod_dataset['title_simple']] as $qa_dataset) { $csv->writeRow([$prod_dataset['title'], $prod_dataset['url'], $prod_dataset['guid'], $prod_dataset['topics'], $prod_dataset['categories'], true, $qa_dataset['title'], $qa_dataset['url'], $qa_dataset['guid'], true, (bool) ($prod_dataset['guid'] == $qa_dataset['guid'])]); } continue; } $csv->writeRow([$prod_dataset['title'], $prod_dataset['url'], $prod_dataset['guid'], $prod_dataset['topics'], $prod_dataset['categories'], false, '', '', '', false, false]); }
} $uat_pbgc_by_title = $uat_pbgc_by_guid = []; foreach ($uat_pbgc as $name => $dataset) { $title = $dataset['title_simple']; $uat_pbgc_by_title[$title] = isset($uat_pbgc_by_title[$title]) ? $uat_pbgc_by_title[$title] : []; $uat_pbgc_by_title[$title][] = $dataset; $guid = trim($dataset['guid']); if ($guid) { $uat_pbgc_by_guid[$guid] = isset($uat_pbgc_by_guid[$guid]) ? $uat_pbgc_by_guid[$guid] : []; $uat_pbgc_by_guid[$guid][] = $dataset; } } echo 'prod_vs_uat.csv' . PHP_EOL; is_file($results_dir . '/prod_vs_uat_pbgc_geospatial.csv') && unlink($results_dir . '/prod_vs_uat_pbgc_geospatial.csv'); $csv = new Writer($results_dir . '/prod_vs_uat_pbgc_geospatial.csv'); $csv->writeRow(['Prod Title', 'Prod URL', 'Prod GUID', 'Prod Topics', 'Prod Categories', 'Matched', 'UAT Title', 'UAT URL', 'UAT GUID', 'URL Match', 'Title Match', 'GUID Match']); foreach ($prod_pbgc as $name => $prod_dataset) { if (isset($uat_pbgc_by_guid[$prod_dataset['guid']])) { foreach ($uat_pbgc_by_guid[$prod_dataset['guid']] as $uat_dataset) { $csv->writeRow([$prod_dataset['title'], $prod_dataset['url'], $prod_dataset['guid'], $prod_dataset['topics'], $prod_dataset['categories'], true, $uat_dataset['title'], $uat_dataset['url'], $uat_dataset['guid'], (bool) ($prod_dataset['name'] && $prod_dataset['name'] == $uat_dataset['name']), (bool) ($prod_dataset['title_simple'] && $prod_dataset['title_simple'] == $uat_dataset['title_simple']), true]); } continue; } if (isset($uat_pbgc_by_title[$prod_dataset['title_simple']])) { foreach ($uat_pbgc_by_title[$prod_dataset['title_simple']] as $uat_dataset) { $csv->writeRow([$prod_dataset['title'], $prod_dataset['url'], $prod_dataset['guid'], $prod_dataset['topics'], $prod_dataset['categories'], true, $uat_dataset['title'], $uat_dataset['url'], $uat_dataset['guid'], (bool) ($prod_dataset['name'] && $prod_dataset['name'] == $uat_dataset['name']), true, (bool) ($prod_dataset['guid'] && $prod_dataset['guid'] == $uat_dataset['guid'])]); } continue; } $csv->writeRow([$prod_dataset['title'], $prod_dataset['url'], $prod_dataset['guid'], $prod_dataset['topics'], $prod_dataset['categories'], false, '', '', '', false, false]); }
// We don't want the header (use curl_getinfo()) curl_setopt($curl_ch, CURLOPT_HEADER, false); // Track the handle's request string curl_setopt($curl_ch, CURLINFO_HEADER_OUT, true); // Attempt to retrieve the modification date of the remote document. curl_setopt($curl_ch, CURLOPT_FILETIME, true); // Initialize cURL headers foreach (glob(CKANMNGR_DATA_DIR . '/redirects*.csv') as $csv_file) { $status = PHP_EOL . PHP_EOL . basename($csv_file) . PHP_EOL . PHP_EOL; echo $status; $basename = str_replace('.csv', '', basename($csv_file)); // fix wrong END-OF-LINE file_put_contents($csv_file, preg_replace('/[\\r\\n]+/', "\n", file_get_contents($csv_file))); $csv_source = new EasyCSV\Reader($csv_file, 'r+', false); $csv_destination = new EasyCSV\Writer($results_dir . '/' . $basename . '_log.csv'); $csv_destination->writeRow(['from', 'to', 'status']); $i = 0; while (true) { if (!($i++ % 10)) { echo $i . PHP_EOL; } $row = $csv_source->getRow(); if (!$row) { break; } // skip headers if (in_array(trim(strtolower($row[0])), ['from', 'source url'])) { // $csv_destination->writeRow($row); continue; } $from = $row[0];