/** * http://www.data.gov/app/themes/roots-nextdatagov/assets/Json/fed_agency.json */ define('ORGANIZATION_TO_EXPORT', 'Department of Labor'); require_once dirname(__DIR__) . '/inc/common.php'; if (!is_readable($keywords_file_path = CKANMNGR_DATA_DIR . '/search_organizations.csv')) { die($keywords_file_path . ' not readable'); } $organizations_list = file_get_contents($keywords_file_path); $organizations_list = preg_replace('/[\\r\\n]+/', "\n", $organizations_list); $organizations_list = explode("\n", $organizations_list); /** * Create results dir for logs and json results */ $results_dir = CKANMNGR_RESULTS_DIR . date('/Ymd-His') . '_SEARCH_ORGANIZATIONS_' . sizeof($organizations_list); mkdir($results_dir); /** * Search for packages by terms found */ /** * Production */ $CkanManager = new CkanManager(CKAN_API_URL); /** * Staging */ //$CkanManager = new CkanManager(CKAN_STAGING_API_URL); $CkanManager->resultsDir = $results_dir; $CkanManager->searchByOrganizations($organizations_list); // show running time on finish timer();
/** * http://www.data.gov/app/themes/roots-nextdatagov/assets/Json/fed_agency.json */ define('ORGANIZATION_TO_EXPORT', 'Department of Labor'); require_once dirname(__DIR__) . '/inc/common.php'; if (!is_readable($keywords_file_path = CKANMNGR_DATA_DIR . '/search_terms.csv')) { die($keywords_file_path . ' not readable'); } $keywords_list = file_get_contents($keywords_file_path); $keywords_list = preg_replace('/[\\r\\n]+/', "\n", $keywords_list); $keywords_list = explode("\n", $keywords_list); /** * Create results dir for logs and json results */ $results_dir = CKANMNGR_RESULTS_DIR . date('/Ymd-His') . '_SEARCH_TERMS_' . sizeof($keywords_list); mkdir($results_dir); /** * Search for packages by terms found */ /** * Production */ $CkanManager = new CkanManager(CKAN_API_URL); /** * Staging */ //$CkanManager = new CkanManager(CKAN_STAGING_API_URL); $CkanManager->resultsDir = $results_dir; $CkanManager->searchByTerms($keywords_list); // show running time on finish timer();
<?php namespace CKAN\Manager; /** * http://www.data.gov/app/themes/roots-nextdatagov/assets/Json/fed_agency.json */ //$search = isset($argv[1]) ? trim($argv[1]) : false; // //if (!$search) { // die('Please define search by first param' . PHP_EOL); //} // //$strip_search = preg_replace("/\\(([a-z]+-[a-z\\-]*[a-z]+)\\)/", '"${1}"', $search); // //$filename_strip_search = preg_replace("/[^a-zA-Z0-9\\ ]+/i", '', $search); //die($strip_search.PHP_EOL); require_once dirname(__DIR__) . '/inc/common.php'; /** * Create results dir for logs and json results */ $results_dir = CKANMNGR_RESULTS_DIR . date('/Ymd-His') . '_EXPORT_RESOURCE_LIST'; mkdir($results_dir); $CkanManager = new CkanManager(CKAN_API_URL); //$CkanManager = new CkanManager(INVENTORY_CKAN_PROD_API_URL, INVENTORY_CKAN_PROD_API_KEY); //$CkanManager = new CkanManager(CKAN_STAGING_API_URL); $CkanManager->resultsDir = $results_dir; $CkanManager->exportResourceList(500); // show running time on finish timer();
<?php namespace CKAN\Manager; require_once dirname(dirname(__DIR__)) . '/inc/common.php'; $results_dir = CKANMNGR_RESULTS_DIR . date('/Ymd-His') . '_FIND_MATCHES_ONE_FILE'; mkdir($results_dir); $CkanManager = new CkanManager(CKAN_QA_API_URL); //$CkanManager = new CkanManager(CKAN_API_URL); $CkanManager->resultsDir = $results_dir; $CkanManager->findMatchesOneFile();
<?php namespace CKAN\Manager; echo "Organizations stats" . PHP_EOL; require_once dirname(__DIR__) . '/inc/common.php'; define('START', isset($argv[1]) ? trim($argv[1]) : false); define('STOP', isset($argv[2]) ? trim($argv[2]) : false); //define('LIST_ORGS_ONLY', true); /** * Create results dir for logs and json results */ $results_dir = CKANMNGR_RESULTS_DIR . date('/Ymd-His') . '_ORG_STATS'; mkdir($results_dir); /** * Production */ $CkanManager = new CkanManager(INVENTORY_CKAN_PROD_API_URL, INVENTORY_CKAN_PROD_API_KEY); //$CkanManager = new CkanManager(CKAN_API_URL, CKAN_API_KEY); /** * Staging */ //$CkanManager = new CkanManager(CKAN_STAGING_API_URL); $CkanManager->resultsDir = $results_dir; $CkanManager->organizations_stats(); if ($CkanManager->logOutput) { file_put_contents($results_dir . '/log.csv', $CkanManager->logOutput); } // show running time on finish timer();
<?php namespace CKAN\Manager; require_once dirname(__DIR__) . '/inc/common.php'; /** * Create results dir for logs */ $results_dir = CKANMNGR_RESULTS_DIR . date('/Ymd-His') . '_CLEANUP_TAGS'; mkdir($results_dir); $CkanManager = new CkanManager(CKAN_API_URL, CKAN_API_KEY); //$CkanManager = new CkanManager(CKAN_STAGING_API_URL, CKAN_STAGING_API_KEY); //$CkanManager = new CkanManager(CKAN_DEV_API_URL, CKAN_DEV_API_KEY); $CkanManager->resultsDir = $results_dir; $topicTitle = 'ecosystems0617'; $CkanManager->cleanUpTagsByTopic($topicTitle); // show running time on finish timer();
<?php namespace CKAN\Manager; require_once dirname(__DIR__) . '/inc/common.php'; /** * Create results dir for logs */ $results_dir = CKANMNGR_RESULTS_DIR . date('/Ymd-His') . '_DEV_TEST'; mkdir($results_dir); //$CkanManager = new CkanManager(CKAN_API_URL, CKAN_API_KEY); //$CkanManager = new CkanManager(CKAN_STAGING_API_URL, CKAN_STAGING_API_KEY); $CkanManager = new CkanManager(CKAN_DEV_API_URL, CKAN_DEV_API_KEY); /** * UAT */ //$CkanManager = new CkanManager(CKAN_UAT_API_URL, CKAN_UAT_API_KEY); $CkanManager->resultsDir = $results_dir; $CkanManager->test_dev(); // show running time on finish timer();
<?php namespace CKAN\Manager; require_once dirname(dirname(__DIR__)) . '/inc/common.php'; $results_dir = CKANMNGR_RESULTS_DIR . date('/Ymd-His') . '_FIND_MATCHES'; mkdir($results_dir); $CkanManager = new CkanManager(CKAN_API_URL); //$CkanManager = new CkanManager(CKAN_QA_API_URL); $CkanManager->resultsDir = $results_dir; $CkanManager->findMatchesByAgency('doc');
<?php namespace CKAN\Manager; require_once dirname(__DIR__) . '/inc/common.php'; $CkanManager = new CkanManager(CKAN_API_URL, CKAN_API_KEY); //$CkanManager = new CkanManager(CKAN_UAT_API_URL, CKAN_UAT_API_KEY); //$CkanManager = new CkanManager(CKAN_QA_API_URL, CKAN_QA_API_KEY); //$CkanManager = new CkanManager(INVENTORY_CKAN_PROD_API_URL, INVENTORY_CKAN_PROD_API_KEY); $results_dir = CKANMNGR_RESULTS_DIR . date('/Ymd-His') . '_RESOURCE_CREATE'; mkdir($results_dir); $CkanManager->resultsDir = $results_dir; $logFile = $results_dir . '/_log.csv'; $CkanManager->resourceCreate(['package_id' => 'department-of-the-interior-enterprise-data-inventory', 'url' => 'http://data.doi.gov/WAF/edi.json', 'name' => 'EDI Json', 'format' => 'application/json']); file_put_contents($logFile, $CkanManager->logOutput, FILE_APPEND | LOCK_EX); //$CkanManager->logOutput = ''; // show running time on finish timer();
<?php namespace CKAN\Manager; require_once dirname(dirname(__DIR__)) . '/inc/common.php'; $results_dir = CKANMNGR_RESULTS_DIR . date('/Ymd-His') . '_FIND_MATCHES_SEPARATE_FILES'; mkdir($results_dir); $CkanManager = new CkanManager(CKAN_QA_API_URL); //$CkanManager = new CkanManager(CKAN_API_URL); $CkanManager->resultsDir = $results_dir; $CkanManager->findMatchesSeparateFiles();
<?php namespace CKAN\Manager; require_once dirname(dirname(__DIR__)) . '/inc/common.php'; $results_dir = CKANMNGR_RESULTS_DIR . date('/Ymd-His') . '_FIND_MATCHES'; mkdir($results_dir); $CkanManager = new CkanManager(CKAN_API_URL); //$CkanManager = new CkanManager(CKAN_QA_API_URL); $CkanManager->resultsDir = $results_dir; $CkanManager->findMatches();
/** * http://www.data.gov/app/themes/roots-nextdatagov/assets/Json/fed_agency.json */ define('ORGANIZATION_TO_EXPORT', 'Department of Labor'); require_once dirname(__DIR__) . '/inc/common.php'; if (!is_readable($keywords_file_path = CKANMNGR_DATA_DIR . '/search_topics.csv')) { die($keywords_file_path . ' not readable'); } $topics_list = file_get_contents($keywords_file_path); $topics_list = preg_replace('/[\\r\\n]+/', "\n", $topics_list); $topics_list = explode("\n", $topics_list); /** * Create results dir for logs and json results */ $results_dir = CKANMNGR_RESULTS_DIR . date('/Ymd-His') . '_SEARCH_TOPICS_' . sizeof($topics_list); mkdir($results_dir); /** * Search for packages by terms found */ /** * Production */ $CkanManager = new CkanManager(CKAN_API_URL); /** * Staging */ //$CkanManager = new CkanManager(CKAN_STAGING_API_URL); $CkanManager->resultsDir = $results_dir; $CkanManager->searchByTopics($topics_list); // show running time on finish timer();
namespace CKAN\Manager; /** * http://www.data.gov/app/themes/roots-nextdatagov/assets/Json/fed_agency.json */ $topic = isset($argv[1]) ? trim($argv[1]) : false; if (!$topic) { die('You should indicate topic to export, as first argument in shell' . PHP_EOL); } $topic = preg_replace("/[^a-zA-Z0-9\\ ]+/i", '', $topic); require_once dirname(__DIR__) . '/inc/common.php'; /** * Create results dir for logs and json results */ $results_dir = CKANMNGR_RESULTS_DIR . date('/Ymd-His') . '_EXPORT_BY_TOPIC_' . $topic; mkdir($results_dir); /** * Search for packages by terms found */ /** * Production */ $CkanManager = new CkanManager(CKAN_API_URL); /** * Staging */ //$CkanManager = new CkanManager(CKAN_STAGING_API_URL); $CkanManager->resultsDir = $results_dir; $CkanManager->exportDatasetsWithTagsByGroup($topic); // show running time on finish timer();
//$termsArray = $OrgList->getTreeArrayFor(ORGANIZATION_TO_EXPORT); $termsArray = ['pbgc-gov' => 'Pension Benefit Guaranty Corporation']; /** * sometimes there is no parent term (ex. Department of Labor) */ if (!defined('PARENT_TERM')) { define('PARENT_TERM', '_'); } /** * Create results dir for logs and json results */ $results_dir = CKANMNGR_RESULTS_DIR . date('/Ymd-His') . '_EXPORT_PBGC' . PARENT_TERM; mkdir($results_dir); /** * Search for packages by terms found */ /** * Production */ $CkanManager = new CkanManager(CKAN_API_URL); //$CkanManager = new CkanManager(CKAN_UAT_API_URL); //$CkanManager = new CkanManager(CKAN_QA_API_URL); //$CkanManager = new CkanManager(INVENTORY_CKAN_PROD_API_URL, INVENTORY_CKAN_PROD_API_KEY); /** * Staging */ //$CkanManager = new CkanManager(CKAN_STAGING_API_URL); $CkanManager->resultsDir = $results_dir; $CkanManager->exportPackagesByOrgTerms($termsArray); // show running time on finish timer();
<?php namespace CKAN\Manager; require_once dirname(__DIR__) . '/inc/common.php'; $start = isset($argv[1]) ? intval($argv[1]) : 0; $limit = isset($argv[2]) ? intval($argv[2]) : 10000; /** * Create results dir for logs */ //$resultsDir = RESULTS_DIR . date('/Ymd-His') . '_MISSING_GROUPS_DATASETS'; $results_dir = CKANMNGR_RESULTS_DIR . date('/Ymd-1') . '_MISSING_GROUPS_DATASETS'; is_dir($results_dir) || mkdir($results_dir); /** * Production */ $CkanManager = new CkanManager(CKAN_API_URL, CKAN_API_KEY); /** * Staging */ //$CkanManager = new CkanManager(CKAN_STAGING_API_URL, CKAN_STAGING_API_KEY); /** * Dev */ //$CkanManager = new CkanManager(CKAN_DEV_API_URL, CKAN_DEV_API_KEY); $CkanManager->resultsDir = $results_dir; $CkanManager->orphanedTagsSeek($limit, $start); // show running time on finish timer();
<?php /** * @author Alex Perfilov * @date 5/23/14 * */ namespace CKAN\Manager; require_once dirname(__DIR__) . '/inc/common.php'; /** * Create results dir for logs */ $results_dir = CKANMNGR_RESULTS_DIR . date('/Ymd-His') . '_CHECK_STAGING_VS_PROD'; mkdir($results_dir); $CkanManagerProduction = new CkanManager(CKAN_API_URL); $CkanManagerStaging = new CkanManager(CKAN_STAGING_API_URL); $CkanManagerStaging->resultsDir = $results_dir; $CkanManagerProduction->resultsDir = $results_dir; $groups = $CkanManagerStaging->groupsArray(); foreach ($groups as $category) { $CkanManagerStaging->checkGroupAgainstProd($category, $CkanManagerProduction); } // show running time on finish timer();
* http://www.data.gov/app/themes/roots-nextdatagov/assets/Json/fed_agency.json */ $search = isset($argv[1]) ? trim($argv[1]) : false; if (!$search) { die('Please define search by first param' . PHP_EOL); } $strip_search = preg_replace("/\\(([a-z]+-[a-z\\-]*[a-z]+)\\)/", '"${1}"', $search); $filename_strip_search = preg_replace("/[^a-zA-Z0-9\\ ]+/i", '', $search); //die($strip_search.PHP_EOL); require_once dirname(__DIR__) . '/inc/common.php'; /** * Create results dir for logs and json results */ $results_dir = CKANMNGR_RESULTS_DIR . date('/Ymd-His') . '_EXPORT_BY_SEARCH_' . $filename_strip_search; mkdir($results_dir); /** * Search for packages by terms found */ /** * Production */ $CkanManager = new CkanManager(CKAN_API_URL); //$CkanManager = new CkanManager(INVENTORY_CKAN_PROD_API_URL, INVENTORY_CKAN_PROD_API_KEY); /** * Staging */ //$CkanManager = new CkanManager(CKAN_STAGING_API_URL); $CkanManager->resultsDir = $results_dir; $CkanManager->exportDatasetsBySearch($strip_search); // show running time on finish timer();
<?php namespace CKAN\Manager; define('LOG_NAME', 'activeUsers'); echo "Exporting active users" . PHP_EOL; require_once dirname(__DIR__) . '/inc/common.php'; /** * Create results dir for logs and json results */ $results_dir = CKANMNGR_RESULTS_DIR . date('/Ymd-His') . '_EXPORT_' . LOG_NAME; mkdir($results_dir); /** * Search for packages by terms found */ /** * Production */ //$CkanManager = new CkanManager(INVENTORY_CKAN_PROD_API_URL, INVENTORY_CKAN_PROD_API_KEY); $CkanManager = new CkanManager(CKAN_API_URL, CKAN_API_KEY); /** * Staging */ //$CkanManager = new CkanManager(CKAN_STAGING_API_URL); $CkanManager->resultsDir = $results_dir; $CkanManager->activeUsers(); // show running time on finish timer();
<?php namespace CKAN\Manager; /** * @author Alex Perfilov * @date 5/23/14 * */ require_once dirname(__DIR__) . '/inc/common.php'; /** * Create results dir for logs */ $results_dir = CKANMNGR_RESULTS_DIR . date('/Ymd-His') . '_SOCRATA_INTERACTIVE_RESOURCES'; mkdir($results_dir); $CkanManager = new CkanManager(CKAN_API_URL, CKAN_API_KEY); //$CkanManager = new CkanManager(CKAN_STAGING_API_URL, CKAN_STAGING_API_KEY); //$CkanManager = new CkanManager(CKAN_DEV_API_URL, CKAN_DEV_API_KEY); /** * */ define('ERROR_REPORTING', E_ALL & ~E_NOTICE); // https://explore.data.gov/api/views/bxfh-jivs.json /** * */ //define('SOCRATA_URL', 'https://explore.data.gov/api/views/'); // //if (!is_readable($socrata_file_path = DATA_DIR . '/socrata.txt')) { // die($socrata_file_path . ' not readable'); //}
<?php namespace CKAN\Manager; /** * @author Alex Perfilov * @date 5/23/14 * */ require_once dirname(__DIR__) . '/inc/common.php'; /** * Create results dir for logs */ $results_dir = CKANMNGR_RESULTS_DIR . date('/Ymd-His') . '_SOCRATA_PAIRS'; mkdir($results_dir); $CkanManager = new CkanManager(CKAN_API_URL, CKAN_API_KEY); //$CkanManager = new CkanManager(CKAN_STAGING_API_URL, CKAN_STAGING_API_KEY); //$CkanManager = new CkanManager(CKAN_DEV_API_URL, CKAN_DEV_API_KEY); $CkanManager->resultsDir = $results_dir; /** * */ define('ERROR_REPORTING', E_ALL & ~E_NOTICE); // https://explore.data.gov/api/views/bxfh-jivs.json /** * */ define('SOCRATA_URL', 'https://explore.data.gov/api/views/'); if (!is_readable($socrata_file_path = CKANMNGR_DATA_DIR . '/socrata.txt')) { die($socrata_file_path . ' not readable'); }
<?php namespace CKAN\Manager; /** * http://www.data.gov/app/themes/roots-nextdatagov/assets/Json/fed_agency.json */ require_once dirname(__DIR__) . '/inc/common.php'; /** * Create results dir for logs */ $results_dir = CKANMNGR_RESULTS_DIR . date('/Ymd-His') . '_TAG_BY_identifier'; mkdir($results_dir); $CkanManager = new CkanManager(CKAN_API_URL, CKAN_API_KEY); //$CkanManager = new CkanManager(CKAN_STAGING_API_URL, CKAN_STAGING_API_KEY); //$CkanManager = new CkanManager(CKAN_DEV_API_URL, CKAN_DEV_API_KEY); //$CkanManager = new CkanManager(CKAN_DEV2_API_URL, CKAN_DEV2_API_KEY); $CkanManager->resultsDir = $results_dir; $CkanManager->tagByExtraField('identifier', 'source_datajson_identifier'); // show running time on finish timer();