public function get_datagov_json($orgs, $geospatial = false, $rows = 100, $offset = 0, $raw = false, $allow_harvest_sources = 'true')
{
$allow_harvest_sources = empty($allow_harvest_sources) ? 'true' : $allow_harvest_sources;
if ($geospatial == 'both') {
$filter = "%20";
} else {
if ($geospatial == 'true') {
$filter = 'metadata_type:geospatial%20AND%20';
} else {
$filter = '-metadata_type:geospatial%20AND%20';
}
}
if ($allow_harvest_sources !== 'true') {
$filter .= "AND%20-harvest_source_id:[''%20TO%20*]";
}
if (strpos($orgs, 'http://') !== false) {
$uri = $orgs;
$from_export = true;
} else {
$orgs = rawurlencode($orgs);
$query = $filter . "-type:harvest%20AND%20organization:(" . $orgs . ")&rows=" . $rows . '&start=' . $offset;
$uri = 'http://catalog.data.gov/api/3/action/package_search?q=' . $query;
$from_export = false;
}
$datagov_json = curl_from_json($uri, false);
if ($from_export) {
$object_shim = new stdClass();
$object_shim->result = new stdClass();
$object_shim->result->count = count($datagov_json);
$object_shim->result->results = $datagov_json;
$datagov_json = $object_shim;
}
if (empty($datagov_json)) {
return false;
}
if ($raw == true) {
return $datagov_json;
} else {
return $datagov_json->result->results;
}
}
public function match_agency_slugs()
{
if (php_sapi_name() != 'cli') {
return;
}
$agency_slug_api = 'https://idm.data.gov/fed_agency.json';
$agency_slugs = curl_from_json($agency_slug_api, true);
$agency_slugs = $agency_slugs["taxonomies"];
$this->db->select('id, name');
$this->db->where('no_parent', 'true');
$query = $this->db->get('offices');
if ($query->num_rows() > 0) {
$parent_offices = $query->result();
foreach ($parent_offices as $office) {
$this->run_match($agency_slugs, $office);
// Search for child orgs
$this->db->select('id, name');
$this->db->where('parent_office_id', $office->id);
$child_query = $this->db->get('offices');
if ($child_query->num_rows() > 0) {
$child_offices = $child_query->result();
foreach ($child_offices as $child_office) {
$this->run_match($agency_slugs, $office, $child_office);
}
}
}
}
}
public function validate_json($url = null, $json = null, $headers = null, $schema = null, $return_source = false, $component = null)
{
if ($url) {
$json_header = $headers ? $headers : $this->campaign->uri_header($url);
$errors = array();
// Max file size
$max_remote_size = $this->config->item('max_remote_size');
// Only download the data.json if we need to
if (empty($json_header['download_content_length']) || $json_header['download_content_length'] < 0 || !empty($json_header['download_content_length']) && $json_header['download_content_length'] > 0 && $json_header['download_content_length'] < $max_remote_size) {
if (config_item('proxy_host') && config_item('proxy_port')) {
$ch = curl_init($url);
curl_setopt($ch, CURLOPT_HEADER, 0);
$proxy = config_item('proxy_host') . ":" . config_item('proxy_port');
curl_setopt($ch, CURLOPT_PROXY, $proxy);
curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (compatible; CIO.gov Digital Strategy JSON crawler)');
curl_setopt($ch, CURLOPT_AUTOREFERER, 1);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
$json = curl_exec($ch);
} else {
$ch = curl_init($url);
curl_setopt($ch, CURLOPT_HEADER, 0);
curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (compatible; CIO.gov Digital Strategy JSON crawler)');
curl_setopt($ch, CURLOPT_AUTOREFERER, 1);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
$json = curl_exec($ch);
}
if ($json == false) {
$json = curl_from_json($url, false, false);
if (!$json) {
$errors[] = "File not found or couldn't be downloaded";
}
}
}
if (!empty($json) && (empty($json_header['download_content_length']) || $json_header['download_content_length'] < 0)) {
$json_header['download_content_length'] = strlen($json);
}
// See if it exceeds max size
if ($json_header['download_content_length'] > $max_remote_size) {
//$filesize = human_filesize($json_header['download_content_length']);
//$errors[] = "The data.json file is " . $filesize . " which is currently too large to parse with this tool. Sorry.";
// Increase the timeout limit
@set_time_limit(6000);
$this->load->helper('file');
if ($rawfile = $this->archive_file('json-lines', $this->current_office_id, $json_url)) {
$outfile = $rawfile . '.lines.json';
$stream = fopen($rawfile, 'r');
$out_stream = fopen($outfile, 'w+');
//$listener = new DataJsonParser();
$listener = new JsonStreamingParser_Parser_Listener();
$listener->out_file = $out_stream;
if ($this->environment == 'terminal' or $this->environment == 'cron') {
echo 'Attempting to convert to JSON lines' . PHP_EOL;
}
try {
$parser = new JsonStreamingParser_Parser($stream, $listener);
$parser->parse();
} catch (Exception $e) {
fclose($stream);
throw $e;
}
// Get the dataset count
$json_lines_count = $listener->_array_count;
// Delete temporary raw source file
unlink($rawfile);
$out_stream = fopen($outfile, 'r+');
$chunk_cycle = 0;
$chunk_size = 200;
$chunk_count = intval(ceil($json_lines_count / $chunk_size));
$buffer = '';
$response = array();
$response['errors'] = array();
if ($quality !== false) {
$response['qa'] = array();
}
echo "Analyzing {$json_lines_count} lines in {$chunk_count} chunks of {$chunk_size} lines each" . PHP_EOL;
while ($chunk_cycle < $chunk_count) {
$buffer = '';
$counter = 0;
if ($chunk_cycle > 0) {
$key_offset = $chunk_size * $chunk_cycle;
} else {
$key_offset = 0;
}
$next_offset = $key_offset + $chunk_size;
//echo "Analyzing chunk $chunk_cycle of $chunk_count ($key_offset to $next_offset of $json_lines_count)" . PHP_EOL;
if ($chunk_cycle == 0) {
$json_header = fgets($out_stream);
}
while (($buffer .= fgets($out_stream)) && $counter < $chunk_size) {
$counter++;
}
$buffer = $json_header . $buffer;
$buffer = substr($buffer, 0, strlen($buffer) - 2) . ']}';
$validator = $this->campaign->jsonschema_validator($buffer, 'federal-v1.1');
if (!empty($validator['errors'])) {
$response['errors'] = array_merge($response['errors'], $this->process_validation_errors($validator['errors'], $key_offset));
}
$chunk_cycle++;
}
// Delete json lines file
unlink($outfile);
$response['valid'] = empty($response['errors']) ? true : false;
$response['valid_json'] = true;
$response['total_records'] = $json_lines_count;
if (!empty($json_header['download_content_length'])) {
$response['download_content_length'] = $json_header['download_content_length'];
}
if (empty($response['errors'])) {
$response['errors'] = false;
}
return $response;
} else {
$errors[] = "File not found or couldn't be downloaded";
}
}
// See if it's valid JSON
if (!empty($json) && $json_header['download_content_length'] < $max_remote_size) {
// See if raw file is valid
$raw_valid_json = is_json($json);
// See if we can clean up the file to make it valid
if (!$raw_valid_json) {
$json_processed = json_text_filter($json);
$valid_json = is_json($json_processed);
} else {
$valid_json = true;
}
if ($valid_json !== true) {
$errors[] = 'The validator was unable to determine if this was valid JSON';
}
}
if (!empty($errors)) {
$valid_json = isset($valid_json) ? $valid_json : null;
$raw_valid_json = isset($raw_valid_json) ? $raw_valid_json : null;
$response = array('raw_valid_json' => $raw_valid_json, 'valid_json' => $valid_json, 'valid' => false, 'fail' => $errors, 'download_content_length' => $json_header['download_content_length']);
if ($valid_json && $return_source === false) {
$catalog = json_decode($json_processed);
if ($schema == 'federal-v1.1' or $schema == 'non-federal-v1.1') {
$response['total_records'] = count($catalog->dataset);
} else {
$response['total_records'] = count($catalog);
}
}
return $response;
}
}
// filter string for json conversion if we haven't already
if ($json && empty($json_processed)) {
$json_processed = json_text_filter($json);
}
// verify it's valid json
if ($json_processed) {
if (!isset($valid_json)) {
$valid_json = is_json($json_processed);
}
}
if ($json_processed && $valid_json) {
$json_decode = json_decode($json_processed);
if (!empty($json_decode->conformsTo) && $json_decode->conformsTo == 'https://project-open-data.cio.gov/v1.1/schema') {
if ($schema !== 'federal-v1.1' && $schema !== 'non-federal-v1.1') {
if ($schema == 'federal') {
$schema = 'federal-v1.1';
} else {
if ($schema == 'non-federal') {
$schema = 'non-federal-v1.1';
} else {
$schema = 'federal-v1.1';
}
}
}
$this->schema = $schema;
}
if ($schema == 'federal-v1.1' && empty($json_decode->dataset)) {
$errors[] = "This file does not appear to be using the federal-v1.1 schema";
$response = array('raw_valid_json' => $raw_valid_json, 'valid_json' => $valid_json, 'valid' => false, 'fail' => $errors);
return $response;
}
if ($schema !== 'federal-v1.1' && $schema !== 'non-federal-v1.1') {
$chunk_size = 500;
$json_chunks = array_chunk((array) $json_decode, $chunk_size);
} else {
$json_chunks = array((array) $json_decode);
}
$response = array();
$response['errors'] = array();
// save detected schema version to output
$response['schema_version'] = $schema;
foreach ($json_chunks as $chunk_count => $chunk) {
$chunk = json_encode($chunk);
$validator = $this->campaign->jsonschema_validator($chunk, $schema);
if (!empty($validator['errors'])) {
if ($chunk_count) {
$key_offset = $chunk_size * $chunk_count;
$key_offset = $key_offset;
} else {
$key_offset = 0;
}
$response['errors'] = $response['errors'] + $this->process_validation_errors($validator['errors'], $key_offset);
}
}
$valid_json = isset($raw_valid_json) ? $raw_valid_json : $valid_json;
$response['valid'] = empty($response['errors']) ? true : false;
$response['valid_json'] = $valid_json;
if ($schema == 'federal-v1.1' or $schema == 'non-federal-v1.1') {
$response['total_records'] = count($json_decode->dataset);
} else {
$response['total_records'] = count($json_decode);
}
if (!empty($json_header['download_content_length'])) {
$response['download_content_length'] = $json_header['download_content_length'];
}
if (empty($response['errors'])) {
$response['errors'] = false;
}
if ($return_source) {
$dataset_array = ($schema == 'federal-v1.1' or $schema == 'non-federal-v1.1') ? true : false;
$json_decode = filter_json($json_decode, $dataset_array);
$response['source'] = $json_decode;
}
return $response;
} else {
$errors[] = "This does not appear to be valid JSON";
$response = array('valid_json' => false, 'valid' => false, 'fail' => $errors);
if (!empty($json_header['download_content_length'])) {
$response['download_content_length'] = $json_header['download_content_length'];
}
return $response;
}
}
echo $mime_color;
?>
">
<?php
echo $office_campaign->governanceboard_status->content_type;
?>
</span>
</td>
</tr>
<?php
// TO DO - when we have agency files at a valid url, put the checks back
if (property_exists($office_campaign->governanceboard_status, "valid_json")) {
$governanceboard_valid_json = $office_campaign->governanceboard_status->valid_json;
} else {
if ($governanceboard_http_code == 200 && ($governance_board = curl_from_json($office_campaign->governanceboard_status->url, false, true))) {
$governanceboard_valid_json = true;
} else {
$governanceboard_valid_json = false;
}
}
?>
<tr class="<?php
echo $governanceboard_valid_json == true ? 'success' : 'danger';
?>
">
<th scope="row">Valid JSON</th>
<td>
<span class="text-<?php
echo $governanceboard_valid_json == true ? 'success' : 'danger';
public function assemble_org_structure()
{
$url = 'https://idm.data.gov/fed_agency.json';
$agency_list = curl_from_json($url, $array = true, $decode = true);
$taxonomies = $agency_list['taxonomies'];
$return = array();
// This should be the ONLY loop that go through all taxonomies.
foreach ($taxonomies as $taxonomy) {
$taxonomy = $taxonomy['taxonomy'];
// ignore bad ones
if (strlen($taxonomy['unique id']) == 0) {
continue;
}
// ignore 3rd level ones
if ($taxonomy['unique id'] != $taxonomy['term']) {
continue;
}
// Make sure we got $return[$sector], ex. $return['Federal Organization']
if (!isset($return[$taxonomy['vocabulary']])) {
$return[$taxonomy['vocabulary']] = array();
}
if (strlen($taxonomy['Sub Agency']) != 0) {
// This is sub-agency
// $return['Federal Organization']['National Archives and Records Administration']
if (!isset($return[$taxonomy['vocabulary']][$taxonomy['Federal Agency']])) {
// Make sure we got $return[$sector][$unit]
$return[$taxonomy['vocabulary']][$taxonomy['Federal Agency']] = array('id' => "[," . $taxonomy['unique id'] . "]", 'is_cfo' => $taxonomy['is_cfo'], 'subs' => array());
} else {
// Add sub id to existing agency entry, e.g. [id,sub_id1,sub_id2] or [,sub_id1,sub_id2]
$return[$taxonomy['vocabulary']][$taxonomy['Federal Agency']]['id'] = "[" . trim($return[$taxonomy['vocabulary']][$taxonomy['Federal Agency']]['id'], "[]") . "," . $taxonomy['unique id'] . "]";
}
// Add term to parent's subs
$return[$taxonomy['vocabulary']][$taxonomy['Federal Agency']]['subs'][$taxonomy['Sub Agency']] = array('id' => $taxonomy['unique id'], 'is_cfo' => $taxonomy['is_cfo']);
} else {
// ELSE this is ROOT agency
if (!isset($return[$taxonomy['vocabulary']][$taxonomy['Federal Agency']])) {
// Has not been set by its subunits before
$return[$taxonomy['vocabulary']][$taxonomy['Federal Agency']] = array('id' => $taxonomy['unique id'], 'is_cfo' => $taxonomy['is_cfo'], 'subs' => array());
} else {
// Has been added by subunits before. so let us change it from [,sub_id1,sub_id2] to [id,sub_id1,sub_id2]
$return[$taxonomy['vocabulary']][$taxonomy['Federal Agency']]['id'] = "[" . $taxonomy['unique id'] . trim($return[$taxonomy['vocabulary']][$taxonomy['Federal Agency']]['id'], "[]") . "]";
}
}
}
$orgs = $return['Federal Organization'];
$cfo = array();
$non_cfo = array();
foreach ($orgs as $key => $org) {
$org['name'] = $key;
$id = $org['id'];
$id = str_replace('[,', '(', $id);
$id = str_replace(',]', ')', $id);
$id = str_replace(',', ' OR ', $id);
$id = str_replace('[', '(', $id);
$id = str_replace(']', ')', $id);
$org['id'] = $id;
if ($org['is_cfo'] == 'Y') {
$cfo[$key] = $org;
} else {
$non_cfo[$key] = $org;
}
}
ksort($cfo);
ksort($non_cfo);
$return = array_merge($cfo, $non_cfo);
return $return;
}
">
<th>Content Type</th>
<td>
<span class="text-<?php
echo $mime_color;
?>
">
<?php
echo $office_campaign->digitalstrategy_status->content_type;
?>
</span>
</td>
</tr>
<?php
if ($http_code == 200 && ($digital_strategy = curl_from_json($office_campaign->digitalstrategy_status->url, false, true))) {
$valid_json = true;
} else {
$valid_json = false;
}
?>
<tr class="<?php
echo $valid_json == true ? 'success' : 'danger';
?>
">
<th>Valid JSON</th>
<td>
<span class="text-<?php
public function validate_datajson($datajson_url = null, $datajson = null, $headers = null, $schema = null, $return_source = false, $quality = false, $component = null)
{
if ($datajson_url) {
$datajson_header = $headers ? $headers : $this->campaign->uri_header($datajson_url);
$errors = array();
// Max file size
$max_remote_size = $this->config->item('max_remote_size');
// Only download the data.json if we need to
if (empty($datajson_header['download_content_length']) || $datajson_header['download_content_length'] < 0 || !empty($datajson_header['download_content_length']) && $datajson_header['download_content_length'] > 0 && $datajson_header['download_content_length'] < $max_remote_size) {
// Load the JSON
$opts = array('http' => array('method' => "GET", 'user_agent' => "CIO.gov Digital Strategy JSON crawler"));
$context = stream_context_create($opts);
$datajson = @file_get_contents($datajson_url, false, $context, -1, $max_remote_size + 1);
if ($datajson == false) {
$datajson = curl_from_json($datajson_url, false, false);
if (!$datajson) {
$errors[] = "File not found or couldn't be downloaded";
}
}
}
if (!empty($datajson) && (empty($datajson_header['download_content_length']) || $datajson_header['download_content_length'] < 0)) {
$datajson_header['download_content_length'] = strlen($datajson);
}
// See if it exceeds max size
if ($datajson_header['download_content_length'] > $max_remote_size) {
//$filesize = human_filesize($datajson_header['download_content_length']);
//$errors[] = "The data.json file is " . $filesize . " which is currently too large to parse with this tool. Sorry.";
// Increase the timeout limit
@set_time_limit(6000);
$this->load->helper('file');
if ($rawfile = $this->archive_file('datajson-lines', $this->current_office_id, $datajson_url)) {
$outfile = $rawfile . '.lines.json';
$stream = fopen($rawfile, 'r');
$out_stream = fopen($outfile, 'w+');
$listener = new DataJsonParser();
$listener->out_file = $out_stream;
if ($this->environment == 'terminal' or $this->environment == 'cron') {
echo 'Attempting to convert to JSON lines' . PHP_EOL;
}
try {
$parser = new JsonStreamingParser_Parser($stream, $listener);
$parser->parse();
} catch (Exception $e) {
fclose($stream);
throw $e;
}
// Get the dataset count
$datajson_lines_count = $listener->_array_count;
// Delete temporary raw source file
unlink($rawfile);
$out_stream = fopen($outfile, 'r+');
$chunk_cycle = 0;
$chunk_size = 200;
$chunk_count = intval(ceil($datajson_lines_count / $chunk_size));
$buffer = '';
$response = array();
$response['errors'] = array();
if ($quality !== false) {
$response['qa'] = array();
}
echo "Analyzing {$datajson_lines_count} lines in {$chunk_count} chunks of {$chunk_size} lines each" . PHP_EOL;
while ($chunk_cycle < $chunk_count) {
$buffer = '';
$datajson_qa = null;
$counter = 0;
if ($chunk_cycle > 0) {
$key_offset = $chunk_size * $chunk_cycle;
} else {
$key_offset = 0;
}
$next_offset = $key_offset + $chunk_size;
//echo "Analyzing chunk $chunk_cycle of $chunk_count ($key_offset to $next_offset of $datajson_lines_count)" . PHP_EOL;
if ($chunk_cycle == 0) {
$json_header = fgets($out_stream);
}
while (($buffer .= fgets($out_stream)) && $counter < $chunk_size) {
$counter++;
}
$buffer = $json_header . $buffer;
$buffer = substr($buffer, 0, strlen($buffer) - 2) . ']}';
$validator = $this->campaign->jsonschema_validator($buffer, 'federal-v1.1');
if (!empty($validator['errors'])) {
$response['errors'] = array_merge($response['errors'], $this->process_validation_errors($validator['errors'], $key_offset));
}
if ($quality !== false) {
$datajson_qa = $this->campaign->datajson_qa($buffer, 'federal-v1.1', $quality, $component);
if (!empty($datajson_qa)) {
$response['qa'] = array_merge_recursive($response['qa'], $datajson_qa);
}
}
$chunk_cycle++;
}
// Delete json lines file
unlink($outfile);
// ###################################################################
// Needs to be refactored into separate function
// ###################################################################
// Sum QA counts
if (!empty($response['qa'])) {
if (!empty($response['qa']['bureauCodes'])) {
$response['qa']['bureauCodes'] = array_keys($response['qa']['bureauCodes']);
}
if (!empty($response['qa']['programCodes'])) {
$response['qa']['programCodes'] = array_keys($response['qa']['programCodes']);
}
$sum_array_fields = array('API_total', 'downloadURL_present', 'downloadURL_total', 'accessURL_present', 'accessURL_total', 'accessLevel_public', 'accessLevel_restricted', 'accessLevel_nonpublic', 'license_present', 'redaction_present', 'redaction_no_explanation');
foreach ($sum_array_fields as $array_field) {
if (!empty($response['qa'][$array_field]) && is_array($response['qa'][$array_field])) {
$response['qa'][$array_field] = array_sum($response['qa'][$array_field]);
}
}
// Sum validation counts
if (!empty($response['qa']['validation_counts']) && is_array($response['qa']['validation_counts'])) {
foreach ($response['qa']['validation_counts'] as $validation_key => $validation_count) {
if (is_array($response['qa']['validation_counts'][$validation_key])) {
$response['qa']['validation_counts'][$validation_key] = array_sum($response['qa']['validation_counts'][$validation_key]);
}
}
}
}
$response['valid'] = empty($response['errors']) ? true : false;
$response['valid_json'] = true;
$response['total_records'] = $datajson_lines_count;
if (!empty($datajson_header['download_content_length'])) {
$response['download_content_length'] = $datajson_header['download_content_length'];
}
if (empty($response['errors'])) {
$response['errors'] = false;
}
return $response;
// ###################################################################
} else {
$errors[] = "File not found or couldn't be downloaded";
}
}
// See if it's valid JSON
if (!empty($datajson) && $datajson_header['download_content_length'] < $max_remote_size) {
// See if raw file is valid
$raw_valid_json = is_json($datajson);
// See if we can clean up the file to make it valid
if (!$raw_valid_json) {
$datajson_processed = json_text_filter($datajson);
$valid_json = is_json($datajson_processed);
} else {
$valid_json = true;
}
if ($valid_json !== true) {
$errors[] = 'The validator was unable to determine if this was valid JSON';
}
}
if (!empty($errors)) {
$valid_json = isset($valid_json) ? $valid_json : null;
$raw_valid_json = isset($raw_valid_json) ? $raw_valid_json : null;
$response = array('raw_valid_json' => $raw_valid_json, 'valid_json' => $valid_json, 'valid' => false, 'fail' => $errors, 'download_content_length' => $datajson_header['download_content_length']);
if ($valid_json && $return_source === false) {
$catalog = json_decode($datajson_processed);
if ($schema == 'federal-v1.1' or $schema == 'non-federal-v1.1') {
$response['total_records'] = count($catalog->dataset);
} else {
$response['total_records'] = count($catalog);
}
}
return $response;
}
}
// filter string for json conversion if we haven't already
if ($datajson && empty($datajson_processed)) {
$datajson_processed = json_text_filter($datajson);
}
// verify it's valid json
if ($datajson_processed) {
if (!isset($valid_json)) {
$valid_json = is_json($datajson_processed);
}
}
if ($datajson_processed && $valid_json) {
$datajson_decode = json_decode($datajson_processed);
if (!empty($datajson_decode->conformsTo) && $datajson_decode->conformsTo == 'https://project-open-data.cio.gov/v1.1/schema') {
if ($schema !== 'federal-v1.1' && $schema !== 'non-federal-v1.1') {
if ($schema == 'federal') {
$schema = 'federal-v1.1';
} else {
if ($schema == 'non-federal') {
$schema = 'non-federal-v1.1';
} else {
$schema = 'federal-v1.1';
}
}
}
$this->schema = $schema;
}
if ($schema == 'federal-v1.1' && empty($datajson_decode->dataset)) {
$errors[] = "This file does not appear to be using the federal-v1.1 schema";
$response = array('raw_valid_json' => $raw_valid_json, 'valid_json' => $valid_json, 'valid' => false, 'fail' => $errors);
return $response;
}
if ($schema !== 'federal-v1.1' && $schema !== 'non-federal-v1.1') {
$chunk_size = 500;
$datajson_chunks = array_chunk((array) $datajson_decode, $chunk_size);
} else {
$datajson_chunks = array((array) $datajson_decode);
}
$response = array();
$response['errors'] = array();
/*
if ($quality !== false) {
$response['qa'] = array();
}
*/
// save detected schema version to output
$response['schema_version'] = $schema;
foreach ($datajson_chunks as $chunk_count => $chunk) {
$chunk = json_encode($chunk);
$validator = $this->campaign->jsonschema_validator($chunk, $schema);
if (!empty($validator['errors'])) {
if ($chunk_count) {
$key_offset = $chunk_size * $chunk_count;
$key_offset = $key_offset;
} else {
$key_offset = 0;
}
$response['errors'] = $response['errors'] + $this->process_validation_errors($validator['errors'], $key_offset);
}
/*
if ($quality !== false) {
$datajson_qa = $this->campaign->datajson_qa($chunk, $schema, $quality, $component);
if (!empty($datajson_qa)) {
$response['qa'] = array_merge_recursive($response['qa'], $datajson_qa);
}
}
*/
}
// Sum QA counts
if (!empty($response['qa'])) {
if (!empty($response['qa']['bureauCodes'])) {
$response['qa']['bureauCodes'] = array_keys($response['qa']['bureauCodes']);
}
if (!empty($response['qa']['programCodes'])) {
$response['qa']['programCodes'] = array_keys($response['qa']['programCodes']);
}
$sum_array_fields = array('accessURL_present', 'accessURL_total', 'accessLevel_public', 'accessLevel_restricted', 'accessLevel_nonpublic');
foreach ($sum_array_fields as $array_field) {
if (!empty($response['qa'][$array_field]) && is_array($response['qa'][$array_field])) {
$response['qa'][$array_field] = array_sum($response['qa'][$array_field]);
}
}
// Sum validation counts
if (!empty($response['qa']['validation_counts']) && is_array($response['qa']['validation_counts'])) {
foreach ($response['qa']['validation_counts'] as $validation_key => $validation_count) {
if (is_array($response['qa']['validation_counts'][$validation_key])) {
$response['qa']['validation_counts'][$validation_key] = array_sum($response['qa']['validation_counts'][$validation_key]);
}
}
}
}
$valid_json = isset($raw_valid_json) ? $raw_valid_json : $valid_json;
$response['valid'] = empty($response['errors']) ? true : false;
$response['valid_json'] = $valid_json;
if ($schema == 'federal-v1.1' or $schema == 'non-federal-v1.1') {
$response['total_records'] = count($datajson_decode->dataset);
} else {
$response['total_records'] = count($datajson_decode);
}
if (!empty($datajson_header['download_content_length'])) {
$response['download_content_length'] = $datajson_header['download_content_length'];
}
if (empty($response['errors'])) {
$response['errors'] = false;
}
if ($return_source) {
$dataset_array = ($schema == 'federal-v1.1' or $schema == 'non-federal-v1.1') ? true : false;
$datajson_decode = filter_json($datajson_decode, $dataset_array);
$response['source'] = $datajson_decode;
}
return $response;
} else {
$errors[] = "This does not appear to be valid JSON";
$response = array('valid_json' => false, 'valid' => false, 'fail' => $errors);
if (!empty($datajson_header['download_content_length'])) {
$response['download_content_length'] = $datajson_header['download_content_length'];
}
return $response;
}
}