function start_process($resource_id, $call_multiple_instance)
{
$this->resource_id = $resource_id;
$this->call_multiple_instance = $call_multiple_instance;
$this->connectors_to_run = 1;
if (!trim(Functions::get_a_task($this->WORK_IN_PROGRESS_LIST))) {
if (!trim(Functions::get_a_task($this->INITIAL_PROCESS_STATUS))) {
// Divide the big list of ids into small files
Functions::add_a_task("Initial process start", $this->INITIAL_PROCESS_STATUS);
self::divide_text_file(10000);
//orig value 10000 debug
Functions::delete_a_task("Initial process start", $this->INITIAL_PROCESS_STATUS);
}
}
Functions::process_work_list($this);
if (!($task = trim(Functions::get_a_task($this->WORK_IN_PROGRESS_LIST)))) {
// Combine all XML files.
Functions::combine_all_eol_resource_xmls($resource_id, $this->TEMP_FILE_PATH . "temp_DiscoverLife_batch_*.xml");
// Set to force harvest
if (filesize(CONTENT_RESOURCE_LOCAL_PATH . $resource_id . ".xml")) {
$GLOBALS['db_connection']->update("UPDATE resources SET resource_status_id=" . ResourceStatus::force_harvest()->id . " WHERE id=" . $resource_id);
}
// Delete temp files
Functions::delete_temp_files($this->TEMP_FILE_PATH . "batch_", "txt");
Functions::delete_temp_files($this->TEMP_FILE_PATH . "temp_DiscoverLife_" . "batch_", "xml");
}
}
private function set_resource_status_to_force_harvest()
{
// the resource XML response declaration is 516 bytes, so we're checking for something
// slightly larger than that to make sure we don't have a file with a response
// and no content
if (filesize(CONTENT_RESOURCE_LOCAL_PATH . $this->resource_id . ".xml") > 600) {
$GLOBALS['db_connection']->update("UPDATE resources SET resource_status_id=" . ResourceStatus::force_harvest()->id . " WHERE id=" . $this->resource_id);
}
}
public function get_all_taxa($resource_id)
{
// Delete temp files, possible remnants from interrupted runs
Functions::delete_temp_files($this->OBIS_DATA_PATH . "temp_obis_", "xml");
Functions::delete_temp_files($this->OBIS_DATA_PATH . "temp_", "csv");
//divide big file to a more consumable chunks
$file_count = self::divide_big_csv_file(40000);
//debug orig is 40000
if ($file_count === false) {
return false;
}
$all_taxa = array();
$used_collection_ids = array();
for ($i = 1; $i <= $file_count; $i++) {
echo "\nprocessing {$i} => \n";
$arr = self::get_obis_taxa($this->OBIS_DATA_PATH . "temp_" . $i . ".csv", $used_collection_ids);
$page_taxa = $arr[0];
$used_collection_ids = $arr[1];
$xml = \SchemaDocument::get_taxon_xml($page_taxa);
$resource_path = $this->OBIS_DATA_PATH . "temp_obis_" . $i . ".xml";
if (!($OUT = Functions::file_open($resource_path, "w"))) {
return;
}
fwrite($OUT, $xml);
fclose($OUT);
}
// Combine all XML files.
Functions::combine_all_eol_resource_xmls($resource_id, $this->OBIS_DATA_PATH . "temp_obis_*.xml");
// Set to force harvest
if (filesize(CONTENT_RESOURCE_LOCAL_PATH . $resource_id . ".xml")) {
$GLOBALS['db_connection']->update("UPDATE resources SET resource_status_id=" . ResourceStatus::force_harvest()->id . " WHERE id=" . $resource_id);
}
// Delete temp files
Functions::delete_temp_files($this->OBIS_DATA_PATH . "temp_obis_", "xml");
Functions::delete_temp_files($this->OBIS_DATA_PATH . "temp_", "csv");
}
function start_process($resource_id, $call_multiple_instance)
{
$this->resource_id = $resource_id;
$this->call_multiple_instance = $call_multiple_instance;
$this->connectors_to_run = 1;
if (!trim(Functions::get_a_task($this->WORK_IN_PROGRESS_LIST))) {
if (!trim(Functions::get_a_task($this->INITIAL_PROCESS_STATUS))) {
Functions::add_a_task("Initial process start", $this->INITIAL_PROCESS_STATUS);
// step 1: divides the big list of ids into small files
$ids = self::get_id_list();
self::divide_text_file(10000, $ids);
//debug original value 10000
Functions::delete_a_task("Initial process start", $this->INITIAL_PROCESS_STATUS);
//removes a task from task list
}
}
Functions::process_work_list($this);
if (!($task = trim(Functions::get_a_task($this->WORK_IN_PROGRESS_LIST)))) {
// step 3: Combine all XML files. This only runs when all of instances of step 2 are done
self::combine_all_xmls($resource_id);
// set to force harvest
if (filesize(CONTENT_RESOURCE_LOCAL_PATH . $resource_id . ".xml")) {
$GLOBALS['db_connection']->update("UPDATE resources SET resource_status_id=" . ResourceStatus::force_harvest()->id . " WHERE id=" . $resource_id);
}
// delete temp files
Functions::delete_temp_files($this->TEMP_FILE_PATH . "batch_", "txt");
Functions::delete_temp_files($this->TEMP_FILE_PATH . "temp_worms_" . "batch_", "xml");
}
self::save_bad_ids_to_txt();
}
public static function set_resource_status_to_force_harvest($resource_id)
{
if (file_exists(CONTENT_RESOURCE_LOCAL_PATH . $resource_id . ".xml")) {
if (filesize(CONTENT_RESOURCE_LOCAL_PATH . $resource_id . ".xml") > 600) {
$GLOBALS['db_connection']->update("UPDATE resources SET resource_status_id=" . ResourceStatus::force_harvest()->id . " WHERE id=" . $resource_id);
}
} elseif (file_exists(CONTENT_RESOURCE_LOCAL_PATH . "/{$resource_id}/taxon.tab")) {
if (filesize(CONTENT_RESOURCE_LOCAL_PATH . "/{$resource_id}/taxon.tab") > 600) {
$GLOBALS['db_connection']->update("UPDATE resources SET resource_status_id=" . ResourceStatus::force_harvest()->id . " WHERE id=" . $resource_id);
}
}
}
}
}
foreach ($used_taxa as $taxon_parameters) {
$schema_taxa[] = new \SchemaTaxon($taxon_parameters);
}
$new_resource_xml = \SchemaDocument::get_taxon_xml($schema_taxa);
$old_resource_path = CONTENT_RESOURCE_LOCAL_PATH . $resource_id . ".xml";
if (!($OUT = fopen($old_resource_path, "w+"))) {
debug(__CLASS__ . ":" . __LINE__ . ": Couldn't open file: " . $old_resource_path);
return;
}
fwrite($OUT, $new_resource_xml);
fclose($OUT);
// set MorphBank to force harvest
if (filesize(CONTENT_RESOURCE_LOCAL_PATH . $resource_id . ".xml") > 600) {
$GLOBALS['db_connection']->update("UPDATE resources SET resource_status_id=" . ResourceStatus::force_harvest()->id . " WHERE id=" . $resource_id);
}
$elapsed_time_sec = time_elapsed() - $timestart;
echo "\n";
echo "elapsed time = " . $elapsed_time_sec / 60 . " minutes \n";
echo "elapsed time = " . $elapsed_time_sec / 60 / 60 . " hours \n";
echo "\n\n Done processing.";
function get_data_object($id, $created, $modified, $rightsHolder, $license, $agent, $description, $type)
{
$dataObjectParameters = array();
if ($type == "text") {
$dataObjectParameters["identifier"] = "txt_" . $id;
$dataObjectParameters["title"] = "Specimen Info";
$dataObjectParameters["subjects"] = array();
$subjectParameters = array();
$subjectParameters["label"] = "http://rs.tdwg.org/ontology/voc/SPMInfoItems#GeneralDescription";
