function start_process($resource_id, $call_multiple_instance) { $this->resource_id = $resource_id; $this->call_multiple_instance = $call_multiple_instance; $this->connectors_to_run = 1; if (!trim(Functions::get_a_task($this->WORK_IN_PROGRESS_LIST))) { if (!trim(Functions::get_a_task($this->INITIAL_PROCESS_STATUS))) { // Divide the big list of ids into small files Functions::add_a_task("Initial process start", $this->INITIAL_PROCESS_STATUS); self::divide_text_file(10000); //orig value 10000 debug Functions::delete_a_task("Initial process start", $this->INITIAL_PROCESS_STATUS); } } Functions::process_work_list($this); if (!($task = trim(Functions::get_a_task($this->WORK_IN_PROGRESS_LIST)))) { // Combine all XML files. Functions::combine_all_eol_resource_xmls($resource_id, $this->TEMP_FILE_PATH . "temp_DiscoverLife_batch_*.xml"); // Set to force harvest if (filesize(CONTENT_RESOURCE_LOCAL_PATH . $resource_id . ".xml")) { $GLOBALS['db_connection']->update("UPDATE resources SET resource_status_id=" . ResourceStatus::force_harvest()->id . " WHERE id=" . $resource_id); } // Delete temp files Functions::delete_temp_files($this->TEMP_FILE_PATH . "batch_", "txt"); Functions::delete_temp_files($this->TEMP_FILE_PATH . "temp_DiscoverLife_" . "batch_", "xml"); } }
private function set_resource_status_to_force_harvest() { // the resource XML response declaration is 516 bytes, so we're checking for something // slightly larger than that to make sure we don't have a file with a response // and no content if (filesize(CONTENT_RESOURCE_LOCAL_PATH . $this->resource_id . ".xml") > 600) { $GLOBALS['db_connection']->update("UPDATE resources SET resource_status_id=" . ResourceStatus::force_harvest()->id . " WHERE id=" . $this->resource_id); } }
public function get_all_taxa($resource_id) { // Delete temp files, possible remnants from interrupted runs Functions::delete_temp_files($this->OBIS_DATA_PATH . "temp_obis_", "xml"); Functions::delete_temp_files($this->OBIS_DATA_PATH . "temp_", "csv"); //divide big file to a more consumable chunks $file_count = self::divide_big_csv_file(40000); //debug orig is 40000 if ($file_count === false) { return false; } $all_taxa = array(); $used_collection_ids = array(); for ($i = 1; $i <= $file_count; $i++) { echo "\nprocessing {$i} => \n"; $arr = self::get_obis_taxa($this->OBIS_DATA_PATH . "temp_" . $i . ".csv", $used_collection_ids); $page_taxa = $arr[0]; $used_collection_ids = $arr[1]; $xml = \SchemaDocument::get_taxon_xml($page_taxa); $resource_path = $this->OBIS_DATA_PATH . "temp_obis_" . $i . ".xml"; if (!($OUT = Functions::file_open($resource_path, "w"))) { return; } fwrite($OUT, $xml); fclose($OUT); } // Combine all XML files. Functions::combine_all_eol_resource_xmls($resource_id, $this->OBIS_DATA_PATH . "temp_obis_*.xml"); // Set to force harvest if (filesize(CONTENT_RESOURCE_LOCAL_PATH . $resource_id . ".xml")) { $GLOBALS['db_connection']->update("UPDATE resources SET resource_status_id=" . ResourceStatus::force_harvest()->id . " WHERE id=" . $resource_id); } // Delete temp files Functions::delete_temp_files($this->OBIS_DATA_PATH . "temp_obis_", "xml"); Functions::delete_temp_files($this->OBIS_DATA_PATH . "temp_", "csv"); }
function start_process($resource_id, $call_multiple_instance) { $this->resource_id = $resource_id; $this->call_multiple_instance = $call_multiple_instance; $this->connectors_to_run = 1; if (!trim(Functions::get_a_task($this->WORK_IN_PROGRESS_LIST))) { if (!trim(Functions::get_a_task($this->INITIAL_PROCESS_STATUS))) { Functions::add_a_task("Initial process start", $this->INITIAL_PROCESS_STATUS); // step 1: divides the big list of ids into small files $ids = self::get_id_list(); self::divide_text_file(10000, $ids); //debug original value 10000 Functions::delete_a_task("Initial process start", $this->INITIAL_PROCESS_STATUS); //removes a task from task list } } Functions::process_work_list($this); if (!($task = trim(Functions::get_a_task($this->WORK_IN_PROGRESS_LIST)))) { // step 3: Combine all XML files. This only runs when all of instances of step 2 are done self::combine_all_xmls($resource_id); // set to force harvest if (filesize(CONTENT_RESOURCE_LOCAL_PATH . $resource_id . ".xml")) { $GLOBALS['db_connection']->update("UPDATE resources SET resource_status_id=" . ResourceStatus::force_harvest()->id . " WHERE id=" . $resource_id); } // delete temp files Functions::delete_temp_files($this->TEMP_FILE_PATH . "batch_", "txt"); Functions::delete_temp_files($this->TEMP_FILE_PATH . "temp_worms_" . "batch_", "xml"); } self::save_bad_ids_to_txt(); }
public static function set_resource_status_to_force_harvest($resource_id) { if (file_exists(CONTENT_RESOURCE_LOCAL_PATH . $resource_id . ".xml")) { if (filesize(CONTENT_RESOURCE_LOCAL_PATH . $resource_id . ".xml") > 600) { $GLOBALS['db_connection']->update("UPDATE resources SET resource_status_id=" . ResourceStatus::force_harvest()->id . " WHERE id=" . $resource_id); } } elseif (file_exists(CONTENT_RESOURCE_LOCAL_PATH . "/{$resource_id}/taxon.tab")) { if (filesize(CONTENT_RESOURCE_LOCAL_PATH . "/{$resource_id}/taxon.tab") > 600) { $GLOBALS['db_connection']->update("UPDATE resources SET resource_status_id=" . ResourceStatus::force_harvest()->id . " WHERE id=" . $resource_id); } } }
} } foreach ($used_taxa as $taxon_parameters) { $schema_taxa[] = new \SchemaTaxon($taxon_parameters); } $new_resource_xml = \SchemaDocument::get_taxon_xml($schema_taxa); $old_resource_path = CONTENT_RESOURCE_LOCAL_PATH . $resource_id . ".xml"; if (!($OUT = fopen($old_resource_path, "w+"))) { debug(__CLASS__ . ":" . __LINE__ . ": Couldn't open file: " . $old_resource_path); return; } fwrite($OUT, $new_resource_xml); fclose($OUT); // set MorphBank to force harvest if (filesize(CONTENT_RESOURCE_LOCAL_PATH . $resource_id . ".xml") > 600) { $GLOBALS['db_connection']->update("UPDATE resources SET resource_status_id=" . ResourceStatus::force_harvest()->id . " WHERE id=" . $resource_id); } $elapsed_time_sec = time_elapsed() - $timestart; echo "\n"; echo "elapsed time = " . $elapsed_time_sec / 60 . " minutes \n"; echo "elapsed time = " . $elapsed_time_sec / 60 / 60 . " hours \n"; echo "\n\n Done processing."; function get_data_object($id, $created, $modified, $rightsHolder, $license, $agent, $description, $type) { $dataObjectParameters = array(); if ($type == "text") { $dataObjectParameters["identifier"] = "txt_" . $id; $dataObjectParameters["title"] = "Specimen Info"; $dataObjectParameters["subjects"] = array(); $subjectParameters = array(); $subjectParameters["label"] = "http://rs.tdwg.org/ontology/voc/SPMInfoItems#GeneralDescription";