function start_process($resource_id, $call_multiple_instance, $connectors_to_run = 1) { $this->resource_id = $resource_id; $this->call_multiple_instance = $call_multiple_instance; $this->connectors_to_run = $connectors_to_run; if (!trim(Functions::get_a_task($this->WORK_IN_PROGRESS_LIST))) { if (!trim(Functions::get_a_task($this->INITIAL_PROCESS_STATUS))) { Functions::add_a_task("Initial process start", $this->INITIAL_PROCESS_STATUS); // this will prepare a list of all species id; 13 mins. execution self::build_id_list(); // divides the big list of ids into small files self::divide_text_file(10000); //debug orig 10000, for testing use 5 Functions::delete_a_task("Initial process start", $this->INITIAL_PROCESS_STATUS); //remove a task from task list } } Functions::process_work_list($this); if (!($task = trim(Functions::get_a_task($this->WORK_IN_PROGRESS_LIST)))) { // step 3: this should only run when all of instances of step 2 are done sleep(10); //debug orig 10 Functions::combine_all_eol_resource_xmls($resource_id, $this->TEMP_FILE_PATH . "temp_tropicos_batch_*.xml"); Functions::delete_temp_files($this->TEMP_FILE_PATH . "temp_tropicos_batch_", "xml"); //debug comment this line if u want to have a source for checking encoding probs in the XML Functions::delete_temp_files($this->TEMP_FILE_PATH . "batch_", "txt"); Functions::set_resource_status_to_force_harvest($resource_id); } }
function start_process($resource_id, $call_multiple_instance) { $this->resource_id = $resource_id; $this->call_multiple_instance = $call_multiple_instance; $this->connectors_to_run = 1; if (!trim(Functions::get_a_task($this->WORK_IN_PROGRESS_LIST))) { if (!trim(Functions::get_a_task($this->INITIAL_PROCESS_STATUS))) { // Divide the big list of ids into small files Functions::add_a_task("Initial process start", $this->INITIAL_PROCESS_STATUS); self::divide_text_file(10000); //orig value 10000 debug Functions::delete_a_task("Initial process start", $this->INITIAL_PROCESS_STATUS); } } Functions::process_work_list($this); if (!($task = trim(Functions::get_a_task($this->WORK_IN_PROGRESS_LIST)))) { // Combine all XML files. Functions::combine_all_eol_resource_xmls($resource_id, $this->TEMP_FILE_PATH . "temp_DiscoverLife_batch_*.xml"); // Set to force harvest if (filesize(CONTENT_RESOURCE_LOCAL_PATH . $resource_id . ".xml")) { $GLOBALS['db_connection']->update("UPDATE resources SET resource_status_id=" . ResourceStatus::force_harvest()->id . " WHERE id=" . $resource_id); } // Delete temp files Functions::delete_temp_files($this->TEMP_FILE_PATH . "batch_", "txt"); Functions::delete_temp_files($this->TEMP_FILE_PATH . "temp_DiscoverLife_" . "batch_", "xml"); } }
function start_process($resource_id, $call_multiple_instance) { self::$TEMP_FILE_PATH = DOC_ROOT . "/update_resources/connectors/files/DiscoverLife/"; self::$WORK_LIST = DOC_ROOT . "/update_resources/connectors/files/DiscoverLife/work_list.txt"; self::$WORK_IN_PROGRESS_LIST = DOC_ROOT . "/update_resources/connectors/files/DiscoverLife/work_in_progress_list.txt"; self::$INITIAL_PROCESS_STATUS = DOC_ROOT . "/update_resources/connectors/files/DiscoverLife/initial_process_status.txt"; self::$TEXT_FILE_FOR_DL = DOC_ROOT . "/update_resources/connectors/files/DiscoverLife/names_without_pages_in_eol.txt"; //report back to DiscoverLife if (!trim(Functions::get_a_task(self::$WORK_IN_PROGRESS_LIST))) { if (!trim(Functions::get_a_task(self::$INITIAL_PROCESS_STATUS))) { // Divide the big list of ids into small files Functions::add_a_task("Initial process start", self::$INITIAL_PROCESS_STATUS); self::divide_text_file(10000); //orig value 10000 Functions::delete_a_task("Initial process start", self::$INITIAL_PROCESS_STATUS); } } // Run multiple instances, for DiscoverLife ideally a total of 2 while (true) { $task = Functions::get_a_task(self::$WORK_LIST); //get a task to work on if ($task) { print "\n Process this: {$task}"; Functions::delete_a_task($task, self::$WORK_LIST); Functions::add_a_task($task, self::$WORK_IN_PROGRESS_LIST); $task = str_ireplace("\n", "", $task); //remove carriage return got from text file if ($call_multiple_instance) { Functions::run_another_connector_instance($resource_id, 1); //call 1 other instance for a total of 2 instances running $call_multiple_instance = 0; } self::get_all_taxa($task); print "\n Task {$task} is done. \n"; Functions::delete_a_task("{$task}\n", self::$WORK_IN_PROGRESS_LIST); //remove a task from task list } else { print "\n\n [{$task}] Work list done --- " . date('Y-m-d h:i:s a', time()) . "\n"; break; } } if (!($task = trim(Functions::get_a_task(self::$WORK_IN_PROGRESS_LIST)))) { // Combine all XML files. self::combine_all_xmls($resource_id); // Set to force harvest if (filesize(CONTENT_RESOURCE_LOCAL_PATH . $resource_id . ".xml")) { $GLOBALS['db_connection']->update("UPDATE resources SET resource_status_id=" . ResourceStatus::insert('Force Harvest') . " WHERE id=" . $resource_id); } // Delete temp files self::delete_temp_files(self::$TEMP_FILE_PATH . "batch_", "txt"); self::delete_temp_files(CONTENT_RESOURCE_LOCAL_PATH . "DiscoverLife/temp_DiscoverLife_" . "batch_", "xml"); } }
function start_process($resource_id, $call_multiple_instance) { $this->resource_id = $resource_id; $this->call_multiple_instance = $call_multiple_instance; $this->connectors_to_run = 1; if (!trim(Functions::get_a_task($this->WORK_IN_PROGRESS_LIST))) { if (!trim(Functions::get_a_task($this->INITIAL_PROCESS_STATUS))) { // Divide the big list of ids into small files Functions::add_a_task("Initial process start", $this->INITIAL_PROCESS_STATUS); $batch = Functions::create_work_list_from_master_file($this->MASTER_LIST, 5000, $this->TEMP_FILE_PATH, "batch_", $this->WORK_LIST); //debug orig value 5000 Functions::delete_a_task("Initial process start", $this->INITIAL_PROCESS_STATUS); } } Functions::process_work_list($this, $batch); if (!($task = trim(Functions::get_a_task($this->WORK_IN_PROGRESS_LIST)))) { // Combine all XML files. Functions::combine_all_eol_resource_xmls($resource_id, $this->TEMP_FILE_PATH . "temp_Bolds_batch_*.xml"); // Delete temp files Functions::delete_temp_files($this->TEMP_FILE_PATH . "batch_", "txt"); Functions::delete_temp_files($this->TEMP_FILE_PATH . "temp_Bolds_" . "batch_", "xml"); } }
function start_process($resource_id, $call_multiple_instance, $connectors_to_run = 1) { require_library('connectors/BOLDSysAPI'); $this->func = new BOLDSysAPI(); $this->resource_id = $resource_id; $this->call_multiple_instance = $call_multiple_instance; $this->connectors_to_run = $connectors_to_run; if (!trim(Functions::get_a_task($this->WORK_IN_PROGRESS_LIST))) { if (!trim(Functions::get_a_task($this->INITIAL_PROCESS_STATUS))) { // Divide the big list of ids into small files Functions::add_a_task("Initial process start", $this->INITIAL_PROCESS_STATUS); $this->func->create_master_list(); Functions::delete_a_task("Initial process start", $this->INITIAL_PROCESS_STATUS); } } Functions::process_work_list($this); if (!($task = trim(Functions::get_a_task($this->WORK_IN_PROGRESS_LIST)))) { $this->archive_builder->finalize(true); // Set to force harvest Functions::set_resource_status_to_force_harvest($resource_id); // Delete temp files Functions::delete_temp_files($this->TEMP_FILE_PATH . "sl_batch_", "txt"); } }
function start_process($resource_id, $call_multiple_instance, $connectors_to_run = 1) { $this->resource_id = $resource_id; $this->call_multiple_instance = $call_multiple_instance; $this->connectors_to_run = $connectors_to_run; if (!trim(Functions::get_a_task($this->WORK_IN_PROGRESS_LIST))) { if (!trim(Functions::get_a_task($this->INITIAL_PROCESS_STATUS))) { // Divide the big list of ids into small files Functions::add_a_task("Initial process start", $this->INITIAL_PROCESS_STATUS); self::create_master_list(); Functions::delete_a_task("Initial process start", $this->INITIAL_PROCESS_STATUS); } } Functions::process_work_list($this); if (!($task = trim(Functions::get_a_task($this->WORK_IN_PROGRESS_LIST)))) { // Combine all XML files. Functions::combine_all_eol_resource_xmls($resource_id, $this->TEMP_FILE_PATH . "sl_batch_*.xml"); // Set to force harvest Functions::set_resource_status_to_force_harvest($resource_id); // Delete temp files Functions::delete_temp_files($this->TEMP_FILE_PATH . "sl_batch_", "txt"); Functions::delete_temp_files($this->TEMP_FILE_PATH . "sl_batch_", "xml"); //debug Don't delete it if you want to check subsets of the resource XML. } }
function start_process($resource_id, $call_multiple_instance) { $this->resource_id = $resource_id; $this->call_multiple_instance = $call_multiple_instance; $this->connectors_to_run = 1; if (!trim(Functions::get_a_task($this->WORK_IN_PROGRESS_LIST))) { if (!trim(Functions::get_a_task($this->INITIAL_PROCESS_STATUS))) { Functions::add_a_task("Initial process start", $this->INITIAL_PROCESS_STATUS); // step 1: divides the big list of ids into small files $ids = self::get_id_list(); self::divide_text_file(10000, $ids); //debug original value 10000 Functions::delete_a_task("Initial process start", $this->INITIAL_PROCESS_STATUS); //removes a task from task list } } Functions::process_work_list($this); if (!($task = trim(Functions::get_a_task($this->WORK_IN_PROGRESS_LIST)))) { // step 3: Combine all XML files. This only runs when all of instances of step 2 are done self::combine_all_xmls($resource_id); // set to force harvest if (filesize(CONTENT_RESOURCE_LOCAL_PATH . $resource_id . ".xml")) { $GLOBALS['db_connection']->update("UPDATE resources SET resource_status_id=" . ResourceStatus::force_harvest()->id . " WHERE id=" . $resource_id); } // delete temp files Functions::delete_temp_files($this->TEMP_FILE_PATH . "batch_", "txt"); Functions::delete_temp_files($this->TEMP_FILE_PATH . "temp_worms_" . "batch_", "xml"); } self::save_bad_ids_to_txt(); }
public static function process_work_list($class, $batch = null) { while (true) { $task = Functions::get_a_task($class->WORK_LIST); //get task to work on if ($task) { print "\n Process this: {$task}"; Functions::delete_a_task($task, $class->WORK_LIST); //remove a task from task list Functions::add_a_task($task, $class->WORK_IN_PROGRESS_LIST); print "{$task} \n"; $task = str_ireplace("\n", "", $task); //remove carriage return got from text file if ($class->call_multiple_instance) { Functions::run_another_connector_instance($class->resource_id, $class->connectors_to_run); $class->call_multiple_instance = 0; } $class->get_all_taxa($task, $class->TEMP_FILE_PATH, array($batch, $task)); //main connector body print "\n Task {$task} is done. \n"; Functions::delete_a_task("{$task}\n", $class->WORK_IN_PROGRESS_LIST); //remove a task from task list } else { print "\n\n [{$task}] Work list done or list hasn't been created yet " . date('Y-m-d h:i:s a', time()); break; } } }