public static function run_connection($args) { if (pts_client::create_lock(PTS_USER_PATH . 'phoromatic_lock') == false) { trigger_error('Phoromatic is already running.', E_USER_ERROR); return false; } define('PHOROMATIC_PROCESS', true); if (pts_client::$pts_logger == false) { pts_client::$pts_logger = new pts_logger(); } pts_client::$pts_logger->log(pts_title(true) . ' [' . PTS_CORE_VERSION . '] starting Phoromatic client'); if (phodevi::system_uptime() < 60) { echo 'PHOROMATIC: Sleeping for 60 seconds as system freshly started.' . PHP_EOL; pts_client::$pts_logger->log('Sleeping for 60 seconds as system freshly started'); sleep(60); } $server_setup = self::setup_server_addressing($args); //$http_comm = new phoromatic_client_comm_http(); if (!$server_setup) { if (PTS_IS_DAEMONIZED_SERVER_PROCESS) { if (pts_client::executable_in_path('reboot')) { shell_exec('reboot'); sleep(5); } } return false; } $times_failed = 0; $has_success = false; $do_exit = false; $just_started = true; self::setup_system_environment(); pts_client::$pts_logger->log('SYSTEM HARDWARE: ' . phodevi::system_hardware(true)); pts_client::$pts_logger->log('SYSTEM SOFTWARE: ' . phodevi::system_software(true)); while ($do_exit == false) { $server_response = phoromatic::upload_to_remote_server(array('r' => 'start')); if ($server_response == false) { $times_failed++; pts_client::$pts_logger->log('Server response failed'); if ($times_failed >= 2) { trigger_error('Communication with server failed.', E_USER_ERROR); if (PTS_IS_DAEMONIZED_SERVER_PROCESS == false && $times_failed > 5) { return false; } else { if (PTS_IS_DAEMONIZED_SERVER_PROCESS && $times_failed > 10) { if (pts_client::executable_in_path('reboot')) { shell_exec('reboot'); sleep(5); } } } } } else { if (substr($server_response, 0, 1) == '[') { // Likely a notice/warning from server echo PHP_EOL . substr($server_response, 0, strpos($server_response, PHP_EOL)) . PHP_EOL; } else { if (substr($server_response, 0, 1) == '{') { $times_failed = 0; $json = json_decode($server_response, true); if ($has_success == false) { $has_success = true; pts_module::save_file('last-phoromatic-server', self::$server_address . ':' . self::$server_http_port . '/' . self::$account_id); } if ($json != null) { if (isset($json['phoromatic']['error']) && !empty($json['phoromatic']['error'])) { trigger_error($json['phoromatic']['error'], E_USER_ERROR); } if (isset($json['phoromatic']['response']) && !empty($json['phoromatic']['response'])) { echo PHP_EOL . $json['phoromatic']['response'] . PHP_EOL; } } if ($just_started) { if (PTS_IS_DAEMONIZED_SERVER_PROCESS) { $pid = pcntl_fork(); if ($pid == 0) { // Start the tick thread self::tick_thread(); } } $just_started = false; } if (isset($json['phoromatic']['pre_set_sys_env_vars']) && !empty($json['phoromatic']['pre_set_sys_env_vars'])) { // pre_set_sys_env_vars was added during PTS 5.8 development // Sets environment variables on client as specified via the Phoromatic Server's systems page foreach (explode(';', $json['phoromatic']['pre_set_sys_env_vars']) as $i => $v_string) { $var = explode('=', $v_string); if (count($var) == 2) { putenv($var[0] . '=' . $var[1]); } } } switch (isset($json['phoromatic']['task']) ? $json['phoromatic']['task'] : null) { case 'install': phoromatic::update_system_status('Installing Tests'); pts_suite_nye_XmlReader::set_temporary_suite('pre-seed', $json['phoromatic']['test_suite']); pts_test_installer::standard_install('pre-seed'); break; case 'benchmark': // Make sure all latest tests are available pts_openbenchmarking::refresh_repository_lists(null, true); $benchmark_timer = time(); self::$is_running_as_phoromatic_node = true; $test_flags = pts_c::auto_mode | pts_c::batch_mode; $suite_identifier = sha1(time() . rand(2, 1000)); pts_suite_nye_XmlReader::set_temporary_suite($suite_identifier, $json['phoromatic']['test_suite']); self::$p_save_identifier = $json['phoromatic']['trigger_id']; $phoromatic_results_identifier = self::$p_save_identifier; $phoromatic_save_identifier = $json['phoromatic']['save_identifier']; self::$p_schedule_id = isset($json['phoromatic']['schedule_id']) ? $json['phoromatic']['schedule_id'] : false; self::$p_trigger_id = self::$p_save_identifier; $benchmark_ticket_id = isset($json['phoromatic']['benchmark_ticket_id']) ? $json['phoromatic']['benchmark_ticket_id'] : null; phoromatic::update_system_status('Running Benchmarks For: ' . $phoromatic_save_identifier); if (pts_strings::string_bool($json['phoromatic']['settings']['RunInstallCommand'])) { if (isset($json['phoromatic']['pre_install_set_context'])) { phoromatic::set_user_context($json['phoromatic']['pre_install_set_context'], self::$p_trigger_id, self::$p_schedule_id, 'PRE_INSTALL'); } if (pts_strings::string_bool($json['phoromatic']['settings']['ForceInstallTests'])) { $test_flags |= pts_c::force_install; } pts_client::set_test_flags($test_flags); pts_test_installer::standard_install($suite_identifier); if (isset($json['phoromatic']['post_install_set_context'])) { phoromatic::set_user_context($json['phoromatic']['post_install_set_context'], self::$p_trigger_id, self::$p_schedule_id, 'POST_INSTALL'); } } $env_vars = isset($json['phoromatic']['environment_variables']) ? pts_strings::parse_value_string_vars($json['phoromatic']['environment_variables']) : array(); // Do the actual running phodevi::clear_cache(); if (pts_test_run_manager::initial_checks($suite_identifier, 0, 'SHORT')) { self::$test_run_manager = new pts_test_run_manager($test_flags); pts_test_run_manager::set_batch_mode(array('UploadResults' => isset($json['phoromatic']['settings']['UploadResultsToOpenBenchmarking']) && pts_strings::string_bool($json['phoromatic']['settings']['UploadResultsToOpenBenchmarking']), 'SaveResults' => true, 'RunAllTestCombinations' => false, 'OpenBrowser' => false)); // Load the tests to run if (self::$test_run_manager->load_tests_to_run($suite_identifier)) { phoromatic::update_system_status('Tests In Run Queue: ' . implode(', ', self::$test_run_manager->get_tests_to_run_identifiers())); if (isset($json['phoromatic']['pre_run_set_context'])) { phoromatic::set_user_context($json['phoromatic']['pre_run_set_context'], self::$p_trigger_id, self::$p_schedule_id, 'PRE_RUN'); } if (isset($json['phoromatic']['settings']['UploadResultsToOpenBenchmarking']) && pts_strings::string_bool($json['phoromatic']['settings']['UploadResultsToOpenBenchmarking'])) { self::$test_run_manager->auto_upload_to_openbenchmarking(); pts_openbenchmarking_client::override_client_setting('UploadSystemLogsByDefault', pts_strings::string_bool($json['phoromatic']['settings']['UploadSystemLogs'])); } // Save results? // Run the actual tests if (isset($env_vars['PTS_CONCURRENT_TEST_RUNS']) && $env_vars['PTS_CONCURRENT_TEST_RUNS'] > 1) { $total_loop_time = isset($env_vars['TOTAL_LOOP_TIME']) ? $env_vars['TOTAL_LOOP_TIME'] : false; pts_client::$pts_logger->log('STRESS / MULTI-TEST EXECUTION STARTED @ ' . date('Y-m-d H:i:s')); pts_client::$pts_logger->log('CONCURRENT RUNS = ' . $env_vars['PTS_CONCURRENT_TEST_RUNS'] . ' TOTAL LOOP TIME = ' . $total_loop_time); $r = self::$test_run_manager->multi_test_stress_run_execute($env_vars['PTS_CONCURRENT_TEST_RUNS'], $total_loop_time); if ($r == false) { return; } pts_client::$pts_logger->log('STRESS / MULTI-TEST EXECUTION ENDED @ ' . date('Y-m-d H:i:s')); } else { self::$test_run_manager->auto_save_results($phoromatic_save_identifier, $phoromatic_results_identifier, isset($json['phoromatic']['test_description']) ? $json['phoromatic']['test_description'] : 'A Phoromatic run.'); self::$test_run_manager->pre_execution_process(); self::$test_run_manager->call_test_runs(); } phoromatic::update_system_status('Benchmarks Completed For: ' . $phoromatic_save_identifier); self::$test_run_manager->post_execution_process(); $elapsed_benchmark_time = time() - $benchmark_timer; // Handle uploading data to server $result_file = new pts_result_file(self::$test_run_manager->get_file_name()); $upload_system_logs = pts_strings::string_bool($json['phoromatic']['settings']['UploadSystemLogs']); $server_response = self::upload_test_result($result_file, $upload_system_logs, isset($json['phoromatic']['schedule_id']) ? $json['phoromatic']['schedule_id'] : null, $phoromatic_save_identifier, $json['phoromatic']['trigger_id'], $elapsed_benchmark_time, $benchmark_ticket_id); //pts_client::$pts_logger->log('DEBUG RESPONSE MESSAGE: ' . $server_response); if (!pts_strings::string_bool($json['phoromatic']['settings']['ArchiveResultsLocally'])) { pts_client::remove_saved_result_file(self::$test_run_manager->get_file_name()); } } if (isset($json['phoromatic']['post_install_set_context'])) { phoromatic::set_user_context($json['phoromatic']['post_install_set_context'], self::$p_trigger_id, self::$p_schedule_id, 'POST_RUN'); } } self::$p_schedule_id = null; self::$is_running_as_phoromatic_node = false; break; case 'reboot': echo PHP_EOL . 'Phoromatic received a remote command to reboot.' . PHP_EOL; phoromatic::update_system_status('Attempting System Reboot'); if (pts_client::executable_in_path('reboot')) { shell_exec('reboot'); sleep(5); } break; case 'shutdown-if-supports-wake': $supports_wol = false; foreach (pts_network::get_network_wol() as $net_device) { if (strpos($net_device, 'g') !== false) { $supports_wol = true; break; } } if (!$supports_wol) { break; } case 'shutdown': if (isset($json['phoromatic']['client_update_script']) && !empty($json['phoromatic']['client_update_script'])) { self::run_client_update_script($json['phoromatic']['client_update_script']); sleep(10); } echo PHP_EOL . 'Phoromatic received a remote command to shutdown.' . PHP_EOL; phoromatic::update_system_status('Attempting System Shutdown'); if (pts_client::executable_in_path('poweroff')) { shell_exec('poweroff'); sleep(5); } break; case 'maintenance': echo PHP_EOL . 'Idling, system maintenance mode set by Phoromatic Server.' . PHP_EOL; phoromatic::update_system_status('Maintenance Mode'); sleep(60); break; case 'idle': if (isset($json['phoromatic']['client_update_script']) && !empty($json['phoromatic']['client_update_script'])) { self::run_client_update_script($json['phoromatic']['client_update_script']); } //echo PHP_EOL . 'Idling, waiting for task.' . PHP_EOL; phoromatic::update_system_status('Idling, Waiting For Task'); break; case 'exit': echo PHP_EOL . 'Phoromatic received a remote command to exit.' . PHP_EOL; phoromatic::update_system_status('Exiting Phoromatic'); $do_exit = true; break; } } } } if (!$do_exit) { if ($server_response == false) { sleep(rand(10, 30)); } else { sleep(60); } } } pts_client::release_lock(PTS_USER_PATH . 'phoromatic_lock'); }
public static function user_system_process() { $last_communication_minute = date('i'); $communication_attempts = 0; static $current_hw = null; static $current_sw = null; if (define('PHOROMATIC_START', true)) { echo PHP_EOL . 'Registering Status With Phoromatic Server @ ' . date('H:i:s') . PHP_EOL; $times_tried = 0; do { if ($times_tried > 0) { echo PHP_EOL . 'Connection to server failed. Trying again in 60 seconds...' . PHP_EOL; sleep(60); } $update_sd = phoromatic::update_system_details(); $times_tried++; } while (!$update_sd && $times_tried < 5); if (!$update_sd) { echo 'Server connection still failed. Exiting...' . PHP_EOL; return false; } $current_hw = phodevi::system_hardware(true); $current_sw = phodevi::system_software(true); echo PHP_EOL . 'Idling 30 seconds for system to settle...' . PHP_EOL; sleep(30); } do { $exit_loop = false; echo PHP_EOL . 'Checking Status From Phoromatic Server @ ' . date('H:i:s'); if ($last_communication_minute == date('i') && $communication_attempts > 2) { // Something is wrong, Phoromatic shouldn't be communicating with server more than three times a minute $response = M_PHOROMATIC_RESPONSE_IDLE; } else { $server_response = phoromatic::upload_to_remote_server(array('r' => 'status_check')); $xml_parser = new nye_XmlReader($server_response); $response = $xml_parser->getXMLValue(M_PHOROMATIC_GEN_RESPONSE); if (date('i') != $last_communication_minute) { $last_communication_minute = date('i'); $communication_attempts = 0; } $communication_attempts++; } echo ' [' . $response . ']' . PHP_EOL; switch ($response) { case M_PHOROMATIC_RESPONSE_RUN_TEST: $test_flags = pts_c::auto_mode | pts_c::recovery_mode; do { $suite_identifier = 'phoromatic-' . rand(1000, 9999); } while (is_file(PTS_TEST_SUITE_PATH . 'local/' . $suite_identifier . '/suite-definition.xml')); file_put_contents(PTS_TEST_SUITE_PATH . 'local/' . $suite_identifier . '/suite-definition.xml', $server_response); $phoromatic_schedule_id = $xml_parser->getXMLValue(M_PHOROMATIC_ID); $phoromatic_results_identifier = $xml_parser->getXMLValue(M_PHOROMATIC_SYS_NAME); $phoromatic_trigger = $xml_parser->getXMLValue(M_PHOROMATIC_TRIGGER); if (pts_strings::string_bool($xml_parser->getXMLValue(M_PHOROMATIC_RUN_INSTALL_COMMAND, M_PHOROMATIC_RESPONSE_TRUE))) { phoromatic::set_user_context($xml_parser->getXMLValue(M_PHOROMATIC_SET_CONTEXT_PRE_INSTALL), $phoromatic_trigger, $phoromatic_schedule_id, 'INSTALL'); pts_client::set_test_flags($test_flags); pts_test_installer::standard_install($suite_identifier); } phoromatic::set_user_context($xml_parser->getXMLValue(M_PHOROMATIC_SET_CONTEXT_PRE_RUN), $phoromatic_trigger, $phoromatic_schedule_id, 'INSTALL'); // Do the actual running if (pts_test_run_manager::initial_checks($suite_identifier)) { $test_run_manager = new pts_test_run_manager($test_flags); // Load the tests to run if ($test_run_manager->load_tests_to_run($suite_identifier)) { if (pts_strings::string_bool($xml_parser->getXMLValue(M_PHOROMATIC_UPLOAD_TO_GLOBAL, 'FALSE'))) { $test_run_manager->auto_upload_to_openbenchmarking(); } // Save results? $test_run_manager->auto_save_results(date('Y-m-d H:i:s'), $phoromatic_results_identifier, 'A Phoromatic run.'); // Run the actual tests $test_run_manager->pre_execution_process(); $test_run_manager->call_test_runs(); $test_run_manager->post_execution_process(); // Upload to Phoromatic pts_file_io::unlink(PTS_TEST_SUITE_PATH . 'local/' . $suite_identifier . '/suite-definition.xml'); // Upload test results if (is_file(PTS_SAVE_RESULTS_PATH . $save_identifier . '/composite.xml')) { phoromatic::update_system_status('Uploading Test Results'); $times_tried = 0; do { if ($times_tried > 0) { echo PHP_EOL . 'Connection to server failed. Trying again in 60 seconds...' . PHP_EOL; sleep(60); } $uploaded_test_results = phoromatic::upload_test_results($save_identifier, $phoromatic_schedule_id, $phoromatic_results_identifier, $phoromatic_trigger); $times_tried++; } while ($uploaded_test_results == false && $times_tried < 5); if ($uploaded_test_results == false) { echo 'Server connection failed. Exiting...' . PHP_EOL; return false; } if (pts_strings::string_bool($xml_parser->getXMLValue(M_PHOROMATIC_ARCHIVE_RESULTS_LOCALLY, M_PHOROMATIC_RESPONSE_TRUE)) == false) { pts_client::remove_saved_result_file($save_identifier); } } } } break; case M_PHOROMATIC_RESPONSE_EXIT: echo PHP_EOL . 'Phoromatic received a remote command to exit.' . PHP_EOL; phoromatic::update_system_status('Exiting Phoromatic'); pts_client::release_lock(PTS_USER_PATH . 'phoromatic_lock'); $exit_loop = true; break; case M_PHOROMATIC_RESPONSE_SERVER_MAINTENANCE: // The Phoromatic server is down for maintenance, so don't bother updating system status and wait longer before checking back echo PHP_EOL . 'The Phoromatic server is currently down for maintenance. Waiting for service to be restored.' . PHP_EOL; sleep((15 - date('i') % 15) * 60); break; case M_PHOROMATIC_RESPONSE_SHUTDOWN: echo PHP_EOL . 'Shutting down the system.' . PHP_EOL; $exit_loop = true; shell_exec('poweroff'); // Currently assuming root break; case M_PHOROMATIC_RESPONSE_REBOOT: echo PHP_EOL . 'Rebooting the system.' . PHP_EOL; $exit_loop = true; shell_exec('reboot'); // Currently assuming root break; case M_PHOROMATIC_RESPONSE_IDLE: default: phoromatic::update_system_status('Idling, Waiting For Task'); sleep((10 - date('i') % 10) * 60); // Check with server every 10 minutes break; } if (phodevi::system_hardware(true) != $current_hw || phodevi::system_software(true) != $current_sw) { // Hardware and/or software has changed while PTS/Phoromatic has been running, update the Phoromatic Server echo 'Updating Installed Hardware / Software With Phoromatic Server' . PHP_EOL; phoromatic::update_system_details(); $current_hw = phodevi::system_hardware(true); $current_sw = phodevi::system_software(true); } } while ($exit_loop == false); phoromatic::update_system_status('Offline'); }