public function check_download_caches_for_files() { pts_client::$display->test_install_progress_start('Searching Download Caches'); $test_count = count($this->tests_to_install); $remote_files = self::remote_files_available_in_download_caches(); $local_download_caches = self::local_download_caches(); $remote_download_caches = self::remote_download_caches(); $phoromatic_server_caches = pts_test_install_manager::phoromatic_download_server_caches(); foreach ($this->tests_to_install as $i => &$test_install_request) { $test_install_request->scan_download_caches($local_download_caches, $remote_download_caches, $remote_files, $phoromatic_server_caches); pts_client::$display->test_install_progress_update($i / $test_count); } pts_client::$display->test_install_progress_completed(); }
protected static function download_test_files(&$test_install_request, $download_location = false, $no_prompts = false) { // Download needed files for a test if ($test_install_request->get_download_object_count() == 0) { return true; } $identifier = $test_install_request->test_profile->get_identifier(); pts_client::$display->test_install_downloads($test_install_request); if ($download_location == false) { $download_location = $test_install_request->test_profile->get_install_dir(); } pts_file_io::mkdir($download_location); $module_pass = array($identifier, $test_install_request->get_download_objects()); pts_module_manager::module_process('__pre_test_download', $module_pass); foreach ($test_install_request->get_download_objects() as $download_package) { $package_filename = $download_package->get_filename(); $download_destination = $download_location . $package_filename; $download_destination_temp = $download_destination . '.pts'; if ($download_package->get_download_location_type() == null) { // Attempt a possible last-minute look-aside copy cache in case a previous test in the install queue downloaded this file already $lookaside_copy = pts_test_install_manager::file_lookaside_test_installations($download_package); if ($lookaside_copy) { if ($download_package->get_filesize() == 0) { $download_package->set_filesize(filesize($lookaside_copy)); } $download_package->set_download_location('LOOKASIDE_DOWNLOAD_CACHE', array($lookaside_copy)); } } switch ($download_package->get_download_location_type()) { case 'IN_DESTINATION_DIR': pts_client::$display->test_install_download_file('FILE_FOUND', $download_package); continue; case 'REMOTE_DOWNLOAD_CACHE': $download_tries = 0; do { foreach ($download_package->get_download_location_path() as $remote_download_cache_file) { pts_client::$display->test_install_download_file('DOWNLOAD_FROM_CACHE', $download_package); pts_network::download_file($remote_download_cache_file, $download_destination_temp); if (!is_file($download_destination_temp) || filesize($download_destination_temp) == 0) { self::test_install_error(null, $test_install_request, 'The file failed to download from the cache.'); pts_file_io::unlink($download_destination_temp); break; } else { if ($download_package->check_file_hash($download_destination_temp)) { rename($download_destination_temp, $download_destination); break; } else { self::test_install_error(null, $test_install_request, 'The check-sum of the downloaded file failed.'); pts_file_io::unlink($download_destination_temp); } } } $download_tries++; } while (!is_file($download_destination) && $download_tries < 2); if (is_file($download_destination)) { continue; } case 'MAIN_DOWNLOAD_CACHE': case 'LOCAL_DOWNLOAD_CACHE': case 'LOOKASIDE_DOWNLOAD_CACHE': $download_cache_file = pts_arrays::last_element($download_package->get_download_location_path()); if (is_file($download_cache_file)) { if (pts_config::read_bool_config('PhoronixTestSuite/Options/Installation/SymLinkFilesFromCache', 'FALSE') && $download_package->get_download_location_type() != 'LOOKASIDE_DOWNLOAD_CACHE') { // For look-aside copies never symlink (unless a pre-packaged LiveCD) in case the other test ends up being un-installed // SymLinkFilesFromCache is disabled by default pts_client::$display->test_install_download_file('LINK_FROM_CACHE', $download_package); symlink($download_cache_file, $download_destination); } else { // File is to be copied // Try up to two times to copy a file $attempted_copies = 0; do { pts_client::$display->test_install_download_file('COPY_FROM_CACHE', $download_package); // $context = stream_context_create(); // stream_context_set_params($context, array('notification' => array('pts_network', 'stream_status_callback'))); // TODO: get the context working correctly for this copy() copy($download_cache_file, $download_destination_temp); pts_client::$display->test_install_progress_completed(); // Verify that the file was copied fine if ($download_package->check_file_hash($download_destination_temp)) { rename($download_destination_temp, $download_destination); break; } else { self::test_install_error(null, $test_install_request, 'The check-sum of the copied file failed.'); pts_file_io::unlink($download_destination_temp); } $attempted_copies++; } while ($attempted_copies < 2); } if (is_file($download_destination)) { continue; } } default: $package_urls = $download_package->get_download_url_array(); // Download the file if (!is_file($download_destination) && count($package_urls) > 0 && $package_urls[0] != null) { $fail_count = 0; do { if (pts_network::internet_support_available()) { if (!$no_prompts && pts_config::read_bool_config('PhoronixTestSuite/Options/Installation/PromptForDownloadMirror', 'FALSE') && count($package_urls) > 1) { // Prompt user to select mirror do { echo PHP_EOL . 'Available Download Mirrors:' . PHP_EOL . PHP_EOL; $url = pts_user_io::prompt_text_menu('Select Preferred Mirror', $package_urls, false); } while (pts_strings::is_url($url) == false); } else { // Auto-select mirror shuffle($package_urls); do { $url = array_pop($package_urls); } while (pts_strings::is_url($url) == false && !empty($package_urls)); } pts_client::$display->test_install_download_file('DOWNLOAD', $download_package); $download_start = time(); pts_network::download_file($url, $download_destination_temp); $download_end = time(); } else { self::test_install_error(null, $test_install_request, 'Internet support is needed and it\'s disabled or not available.'); return false; } if ($download_package->check_file_hash($download_destination_temp)) { // Download worked if (is_file($download_destination_temp)) { rename($download_destination_temp, $download_destination); } if ($download_package->get_filesize() > 0 && $download_end != $download_start) { pts_download_speed_manager::update_download_speed_average($download_package->get_filesize(), $download_end - $download_start); } } else { // Download failed if (is_file($download_destination_temp) && filesize($download_destination_temp) < 500 && (stripos(file_get_contents($download_destination_temp), 'not found') !== false || strpos(file_get_contents($download_destination_temp), 404) !== false)) { self::test_install_error(null, $test_install_request, 'File Not Found: ' . $url); $md5_failed = false; } else { if (is_file($download_destination_temp) && filesize($download_destination_temp) > 0) { self::test_install_error(null, $test_install_request, 'Checksum Failed: ' . $url); $md5_failed = true; } else { self::test_install_error(null, $test_install_request, 'Download Failed: ' . $url); $md5_failed = false; } } pts_file_io::unlink($download_destination_temp); $fail_count++; if ($fail_count > 3) { $try_again = false; } else { if (count($package_urls) > 0 && $package_urls[0] != null) { self::test_install_error(null, $test_install_request, 'Attempting to download from alternate mirror.'); $try_again = true; } else { if ($no_prompts) { $try_again = false; } else { if ($md5_failed) { $try_again = pts_user_io::prompt_bool_input('Try downloading the file again', true, 'TRY_DOWNLOAD_AGAIN'); } else { $try_again = false; } } if ($try_again) { $package_urls[] = $url; } } } if (!$try_again) { //self::test_install_error(null, $test_install_request, 'Download of Needed Test Dependencies Failed!'); return false; } } } while (!is_file($download_destination)); } pts_module_manager::module_process('__interim_test_download', $module_pass); } } pts_module_manager::module_process('__post_test_download', $identifier); return true; }
public function scan_download_caches($local_download_caches, $remote_download_caches, $remote_files, $phoromatic_server_caches, $skip_hash_checks = false) { $download_location = $this->test_profile->get_install_dir(); $main_download_cache = pts_strings::add_trailing_slash(pts_strings::parse_for_home_directory(pts_config::read_user_config('PhoronixTestSuite/Options/Installation/CacheDirectory', PTS_DOWNLOAD_CACHE_PATH))); foreach ($this->test_files as &$download_package) { $package_filename = $download_package->get_filename(); if (is_file($download_location . $package_filename)) { // File is already there in the test/destination directory, must have been previously downloaded // Could add an MD5 check here to ensure validity, but if it made it here it was already valid unless user modified it if ($download_package->get_filesize() == 0) { $download_package->set_filesize(filesize($download_location . $package_filename)); } $download_package->set_download_location('IN_DESTINATION_DIR'); } else { if (is_file($main_download_cache . $package_filename)) { // In main download cache if ($download_package->get_filesize() == 0) { $download_package->set_filesize(filesize($main_download_cache . $package_filename)); } $download_package->set_download_location('MAIN_DOWNLOAD_CACHE', array($main_download_cache . $package_filename)); } else { if (is_file(PTS_SHARE_PATH . 'download-cache/' . $package_filename)) { // In system's /usr/share download cache if ($download_package->get_filesize() == 0) { $download_package->set_filesize(filesize(PTS_SHARE_PATH . 'download-cache/' . $package_filename)); } $download_package->set_download_location('MAIN_DOWNLOAD_CACHE', array(PTS_SHARE_PATH . 'download-cache/' . $package_filename)); } else { // Scan the local download caches foreach ($local_download_caches as &$cache_directory) { if (is_file($cache_directory . $package_filename) && ($skip_hash_checks || $download_package->check_file_hash($cache_directory . $package_filename))) { if ($download_package->get_filesize() == 0) { $download_package->set_filesize(filesize($cache_directory . $package_filename)); } $download_package->set_download_location('LOCAL_DOWNLOAD_CACHE', array($cache_directory . $package_filename)); break; } } // Look-aside download cache copy // Check to see if the same package name with the same package check-sum is already present in another test installation $lookaside_copy = pts_test_install_manager::file_lookaside_test_installations($download_package); if ($lookaside_copy) { if ($download_package->get_filesize() == 0) { $download_package->set_filesize(filesize($lookaside_copy)); } $download_package->set_download_location('LOOKASIDE_DOWNLOAD_CACHE', array($lookaside_copy)); } // Check Phoromatic server caches if ($download_package->get_download_location_type() == null && $phoromatic_server_caches) { foreach ($phoromatic_server_caches as $server_url => $repo) { if (isset($repo[$package_filename]) && ($skip_hash_checks || $repo[$package_filename]['md5'] == $download_package->get_md5() || $repo[$package_filename]['sha256'] == $download_package->get_sha256() || $download_package->get_sha256() == null && $download_package->get_md5() == null)) { $download_package->set_download_location('REMOTE_DOWNLOAD_CACHE', array($server_url . '/download-cache.php?download=' . $package_filename)); break; } } } // If still not found, check remote download caches if ($download_package->get_download_location_type() == null) { if (isset($remote_files[$package_filename])) { $download_package->set_download_location('REMOTE_DOWNLOAD_CACHE', $remote_files[$package_filename]); } else { if (!empty($remote_download_caches)) { // Check for files manually foreach ($remote_download_caches as $remote_dir) { $remote_file = $remote_dir . $package_filename; $stream_context = pts_network::stream_context_create(); $file_pointer = fopen($remote_file, 'r', false, $stream_context); if ($file_pointer !== false) { $download_package->set_download_location('REMOTE_DOWNLOAD_CACHE', $remote_file); break; } } } } } } } } } }
public static function run_matisk($args) { echo PHP_EOL . 'MATISK For The Phoronix Test Suite' . PHP_EOL; if (!isset($args[0]) || !is_file($args[0])) { echo PHP_EOL . 'You must specify a MATISK INI file to load.' . PHP_EOL . PHP_EOL; return false; } self::$matisk_config_dir = dirname($args[0]) . '/'; pts_file_io::mkdir(pts_module::save_dir()); $ini = parse_ini_file($args[0], true); foreach (self::$ini_struct as $section => $items) { foreach ($items as $key => $r) { if (is_array($r) && !isset($ini[$section][$key])) { $ini[$section][$key] = $r[0]; } } } // Checks if (pts_test_suite::is_suite($ini['workload']['suite']) == false) { // See if the XML suite-definition was just tossed into the same directory if (($xml_file = self::find_file($ini['workload']['suite'] . '.xml')) !== false) { pts_file_io::mkdir(PTS_TEST_SUITE_PATH . 'local/' . $ini['workload']['suite']); copy($xml_file, PTS_TEST_SUITE_PATH . 'local/' . $ini['workload']['suite'] . '/suite-definition.xml'); } if (pts_test_suite::is_suite($ini['workload']['suite']) == false) { echo PHP_EOL . 'A test suite must be specified to execute. If a suite needs to be constructed, run: ' . PHP_EOL . 'phoronix-test-suite build-suite' . PHP_EOL . PHP_EOL; return false; } } if ($ini['set_context']['external_contexts'] != null) { switch ($ini['set_context']['external_contexts_delimiter']) { case 'EOL': case '': $ini['set_context']['external_contexts_delimiter'] = PHP_EOL; break; case 'TAB': $ini['set_context']['external_contexts_delimiter'] = "\t"; break; } if ($ff = self::find_file($ini['set_context']['external_contexts'])) { if (is_executable($ff)) { $ini['set_context']['context'] = shell_exec($ff . ' 2> /dev/null'); } else { $ini['set_context']['context'] = file_get_contents($ff); } } else { // Hopefully it's a command to execute then... $ini['set_context']['context'] = shell_exec($ini['set_context']['external_contexts'] . ' 2> /dev/null'); } $ini['set_context']['context'] = explode($ini['set_context']['external_contexts_delimiter'], $ini['set_context']['context']); } else { if ($ini['set_context']['context'] != null && !is_array($ini['set_context']['context'])) { $ini['set_context']['context'] = array($ini['set_context']['context']); } } if (is_array($ini['set_context']['context']) && count($ini['set_context']['context']) > 0) { foreach ($ini['set_context']['context'] as $i => $context) { if ($context == null) { unset($ini['set_context']['context'][$i]); } } // Context testing if (count($ini['set_context']['context']) > 0 && $ini['set_context']['pre_run'] == null && $ini['set_context']['pre_install'] == null) { echo PHP_EOL . 'The pre_run or pre_install set_context fields must be set in order to set the system\'s context.' . PHP_EOL; return false; } if ($ini['set_context']['reverse_context_order']) { $ini['set_context']['context'] = array_reverse($ini['set_context']['context']); } } if (pts_strings::string_bool($ini['workload']['save_results'])) { if ($ini['workload']['save_name'] == null) { echo PHP_EOL . 'The save_name field cannot be left empty when saving the test results.' . PHP_EOL; return false; } /* if($ini['workload']['result_identifier'] == null) { echo PHP_EOL . 'The result_identifier field cannot be left empty when saving the test results.' . PHP_EOL; return false; } */ } if (!empty($ini['environmental_variables']) && is_array($ini['environmental_variables'])) { foreach ($ini['environmental_variables'] as $key => $value) { putenv(trim($key) . '=' . trim($value)); } } if (empty($ini['set_context']['context'])) { $ini['set_context']['context'] = array($ini['workload']['result_identifier']); } if (pts_strings::string_bool($ini['set_context']['log_context_outputs'])) { pts_file_io::mkdir(pts_module::save_dir() . $ini['workload']['save_name']); } $spent_context_file = pts_module::save_dir() . $ini['workload']['save_name'] . '.spent-contexts'; if (!is_file($spent_context_file)) { touch($spent_context_file); } else { // If recovering from an existing run, don't rerun contexts that were already executed $spent_contexts = pts_file_io::file_get_contents($spent_context_file); $spent_contexts = explode(PHP_EOL, $spent_contexts); foreach ($spent_contexts as $sc) { if (($key = array_search($sc, $ini['set_context']['context'])) !== false) { unset($ini['set_context']['context'][$key]); } } } if ($ini['set_context']['reboot_support'] && phodevi::is_linux()) { // In case a set-context involves a reboot, auto-recover $xdg_config_home = is_dir('/etc/xdg/autostart') && is_writable('/etc/xdg/autostart') ? '/etc/xdg/autostart' : pts_client::read_env('XDG_CONFIG_HOME'); if ($xdg_config_home == false) { $xdg_config_home = pts_client::user_home_directory() . '.config'; } if ($xdg_config_home != false && is_dir($xdg_config_home)) { $autostart_dir = $xdg_config_home . '/autostart/'; pts_file_io::mkdir($xdg_config_home . '/autostart/'); } file_put_contents($xdg_config_home . '/autostart/phoronix-test-suite-matisk.desktop', ' [Desktop Entry] Name=Phoronix Test Suite Matisk Recovery GenericName=Phoronix Test Suite Comment=Matisk Auto-Recovery Support Exec=gnome-terminal -e \'phoronix-test-suite matisk ' . $args[0] . '\' Icon=phoronix-test-suite Type=Application Encoding=UTF-8 Categories=System;Monitor;'); } if ($ini['installation']['block_phodevi_caching']) { // Block Phodevi caching if changing out system components and there is a chance one of the strings of changed contexts might be cached (e.g. OpenGL user-space driver) phodevi::$allow_phodevi_caching = false; } if (phodevi::system_uptime() < 60) { echo PHP_EOL . 'Sleeping 45 seconds while waiting for the system to settle...' . PHP_EOL; sleep(45); } self::$ini = $ini; $total_context_count = count(self::$ini['set_context']['context']); while (($context = array_shift(self::$ini['set_context']['context'])) !== null) { echo PHP_EOL . ($total_context_count - count(self::$ini['set_context']['context'])) . ' of ' . $total_context_count . ' in test execution queue [' . $context . ']' . PHP_EOL . PHP_EOL; self::$context = $context; if (pts_strings::string_bool(self::$ini['installation']['install_check']) || $ini['set_context']['pre_install'] != null) { self::process_user_config_external_hook_process('pre_install'); $force_install = false; $no_prompts = true; if (pts_strings::string_bool(self::$ini['installation']['force_install'])) { $force_install = true; } if (self::$ini['installation']['external_download_cache'] != null) { pts_test_install_manager::add_external_download_cache(self::$ini['installation']['external_download_cache']); } // Do the actual test installation pts_test_installer::standard_install(self::$ini['workload']['suite'], $force_install, $no_prompts); self::process_user_config_external_hook_process('post_install'); } $batch_mode = false; $auto_mode = true; $test_run_manager = new pts_test_run_manager($batch_mode, $auto_mode); if ($test_run_manager->initial_checks(self::$ini['workload']['suite']) == false) { return false; } if (self::$skip_test_set == false) { self::process_user_config_external_hook_process('pre_run'); // Load the tests to run if ($test_run_manager->load_tests_to_run(self::$ini['workload']['suite']) == false) { return false; } // Save results? $result_identifier = $ini['workload']['result_identifier']; if ($result_identifier == null) { $result_identifier = '$MATISK_CONTEXT'; } // Allow $MATIISK_CONTEXT as a valid user variable to pass it... $result_identifier = str_replace('$MATISK_CONTEXT', self::$context, $result_identifier); $test_run_manager->set_save_name(self::$ini['workload']['save_name']); $test_run_manager->set_results_identifier($result_identifier); $test_run_manager->set_description(self::$ini['workload']['description']); // Don't upload results unless it's the last in queue where the context count is now 0 $test_run_manager->auto_upload_to_openbenchmarking(count(self::$ini['set_context']['context']) == 0 && self::$ini['general']['upload_to_openbenchmarking']); // Run the actual tests $test_run_manager->pre_execution_process(); $test_run_manager->call_test_runs(); $test_run_manager->post_execution_process(); } self::$skip_test_set = false; file_put_contents($spent_context_file, self::$context . PHP_EOL, FILE_APPEND); pts_file_io::unlink(pts_module::save_dir() . self::$context . '.last-call'); self::process_user_config_external_hook_process('post_run'); } unlink($spent_context_file); isset($xdg_config_home) && pts_file_io::unlink($xdg_config_home . '/autostart/phoronix-test-suite-matisk.desktop'); }