public function check_download_caches_for_files()
{
pts_client::$display->test_install_progress_start('Searching Download Caches');
$test_count = count($this->tests_to_install);
$remote_files = self::remote_files_available_in_download_caches();
$local_download_caches = self::local_download_caches();
$remote_download_caches = self::remote_download_caches();
$phoromatic_server_caches = pts_test_install_manager::phoromatic_download_server_caches();
foreach ($this->tests_to_install as $i => &$test_install_request) {
$test_install_request->scan_download_caches($local_download_caches, $remote_download_caches, $remote_files, $phoromatic_server_caches);
pts_client::$display->test_install_progress_update($i / $test_count);
}
pts_client::$display->test_install_progress_completed();
}
protected static function download_test_files(&$test_install_request, $download_location = false, $no_prompts = false)
{
// Download needed files for a test
if ($test_install_request->get_download_object_count() == 0) {
return true;
}
$identifier = $test_install_request->test_profile->get_identifier();
pts_client::$display->test_install_downloads($test_install_request);
if ($download_location == false) {
$download_location = $test_install_request->test_profile->get_install_dir();
}
pts_file_io::mkdir($download_location);
$module_pass = array($identifier, $test_install_request->get_download_objects());
pts_module_manager::module_process('__pre_test_download', $module_pass);
foreach ($test_install_request->get_download_objects() as $download_package) {
$package_filename = $download_package->get_filename();
$download_destination = $download_location . $package_filename;
$download_destination_temp = $download_destination . '.pts';
if ($download_package->get_download_location_type() == null) {
// Attempt a possible last-minute look-aside copy cache in case a previous test in the install queue downloaded this file already
$lookaside_copy = pts_test_install_manager::file_lookaside_test_installations($download_package);
if ($lookaside_copy) {
if ($download_package->get_filesize() == 0) {
$download_package->set_filesize(filesize($lookaside_copy));
}
$download_package->set_download_location('LOOKASIDE_DOWNLOAD_CACHE', array($lookaside_copy));
}
}
switch ($download_package->get_download_location_type()) {
case 'IN_DESTINATION_DIR':
pts_client::$display->test_install_download_file('FILE_FOUND', $download_package);
continue;
case 'REMOTE_DOWNLOAD_CACHE':
$download_tries = 0;
do {
foreach ($download_package->get_download_location_path() as $remote_download_cache_file) {
pts_client::$display->test_install_download_file('DOWNLOAD_FROM_CACHE', $download_package);
pts_network::download_file($remote_download_cache_file, $download_destination_temp);
if (!is_file($download_destination_temp) || filesize($download_destination_temp) == 0) {
self::test_install_error(null, $test_install_request, 'The file failed to download from the cache.');
pts_file_io::unlink($download_destination_temp);
break;
} else {
if ($download_package->check_file_hash($download_destination_temp)) {
rename($download_destination_temp, $download_destination);
break;
} else {
self::test_install_error(null, $test_install_request, 'The check-sum of the downloaded file failed.');
pts_file_io::unlink($download_destination_temp);
}
}
}
$download_tries++;
} while (!is_file($download_destination) && $download_tries < 2);
if (is_file($download_destination)) {
continue;
}
case 'MAIN_DOWNLOAD_CACHE':
case 'LOCAL_DOWNLOAD_CACHE':
case 'LOOKASIDE_DOWNLOAD_CACHE':
$download_cache_file = pts_arrays::last_element($download_package->get_download_location_path());
if (is_file($download_cache_file)) {
if (pts_config::read_bool_config('PhoronixTestSuite/Options/Installation/SymLinkFilesFromCache', 'FALSE') && $download_package->get_download_location_type() != 'LOOKASIDE_DOWNLOAD_CACHE') {
// For look-aside copies never symlink (unless a pre-packaged LiveCD) in case the other test ends up being un-installed
// SymLinkFilesFromCache is disabled by default
pts_client::$display->test_install_download_file('LINK_FROM_CACHE', $download_package);
symlink($download_cache_file, $download_destination);
} else {
// File is to be copied
// Try up to two times to copy a file
$attempted_copies = 0;
do {
pts_client::$display->test_install_download_file('COPY_FROM_CACHE', $download_package);
// $context = stream_context_create();
// stream_context_set_params($context, array('notification' => array('pts_network', 'stream_status_callback')));
// TODO: get the context working correctly for this copy()
copy($download_cache_file, $download_destination_temp);
pts_client::$display->test_install_progress_completed();
// Verify that the file was copied fine
if ($download_package->check_file_hash($download_destination_temp)) {
rename($download_destination_temp, $download_destination);
break;
} else {
self::test_install_error(null, $test_install_request, 'The check-sum of the copied file failed.');
pts_file_io::unlink($download_destination_temp);
}
$attempted_copies++;
} while ($attempted_copies < 2);
}
if (is_file($download_destination)) {
continue;
}
}
default:
$package_urls = $download_package->get_download_url_array();
// Download the file
if (!is_file($download_destination) && count($package_urls) > 0 && $package_urls[0] != null) {
$fail_count = 0;
do {
if (pts_network::internet_support_available()) {
if (!$no_prompts && pts_config::read_bool_config('PhoronixTestSuite/Options/Installation/PromptForDownloadMirror', 'FALSE') && count($package_urls) > 1) {
// Prompt user to select mirror
do {
echo PHP_EOL . 'Available Download Mirrors:' . PHP_EOL . PHP_EOL;
$url = pts_user_io::prompt_text_menu('Select Preferred Mirror', $package_urls, false);
} while (pts_strings::is_url($url) == false);
} else {
// Auto-select mirror
shuffle($package_urls);
do {
$url = array_pop($package_urls);
} while (pts_strings::is_url($url) == false && !empty($package_urls));
}
pts_client::$display->test_install_download_file('DOWNLOAD', $download_package);
$download_start = time();
pts_network::download_file($url, $download_destination_temp);
$download_end = time();
} else {
self::test_install_error(null, $test_install_request, 'Internet support is needed and it\'s disabled or not available.');
return false;
}
if ($download_package->check_file_hash($download_destination_temp)) {
// Download worked
if (is_file($download_destination_temp)) {
rename($download_destination_temp, $download_destination);
}
if ($download_package->get_filesize() > 0 && $download_end != $download_start) {
pts_download_speed_manager::update_download_speed_average($download_package->get_filesize(), $download_end - $download_start);
}
} else {
// Download failed
if (is_file($download_destination_temp) && filesize($download_destination_temp) < 500 && (stripos(file_get_contents($download_destination_temp), 'not found') !== false || strpos(file_get_contents($download_destination_temp), 404) !== false)) {
self::test_install_error(null, $test_install_request, 'File Not Found: ' . $url);
$md5_failed = false;
} else {
if (is_file($download_destination_temp) && filesize($download_destination_temp) > 0) {
self::test_install_error(null, $test_install_request, 'Checksum Failed: ' . $url);
$md5_failed = true;
} else {
self::test_install_error(null, $test_install_request, 'Download Failed: ' . $url);
$md5_failed = false;
}
}
pts_file_io::unlink($download_destination_temp);
$fail_count++;
if ($fail_count > 3) {
$try_again = false;
} else {
if (count($package_urls) > 0 && $package_urls[0] != null) {
self::test_install_error(null, $test_install_request, 'Attempting to download from alternate mirror.');
$try_again = true;
} else {
if ($no_prompts) {
$try_again = false;
} else {
if ($md5_failed) {
$try_again = pts_user_io::prompt_bool_input('Try downloading the file again', true, 'TRY_DOWNLOAD_AGAIN');
} else {
$try_again = false;
}
}
if ($try_again) {
$package_urls[] = $url;
}
}
}
if (!$try_again) {
//self::test_install_error(null, $test_install_request, 'Download of Needed Test Dependencies Failed!');
return false;
}
}
} while (!is_file($download_destination));
}
pts_module_manager::module_process('__interim_test_download', $module_pass);
}
}
pts_module_manager::module_process('__post_test_download', $identifier);
return true;
}
public function scan_download_caches($local_download_caches, $remote_download_caches, $remote_files, $phoromatic_server_caches, $skip_hash_checks = false)
{
$download_location = $this->test_profile->get_install_dir();
$main_download_cache = pts_strings::add_trailing_slash(pts_strings::parse_for_home_directory(pts_config::read_user_config('PhoronixTestSuite/Options/Installation/CacheDirectory', PTS_DOWNLOAD_CACHE_PATH)));
foreach ($this->test_files as &$download_package) {
$package_filename = $download_package->get_filename();
if (is_file($download_location . $package_filename)) {
// File is already there in the test/destination directory, must have been previously downloaded
// Could add an MD5 check here to ensure validity, but if it made it here it was already valid unless user modified it
if ($download_package->get_filesize() == 0) {
$download_package->set_filesize(filesize($download_location . $package_filename));
}
$download_package->set_download_location('IN_DESTINATION_DIR');
} else {
if (is_file($main_download_cache . $package_filename)) {
// In main download cache
if ($download_package->get_filesize() == 0) {
$download_package->set_filesize(filesize($main_download_cache . $package_filename));
}
$download_package->set_download_location('MAIN_DOWNLOAD_CACHE', array($main_download_cache . $package_filename));
} else {
if (is_file(PTS_SHARE_PATH . 'download-cache/' . $package_filename)) {
// In system's /usr/share download cache
if ($download_package->get_filesize() == 0) {
$download_package->set_filesize(filesize(PTS_SHARE_PATH . 'download-cache/' . $package_filename));
}
$download_package->set_download_location('MAIN_DOWNLOAD_CACHE', array(PTS_SHARE_PATH . 'download-cache/' . $package_filename));
} else {
// Scan the local download caches
foreach ($local_download_caches as &$cache_directory) {
if (is_file($cache_directory . $package_filename) && ($skip_hash_checks || $download_package->check_file_hash($cache_directory . $package_filename))) {
if ($download_package->get_filesize() == 0) {
$download_package->set_filesize(filesize($cache_directory . $package_filename));
}
$download_package->set_download_location('LOCAL_DOWNLOAD_CACHE', array($cache_directory . $package_filename));
break;
}
}
// Look-aside download cache copy
// Check to see if the same package name with the same package check-sum is already present in another test installation
$lookaside_copy = pts_test_install_manager::file_lookaside_test_installations($download_package);
if ($lookaside_copy) {
if ($download_package->get_filesize() == 0) {
$download_package->set_filesize(filesize($lookaside_copy));
}
$download_package->set_download_location('LOOKASIDE_DOWNLOAD_CACHE', array($lookaside_copy));
}
// Check Phoromatic server caches
if ($download_package->get_download_location_type() == null && $phoromatic_server_caches) {
foreach ($phoromatic_server_caches as $server_url => $repo) {
if (isset($repo[$package_filename]) && ($skip_hash_checks || $repo[$package_filename]['md5'] == $download_package->get_md5() || $repo[$package_filename]['sha256'] == $download_package->get_sha256() || $download_package->get_sha256() == null && $download_package->get_md5() == null)) {
$download_package->set_download_location('REMOTE_DOWNLOAD_CACHE', array($server_url . '/download-cache.php?download=' . $package_filename));
break;
}
}
}
// If still not found, check remote download caches
if ($download_package->get_download_location_type() == null) {
if (isset($remote_files[$package_filename])) {
$download_package->set_download_location('REMOTE_DOWNLOAD_CACHE', $remote_files[$package_filename]);
} else {
if (!empty($remote_download_caches)) {
// Check for files manually
foreach ($remote_download_caches as $remote_dir) {
$remote_file = $remote_dir . $package_filename;
$stream_context = pts_network::stream_context_create();
$file_pointer = fopen($remote_file, 'r', false, $stream_context);
if ($file_pointer !== false) {
$download_package->set_download_location('REMOTE_DOWNLOAD_CACHE', $remote_file);
break;
}
}
}
}
}
}
}
}
}
}
public static function run_matisk($args)
{
echo PHP_EOL . 'MATISK For The Phoronix Test Suite' . PHP_EOL;
if (!isset($args[0]) || !is_file($args[0])) {
echo PHP_EOL . 'You must specify a MATISK INI file to load.' . PHP_EOL . PHP_EOL;
return false;
}
self::$matisk_config_dir = dirname($args[0]) . '/';
pts_file_io::mkdir(pts_module::save_dir());
$ini = parse_ini_file($args[0], true);
foreach (self::$ini_struct as $section => $items) {
foreach ($items as $key => $r) {
if (is_array($r) && !isset($ini[$section][$key])) {
$ini[$section][$key] = $r[0];
}
}
}
// Checks
if (pts_test_suite::is_suite($ini['workload']['suite']) == false) {
// See if the XML suite-definition was just tossed into the same directory
if (($xml_file = self::find_file($ini['workload']['suite'] . '.xml')) !== false) {
pts_file_io::mkdir(PTS_TEST_SUITE_PATH . 'local/' . $ini['workload']['suite']);
copy($xml_file, PTS_TEST_SUITE_PATH . 'local/' . $ini['workload']['suite'] . '/suite-definition.xml');
}
if (pts_test_suite::is_suite($ini['workload']['suite']) == false) {
echo PHP_EOL . 'A test suite must be specified to execute. If a suite needs to be constructed, run: ' . PHP_EOL . 'phoronix-test-suite build-suite' . PHP_EOL . PHP_EOL;
return false;
}
}
if ($ini['set_context']['external_contexts'] != null) {
switch ($ini['set_context']['external_contexts_delimiter']) {
case 'EOL':
case '':
$ini['set_context']['external_contexts_delimiter'] = PHP_EOL;
break;
case 'TAB':
$ini['set_context']['external_contexts_delimiter'] = "\t";
break;
}
if ($ff = self::find_file($ini['set_context']['external_contexts'])) {
if (is_executable($ff)) {
$ini['set_context']['context'] = shell_exec($ff . ' 2> /dev/null');
} else {
$ini['set_context']['context'] = file_get_contents($ff);
}
} else {
// Hopefully it's a command to execute then...
$ini['set_context']['context'] = shell_exec($ini['set_context']['external_contexts'] . ' 2> /dev/null');
}
$ini['set_context']['context'] = explode($ini['set_context']['external_contexts_delimiter'], $ini['set_context']['context']);
} else {
if ($ini['set_context']['context'] != null && !is_array($ini['set_context']['context'])) {
$ini['set_context']['context'] = array($ini['set_context']['context']);
}
}
if (is_array($ini['set_context']['context']) && count($ini['set_context']['context']) > 0) {
foreach ($ini['set_context']['context'] as $i => $context) {
if ($context == null) {
unset($ini['set_context']['context'][$i]);
}
}
// Context testing
if (count($ini['set_context']['context']) > 0 && $ini['set_context']['pre_run'] == null && $ini['set_context']['pre_install'] == null) {
echo PHP_EOL . 'The pre_run or pre_install set_context fields must be set in order to set the system\'s context.' . PHP_EOL;
return false;
}
if ($ini['set_context']['reverse_context_order']) {
$ini['set_context']['context'] = array_reverse($ini['set_context']['context']);
}
}
if (pts_strings::string_bool($ini['workload']['save_results'])) {
if ($ini['workload']['save_name'] == null) {
echo PHP_EOL . 'The save_name field cannot be left empty when saving the test results.' . PHP_EOL;
return false;
}
/*
if($ini['workload']['result_identifier'] == null)
{
echo PHP_EOL . 'The result_identifier field cannot be left empty when saving the test results.' . PHP_EOL;
return false;
}
*/
}
if (!empty($ini['environmental_variables']) && is_array($ini['environmental_variables'])) {
foreach ($ini['environmental_variables'] as $key => $value) {
putenv(trim($key) . '=' . trim($value));
}
}
if (empty($ini['set_context']['context'])) {
$ini['set_context']['context'] = array($ini['workload']['result_identifier']);
}
if (pts_strings::string_bool($ini['set_context']['log_context_outputs'])) {
pts_file_io::mkdir(pts_module::save_dir() . $ini['workload']['save_name']);
}
$spent_context_file = pts_module::save_dir() . $ini['workload']['save_name'] . '.spent-contexts';
if (!is_file($spent_context_file)) {
touch($spent_context_file);
} else {
// If recovering from an existing run, don't rerun contexts that were already executed
$spent_contexts = pts_file_io::file_get_contents($spent_context_file);
$spent_contexts = explode(PHP_EOL, $spent_contexts);
foreach ($spent_contexts as $sc) {
if (($key = array_search($sc, $ini['set_context']['context'])) !== false) {
unset($ini['set_context']['context'][$key]);
}
}
}
if ($ini['set_context']['reboot_support'] && phodevi::is_linux()) {
// In case a set-context involves a reboot, auto-recover
$xdg_config_home = is_dir('/etc/xdg/autostart') && is_writable('/etc/xdg/autostart') ? '/etc/xdg/autostart' : pts_client::read_env('XDG_CONFIG_HOME');
if ($xdg_config_home == false) {
$xdg_config_home = pts_client::user_home_directory() . '.config';
}
if ($xdg_config_home != false && is_dir($xdg_config_home)) {
$autostart_dir = $xdg_config_home . '/autostart/';
pts_file_io::mkdir($xdg_config_home . '/autostart/');
}
file_put_contents($xdg_config_home . '/autostart/phoronix-test-suite-matisk.desktop', '
[Desktop Entry]
Name=Phoronix Test Suite Matisk Recovery
GenericName=Phoronix Test Suite
Comment=Matisk Auto-Recovery Support
Exec=gnome-terminal -e \'phoronix-test-suite matisk ' . $args[0] . '\'
Icon=phoronix-test-suite
Type=Application
Encoding=UTF-8
Categories=System;Monitor;');
}
if ($ini['installation']['block_phodevi_caching']) {
// Block Phodevi caching if changing out system components and there is a chance one of the strings of changed contexts might be cached (e.g. OpenGL user-space driver)
phodevi::$allow_phodevi_caching = false;
}
if (phodevi::system_uptime() < 60) {
echo PHP_EOL . 'Sleeping 45 seconds while waiting for the system to settle...' . PHP_EOL;
sleep(45);
}
self::$ini = $ini;
$total_context_count = count(self::$ini['set_context']['context']);
while (($context = array_shift(self::$ini['set_context']['context'])) !== null) {
echo PHP_EOL . ($total_context_count - count(self::$ini['set_context']['context'])) . ' of ' . $total_context_count . ' in test execution queue [' . $context . ']' . PHP_EOL . PHP_EOL;
self::$context = $context;
if (pts_strings::string_bool(self::$ini['installation']['install_check']) || $ini['set_context']['pre_install'] != null) {
self::process_user_config_external_hook_process('pre_install');
$force_install = false;
$no_prompts = true;
if (pts_strings::string_bool(self::$ini['installation']['force_install'])) {
$force_install = true;
}
if (self::$ini['installation']['external_download_cache'] != null) {
pts_test_install_manager::add_external_download_cache(self::$ini['installation']['external_download_cache']);
}
// Do the actual test installation
pts_test_installer::standard_install(self::$ini['workload']['suite'], $force_install, $no_prompts);
self::process_user_config_external_hook_process('post_install');
}
$batch_mode = false;
$auto_mode = true;
$test_run_manager = new pts_test_run_manager($batch_mode, $auto_mode);
if ($test_run_manager->initial_checks(self::$ini['workload']['suite']) == false) {
return false;
}
if (self::$skip_test_set == false) {
self::process_user_config_external_hook_process('pre_run');
// Load the tests to run
if ($test_run_manager->load_tests_to_run(self::$ini['workload']['suite']) == false) {
return false;
}
// Save results?
$result_identifier = $ini['workload']['result_identifier'];
if ($result_identifier == null) {
$result_identifier = '$MATISK_CONTEXT';
}
// Allow $MATIISK_CONTEXT as a valid user variable to pass it...
$result_identifier = str_replace('$MATISK_CONTEXT', self::$context, $result_identifier);
$test_run_manager->set_save_name(self::$ini['workload']['save_name']);
$test_run_manager->set_results_identifier($result_identifier);
$test_run_manager->set_description(self::$ini['workload']['description']);
// Don't upload results unless it's the last in queue where the context count is now 0
$test_run_manager->auto_upload_to_openbenchmarking(count(self::$ini['set_context']['context']) == 0 && self::$ini['general']['upload_to_openbenchmarking']);
// Run the actual tests
$test_run_manager->pre_execution_process();
$test_run_manager->call_test_runs();
$test_run_manager->post_execution_process();
}
self::$skip_test_set = false;
file_put_contents($spent_context_file, self::$context . PHP_EOL, FILE_APPEND);
pts_file_io::unlink(pts_module::save_dir() . self::$context . '.last-call');
self::process_user_config_external_hook_process('post_run');
}
unlink($spent_context_file);
isset($xdg_config_home) && pts_file_io::unlink($xdg_config_home . '/autostart/phoronix-test-suite-matisk.desktop');
}
