public function increase_run_count_check(&$test_results, $scheduled_times_to_run, $latest_test_run_time)
{
// First make sure this test doesn't take too long to run where we don't want dynamic handling
if (floor($latest_test_run_time / 60) > $this->dynamic_run_count_on_length_or_less) {
return false;
}
// Determine if results are statistically significant, otherwise up the run count
$std_dev = pts_math::percent_standard_deviation($test_results->test_result_buffer->get_values());
if ($std_dev >= $this->dynamic_run_count_std_deviation_threshold) {
static $last_run_count = 128;
// just a number that should always cause the first check below to be true
static $run_std_devs;
$times_already_ran = $test_results->test_result_buffer->get_count();
if ($times_already_ran <= $last_run_count) {
// We're now onto a new test so clear out the array
$run_std_devs = array();
}
$last_run_count = $times_already_ran;
$run_std_devs[$last_run_count] = $std_dev;
// If we haven't reached scheduled times to run x 2, increase count straight away
if ($times_already_ran < $scheduled_times_to_run * 2) {
return true;
} else {
if ($times_already_ran < $scheduled_times_to_run * 3) {
// More aggressive determination whether to still keep increasing the run count
$first_and_now_diff = pts_arrays::first_element($run_std_devs) - pts_arrays::last_element($run_std_devs);
// Increasing the run count at least looks to be helping...
if ($first_and_now_diff > pts_arrays::first_element($run_std_devs) / 2) {
// If we are at least making progress in the right direction, increase the run count some more
return true;
}
// TODO: could add more checks and take better advantage of the array of data to better determine if it's still worth increasing
}
}
}
// Check to see if there is an external/custom script to export the results to in determining whether results are valid
if (($ex_file = $this->dynamic_run_count_export_script) != null && is_executable($ex_file) || is_executable($ex_file = PTS_USER_PATH . $this->dynamic_run_count_export_script)) {
$exit_status = trim(shell_exec($ex_file . ' ' . $test_results->test_result_buffer->get_values_as_string() . ' > /dev/null 2>&1; echo $?'));
switch ($exit_status) {
case 1:
// Run the test again
return true;
case 2:
// Results are bad, abandon testing and do not record results
return -1;
case 0:
default:
// Return was 0 or something else, results are valid, or was some other exit status
break;
}
}
// No reason to increase the run count with none of the previous checks requesting otherwise
return false;
}
protected static function process_user_config_external_hook_process($process, $cmd_value, $description_string = null, &$passed_obj = null)
{
if (!empty($cmd_value) && (is_executable($cmd_value) || ($cmd_value = pts_client::executable_in_path($cmd_value)))) {
$descriptor_spec = array(0 => array('pipe', 'r'), 1 => array('pipe', 'w'), 2 => array('pipe', 'w'));
$env_vars = array('PTS_EXTERNAL_TEST_HOOK' => $process);
if ($passed_obj instanceof pts_test_result) {
$env_vars['PTS_EXTERNAL_TEST_IDENTIFIER'] = $passed_obj->test_profile->get_identifier();
$env_vars['PTS_EXTERNAL_TEST_RUN_POSITION'] = $passed_obj->test_result_buffer->get_count() + 1;
$env_vars['PTS_EXTERNAL_TEST_RUN_COUNT'] = $passed_obj->test_profile->get_times_to_run();
$env_vars['PTS_EXTERNAL_TEST_ARGS'] = $passed_obj->get_arguments();
$env_vars['PTS_EXTERNAL_TEST_DESCRIPTION'] = $passed_obj->get_arguments_description();
$env_vars['PTS_EXTERNAL_TEST_RESULT_SET'] = $passed_obj->test_result_buffer->get_values_as_string();
$env_vars['PTS_EXTERNAL_TEST_RESULT'] = $passed_obj->get_result() != 0 ? $passed_obj->get_result() : pts_arrays::last_element($passed_obj->test_result_buffer->get_values());
$env_vars['PTS_EXTERNAL_TEST_HASH'] = bin2hex($passed_obj->get_comparison_hash());
$env_vars['PTS_EXTERNAL_TEST_STD_DEV_PERCENT'] = pts_math::percent_standard_deviation($passed_obj->test_result_buffer->get_values());
if (is_file($passed_obj->test_profile->get_install_dir() . 'cache-share-' . PTS_INIT_TIME . '.pt2so')) {
// There's a cache share present
$env_vars['PTS_EXTERNAL_TEST_CACHE_SHARE'] = 1;
}
} else {
if ($passed_obj instanceof pts_test_run_manager) {
$env_vars['PTS_EXTERNAL_TESTS_IN_QUEUE'] = implode(':', $passed_obj->get_tests_to_run_identifiers());
$env_vars['PTS_EXTERNAL_TEST_FILE_NAME'] = $passed_obj->get_file_name();
$env_vars['PTS_EXTERNAL_TEST_IDENTIFIER'] = $passed_obj->get_results_identifier();
}
}
$description_string != null && pts_client::$display->test_run_instance_error($description_string);
$proc = proc_open($cmd_value, $descriptor_spec, $pipes, null, $env_vars);
$std_output = stream_get_contents($pipes[1]);
$return_value = proc_close($proc);
// If you want PTS to exit or something when your script returns !0, you could add an 'exit;' or whatever you want below
// The contents of $std_output is anything that may have been written by your script, if you want it to be interpreted by anything in this module
if ($return_value != 0) {
return false;
}
}
return true;
}
public static function result_file_to_result_table(&$result_file, &$system_id_keys = null, &$result_object_index = -1, &$flag_delta_results = false, $extra_attributes = null)
{
$result_table = array();
$result_tests = array();
$result_counter = 0;
foreach ($result_file->get_system_identifiers() as $sys_identifier) {
$result_table[$sys_identifier] = null;
}
foreach ($result_file->get_result_objects($result_object_index) as $ri => $result_object) {
if ($result_object->test_profile->get_identifier() == null) {
continue;
}
if ($extra_attributes != null) {
if (isset($extra_attributes['reverse_result_buffer'])) {
$result_object->test_result_buffer->buffer_values_reverse();
}
if (isset($extra_attributes['normalize_result_buffer'])) {
if (isset($extra_attributes['highlight_graph_values']) && is_array($extra_attributes['highlight_graph_values']) && count($extra_attributes['highlight_graph_values']) == 1) {
$normalize_against = $extra_attributes['highlight_graph_values'][0];
} else {
$normalize_against = false;
}
$result_object->normalize_buffer_values($normalize_against);
}
}
if ($result_object_index != -1) {
if (is_array($result_object_index)) {
$result_tests[$result_counter] = new pts_graph_ir_value($result_object->get_arguments_description());
} else {
$result_tests[$result_counter] = new pts_graph_ir_value('Results');
}
} else {
$result_tests[$result_counter] = new pts_graph_ir_value($result_object->test_profile->get_title());
$result_tests[$result_counter]->set_attribute('title', $result_object->get_arguments_description());
if ($result_object->test_profile->get_identifier() != null) {
$result_tests[$result_counter]->set_attribute('href', 'http://openbenchmarking.org/test/' . $result_object->test_profile->get_identifier());
}
}
if ($result_object->test_profile->get_identifier() == null) {
if ($result_object->test_profile->get_display_format() == 'BAR_GRAPH') {
$result_tests[$result_counter]->set_attribute('alert', true);
foreach ($result_object->test_result_buffer->get_buffer_items() as $index => $buffer_item) {
$identifier = $buffer_item->get_result_identifier();
$value = $buffer_item->get_result_value();
$result_table[$identifier][$result_counter] = new pts_graph_ir_value($value, array('alert' => true));
}
$result_counter++;
}
continue;
}
switch ($result_object->test_profile->get_display_format()) {
case 'BAR_GRAPH':
$best_value = 0;
$worst_value = 0;
if (!defined('PHOROMATIC_TRACKER') && count($result_object->test_result_buffer->get_values()) > 1) {
switch ($result_object->test_profile->get_result_proportion()) {
case 'HIB':
$best_value = max($result_object->test_result_buffer->get_values());
$worst_value = min($result_object->test_result_buffer->get_values());
break;
case 'LIB':
$best_value = min($result_object->test_result_buffer->get_values());
$worst_value = max($result_object->test_result_buffer->get_values());
break;
}
}
$prev_value = 0;
$prev_identifier = null;
$prev_identifier_0 = null;
$values_in_buffer = $result_object->test_result_buffer->get_values();
sort($values_in_buffer);
$min_value_in_buffer = $values_in_buffer[0];
if ($min_value_in_buffer == 0) {
// Go through the values until something not 0, otherwise down in the code will be a divide by zero
for ($i = 1; $i < count($values_in_buffer) && $min_value_in_buffer == 0; $i++) {
$min_value_in_buffer = $values_in_buffer[$i];
}
}
$max_value_in_buffer = $values_in_buffer[count($values_in_buffer) - 1];
foreach ($result_object->test_result_buffer->get_buffer_items() as $index => $buffer_item) {
$identifier = $buffer_item->get_result_identifier();
$value = $buffer_item->get_result_value();
$raw_values = pts_strings::colon_explode($buffer_item->get_result_raw());
$percent_std = pts_math::set_precision(pts_math::percent_standard_deviation($raw_values), 2);
$std_error = pts_math::set_precision(pts_math::standard_error($raw_values), 2);
$delta = 0;
if (defined('PHOROMATIC_TRACKER')) {
$identifier_r = pts_strings::colon_explode($identifier);
if ($identifier_r[0] == $prev_identifier_0 && $prev_value != 0) {
$delta = pts_math::set_precision(abs(1 - $value / $prev_value), 4);
if ($delta > 0.02 && $delta > pts_math::standard_deviation($raw_values)) {
switch ($result_object->test_profile->get_result_proportion()) {
case 'HIB':
if ($value < $prev_value) {
$delta = 0 - $delta;
}
break;
case 'LIB':
if ($value > $prev_value) {
$delta = 0 - $delta;
}
break;
}
} else {
$delta = 0;
}
}
$prev_identifier_0 = $identifier_r[0];
$highlight = false;
$alert = false;
} else {
if ($result_file->is_multi_way_comparison()) {
// TODO: make it work better for highlighting multiple winners in multi-way comparisons
$highlight = false;
$alert = false;
// TODO: get this working right
if (false && $index % 2 == 1 && $prev_value != 0) {
switch ($result_object->test_profile->get_result_proportion()) {
case 'HIB':
if ($value > $prev_value) {
$highlight = true;
} else {
$result_table[$prev_identifier][$result_counter]->set_attribute('highlight', true);
$result_table[$prev_identifier][$result_counter]->set_attribute('delta', -1);
}
break;
case 'LIB':
if ($value < $prev_value) {
$highlight = true;
} else {
$result_table[$prev_identifier][$result_counter]->set_attribute('highlight', true);
$result_table[$prev_identifier][$result_counter]->set_attribute('delta', -1);
}
break;
}
}
} else {
$alert = $worst_value == $value;
$highlight = $best_value == $value;
}
if ($min_value_in_buffer != $max_value_in_buffer) {
switch ($result_object->test_profile->get_result_proportion()) {
case 'HIB':
$delta = pts_math::set_precision($value / $min_value_in_buffer, 2);
break;
case 'LIB':
$delta = pts_math::set_precision(1 - $value / $max_value_in_buffer + 1, 2);
break;
}
}
}
$attributes = array('std_percent' => $percent_std, 'std_error' => $std_error, 'delta' => $delta, 'highlight' => $highlight, 'alert' => $alert);
if ($delta > $percent_std && $flag_delta_results !== false) {
$flag_delta_results[$ri] = $delta;
}
$result_table[$identifier][$result_counter] = new pts_graph_ir_value($value, $attributes);
$prev_identifier = $identifier;
$prev_value = $value;
}
break;
case 'LINE_GRAPH':
case 'FILLED_LINE_GRAPH':
$result_tests[$result_counter] = new pts_graph_ir_value($result_object->test_profile->get_title() . ' (Avg)');
foreach ($result_object->test_result_buffer->get_buffer_items() as $index => $buffer_item) {
$identifier = $buffer_item->get_result_identifier();
$values = pts_strings::comma_explode($buffer_item->get_result_value());
$avg_value = pts_math::set_precision(array_sum($values) / count($values), 2);
$result_table[$identifier][$result_counter] = new pts_graph_ir_value($avg_value);
}
break;
}
$result_counter++;
}
if ($result_counter == 1) {
// This should provide some additional information under normal modes
$has_written_std = false;
$has_written_diff = false;
$has_written_error = false;
foreach ($result_table as $identifier => $info) {
if (!isset($info[$result_counter - 1])) {
continue;
}
$std_percent = $info[$result_counter - 1]->get_attribute('std_percent');
$std_error = $info[$result_counter - 1]->get_attribute('std_error');
$delta = $info[$result_counter - 1]->get_attribute('delta');
if ($delta != 0) {
$result_table[$identifier][] = new pts_graph_ir_value($delta . 'x');
$has_written_diff = true;
}
if ($std_error != 0) {
$result_table[$identifier][] = new pts_graph_ir_value($std_error);
$has_written_error = true;
}
if ($std_percent != 0) {
$result_table[$identifier][] = new pts_graph_ir_value($std_percent . '%');
$has_written_std = true;
}
}
if ($has_written_diff) {
$result_tests[] = new pts_graph_ir_value('Difference');
}
if ($has_written_error) {
$result_tests[] = new pts_graph_ir_value('Standard Error');
}
if ($has_written_std) {
$result_tests[] = new pts_graph_ir_value('Standard Deviation');
}
}
if (defined('PHOROMATIC_TRACKER')) {
// Resort the results by SYSTEM, then date
$systems_table = array();
$sorted_table = array();
foreach ($result_table as $system_identifier => &$identifier_table) {
$identifier = pts_strings::colon_explode($system_identifier);
if (!isset($systems_table[$identifier[0]])) {
$systems_table[$identifier[0]] = array();
}
$systems_table[$identifier[0]][$system_identifier] = $identifier_table;
}
$result_table = array();
$result_systems = array();
foreach ($systems_table as &$group) {
foreach ($group as $identifier => $table) {
$result_table[$identifier] = $table;
$identifier = pts_strings::colon_explode($identifier);
$show_id = isset($identifier[1]) ? $identifier[1] : $identifier[0];
/*
if($system_id_keys != null && ($s = array_search($identifier[0], $system_id_keys)) !== false)
{
$system_id = $s;
}
else
{
$system_id = null;
}*/
$result_systems[] = $show_id;
}
}
} else {
$result_systems = array();
foreach (array_keys($result_table) as $id) {
$result_systems[] = $id;
}
}
return array($result_tests, $result_systems, $result_table);
}
public function test_run_instance_complete(&$test_result)
{
if ($this->expected_trial_run_count > 1 && $this->trial_run_count_current >= $this->expected_trial_run_count) {
$values = $test_result->test_result_buffer->get_values();
if (count($values) > 1) {
echo ($this->trial_run_count_current < 10 ? ' ' : null) . ' [Std. Dev: ' . pts_math::set_precision(pts_math::percent_standard_deviation($values), 2) . '%]';
}
}
}
