Пример #1
0
 /**
  * invoked before starting tests
  * @return void
  */
 public function start()
 {
     if (isset($this->options['collectd_rrd'])) {
         ch_collectd_rrd_start($this->options['collectd_rrd_dir'], isset($this->options['verbose']));
     }
 }
Пример #2
0
 /**
  * initiates stream scaling testing. returns TRUE on success, FALSE otherwise
  * @return boolean
  */
 public function test()
 {
     $rrdStarted = isset($this->options['collectd_rrd']) ? ch_collectd_rrd_start($this->options['collectd_rrd_dir'], isset($this->options['verbose'])) : FALSE;
     $success = FALSE;
     $testsCompleted = 0;
     $testsFailed = 0;
     $testStarted = time();
     // apply sleep period before starting testing
     if (isset($this->options['sleep_before_start'])) {
         $pieces = explode('-', $this->options['sleep_before_start']);
         $min = trim($pieces[0]);
         $max = isset($pieces[1]) ? trim($pieces[1]) : NULL;
         if (is_numeric($min) && (!$max || is_numeric($max) && $max > $min)) {
             $sleep = $min && $max ? rand($min, $max) : $min;
             print_msg(sprintf('Sleeping fore %d seconds before starting testing due to --sleep_before_start %s', $sleep, $this->options['sleep_before_start']), $this->verbose, __FILE__, __LINE__);
             sleep($sleep);
         } else {
             print_msg(sprintf('--sleep_before_start %s is not valid', $this->options['sleep_before_start']), $this->verbose, __FILE__, __LINE__, TRUE);
         }
     }
     print_msg(sprintf('Initiating testing for %d test endpoints', count($this->options['test_endpoint'])), $this->verbose, __FILE__, __LINE__);
     // randomize testing order
     $keys = array_keys($this->options['test_endpoint']);
     if (isset($this->options['randomize']) && $this->options['randomize']) {
         print_msg(sprintf('Randomizing test order'), $this->verbose, __FILE__, __LINE__);
         shuffle($keys);
     }
     foreach ($keys as $testNum => $i) {
         $endpoints = $this->options['test_endpoint'][$i];
         // max test time
         if (isset($this->options['max_runtime']) && $testStarted + $this->options['max_runtime'] <= time()) {
             print_msg(sprintf('--max_time %d seconds reached - aborting testing', $this->options['max_runtime']), $this->verbose, __FILE__, __LINE__);
             break;
         } else {
             if (isset($this->options['max_tests']) && $testsCompleted >= $this->options['max_tests']) {
                 print_msg(sprintf('--max_tests %d reached - aborting testing', $this->options['max_tests']), $this->verbose, __FILE__, __LINE__);
                 break;
             } else {
                 if (isset($this->options['abort_threshold']) && $testsFailed >= $this->options['abort_threshold']) {
                     $this->aborted = TRUE;
                     print_msg(sprintf('--abort_threshold %d reached - aborting testing', $this->options['abort_threshold']), $this->verbose, __FILE__, __LINE__, TRUE);
                     break;
                 } else {
                     if (!$this->validateSameConstraints($i)) {
                         print_msg(sprintf('Skipping testing for endpoint %s because one or more --same* constraints did not match', $endpoints[0]), $this->verbose, __FILE__, __LINE__);
                         continue;
                     }
                 }
             }
         }
         $tests = array_key_exists($i, $this->options['test']) ? $this->options['test'][$i] : $this->options['test'][0];
         $isThroughput = FALSE;
         // replace throughput with downlink + uplink
         if (in_array('throughput', $tests)) {
             $isThroughput = TRUE;
             if (!in_array('downlink', $tests)) {
                 $tests[] = 'downlink';
             }
             if (!in_array('uplink', $tests)) {
                 $tests[] = 'uplink';
             }
             unset($tests[array_search('throughput', $tests)]);
         }
         $serviceId = isset($this->options['test_service_id']) ? array_key_exists($i, $this->options['test_service_id']) ? $this->options['test_service_id'][$i] : $this->options['test_service_id'][0] : NULL;
         $providerId = isset($this->options['test_provider_id']) ? array_key_exists($i, $this->options['test_provider_id']) ? $this->options['test_provider_id'][$i] : $this->options['test_provider_id'][0] : NULL;
         $serviceType = isset($this->options['test_service_type']) ? array_key_exists($i, $this->options['test_service_type']) ? $this->options['test_service_type'][$i] : $this->options['test_service_type'][0] : NULL;
         if ($serviceId && !$serviceType && count($pieces = explode(':', $serviceId)) == 2 && in_array($pieces[1], array('servers', 'vps', 'compute', 'storage', 'cdn', 'dns'))) {
             $serviceType = $pieces[1] == 'servers' || $pieces[1] == 'vps' ? 'compute' : $pieces[1];
         }
         if ($serviceType) {
             $supportedTests = array();
             switch ($serviceType) {
                 case 'compute':
                 case 'paas':
                     $supportedTests[] = 'uplink';
                     $supportedTests[] = 'downlink';
                     $supportedTests[] = 'latency';
                     break;
                 case 'storage':
                 case 'cdn':
                     $supportedTests[] = 'downlink';
                     $supportedTests[] = 'latency';
                     break;
                 case 'dns':
                     $supportedTests[] = 'dns';
                     break;
             }
             // no tests to run
             $btests = array();
             foreach ($tests as $test) {
                 $btests[] = $test;
             }
             $tests = array_intersect($tests, $supportedTests);
             if (!count($tests)) {
                 print_msg(sprintf('Skipping testing for endpoint %s because supported tests [%s] are not included in --test [%s]. providerId: %s; serviceId: %s; serviceType: %s', $endpoints[0], implode(', ', $supportedTests), implode(', ', $btests), $providerId, $serviceId, $serviceType), $this->verbose, __FILE__, __LINE__);
                 continue;
             }
         }
         if (isset($this->options['randomize']) && $this->options['randomize']) {
             shuffle($tests);
         }
         print_msg(sprintf('Starting [%s] testing of endpoint %s [%d of %d]. providerId: %s; serviceId: %s; serviceType: %s', implode(', ', $tests), $endpoints[0], $testNum + 1, count($keys), $providerId, $serviceId, $serviceType), $this->verbose, __FILE__, __LINE__);
         foreach ($tests as $test) {
             // check for endpoints/service/providers to skip latency testing for
             if ($test == 'latency' && isset($this->options['latency_skip'])) {
                 $hostname = get_hostname($endpoints[0]);
                 if (in_array($hostname, $this->options['latency_skip']) || in_array($serviceId, $this->options['latency_skip']) || in_array($providerId, $this->options['latency_skip'])) {
                     print_msg(sprintf('Skipping latency test for endpoint %s; service %s; provider %s; due to --latency_skip constraint', $endpoints[0], $serviceId, $providerId), $this->verbose, __FILE__, __LINE__);
                     continue;
                 }
             }
             // spacing
             if ($testNum > 0 && isset($this->options['spacing'])) {
                 usleep($this->options['spacing'] * 1000);
                 print_msg(sprintf('Applied test spacing of %d ms', $this->options['spacing']), $this->verbose, __FILE__, __LINE__);
             }
             $results = array();
             $privateEndpoint = FALSE;
             print_msg(sprintf('Starting %s test against endpoint %s', $test, $endpoints[0]), $this->verbose, __FILE__, __LINE__);
             // iterate through both private and public endpoint addresses
             $testStart = NULL;
             $testStop = NULL;
             foreach (array_reverse($endpoints) as $n => $endpoint) {
                 if ($test != 'dns' && count($endpoints) > 1 && $n == 0 && !$this->usePrivateNetwork($i)) {
                     print_msg(sprintf('Skipping private network endpoint %s because services are not related', $endpoint), $this->verbose, __FILE__, __LINE__);
                     continue;
                 }
                 $testStart = date('Y-m-d H:i:s');
                 switch ($test) {
                     case 'latency':
                         $results['latency'] = $this->testLatency($endpoint);
                         break;
                     case 'downlink':
                         $results['downlink'] = $this->testThroughput($endpoint, $i);
                         break;
                     case 'uplink':
                         $results['uplink'] = $this->testThroughput($endpoint, $i, TRUE);
                         break;
                     case 'dns':
                         $results['dns'] = $this->testDns($endpoints);
                         break;
                 }
                 $testStop = date('Y-m-d H:i:s');
                 // check if test completed
                 $done = $test == 'dns' ? TRUE : FALSE;
                 foreach (array_keys($results) as $key) {
                     if (is_array($results[$key])) {
                         $done = TRUE;
                     }
                 }
                 if ($done && $test != 'dns') {
                     if (count($endpoints) > 1 && $n == 0) {
                         $privateEndpoint = TRUE;
                     }
                     break;
                 }
             }
             foreach ($results as $test => $metrics) {
                 $success = TRUE;
                 $row = array('test' => $test, 'test_endpoint' => $endpoint, 'test_ip' => gethostbyname(get_hostname(str_replace('*', rand(), $endpoint))), 'test_started' => $testStart, 'test_stopped' => $testStop);
                 if ($country = isset($this->options['test_location_country'][$i]) ? $this->options['test_location_country'][$i] : NULL) {
                     $state = isset($this->options['test_location_state'][$i]) ? $this->options['test_location_state'][$i] : NULL;
                     if ($geoRegion = $this->getGeoRegion($country, $state)) {
                         $row['test_geo_region'] = $geoRegion;
                     }
                 }
                 // add additional test_* result attributes
                 foreach (array('test_instance_id', 'test_location', 'test_location_country', 'test_location_state', 'test_provider', 'test_provider_id', 'test_region', 'test_service', 'test_service_id', 'test_service_type') as $param) {
                     if (isset($this->options[$param]) && (array_key_exists($i, $this->options[$param]) || isset($this->options[$param][0]))) {
                         $row[$param] = array_key_exists($i, $this->options[$param]) ? $this->options[$param][$i] : $this->options[$param][0];
                     }
                 }
                 // private endpoint?
                 if ($privateEndpoint) {
                     $row['test_private_endpoint'] = TRUE;
                     if (isset($this->options['test_private_network_type'])) {
                         $row['test_private_network_type'] = array_key_exists($i, $this->options['test_private_network_type']) ? $this->options['test_private_network_type'][$i] : $this->options['test_private_network_type'][0];
                     }
                 }
                 $row['timeout'] = $this->options[sprintf('%s_timeout', $test == 'dns' || $test == 'latency' ? $test : 'throughput')];
                 if (isset($metrics) && is_array($metrics)) {
                     $testsCompleted++;
                     if (isset($metrics['metrics'])) {
                         $row = array_merge($row, $metrics);
                     } else {
                         $row['metrics'] = $metrics;
                     }
                     // determine status from tests_failed and tests_success
                     $status = 'fail';
                     foreach ($row['metrics'] as $n => $metric) {
                         if (is_numeric($metric)) {
                             $status = 'success';
                         }
                     }
                     if (isset($row['tests_failed']) && $row['tests_failed'] > 0) {
                         $status = isset($row['tests_success']) && !$row['tests_success'] ? 'fail' : 'partial';
                     } else {
                         if (isset($row['tests_success']) && $row['tests_success'] > 0) {
                             $status = 'success';
                         }
                     }
                     // calculate metric statistical values
                     $lowerBetter = $test == 'latency' || $test == 'dns' || isset($this->options['throughput_time']);
                     $lowerBetter ? rsort($row['metrics']) : sort($row['metrics']);
                     $discardSlowest = isset($this->options['discard_slowest']) ? $this->options['discard_slowest'] : 0;
                     $discardFastest = isset($this->options['discard_fastest']) ? $this->options['discard_fastest'] : 0;
                     if ($discardSlowest || $discardFastest) {
                         print_msg(sprintf('Trimming %s of slowest and %s of fastest metrics [%s]; Lower better: %s', $discardSlowest . '%', $discardFastest . '%', implode(', ', $row['metrics']), $lowerBetter), $this->verbose, __FILE__, __LINE__);
                         $row['metrics'] = trim_points($row['metrics'], $discardSlowest, $discardFastest);
                         print_msg(sprintf('Trimmed metrics [%s]', implode(', ', $row['metrics'])), $this->verbose, __FILE__, __LINE__);
                     }
                     $row['metric'] = get_median($row['metrics']);
                     $row['metric_10'] = get_percentile($row['metrics'], 10, $lowerBetter);
                     $row['metric_25'] = get_percentile($row['metrics'], 25, $lowerBetter);
                     $row['metric_75'] = get_percentile($row['metrics'], 75, $lowerBetter);
                     $row['metric_90'] = get_percentile($row['metrics'], 90, $lowerBetter);
                     $row['metric_fastest'] = $row['metrics'][count($row['metrics']) - 1];
                     $row['metric_mean'] = get_mean($row['metrics']);
                     $row['metric_slowest'] = $row['metrics'][0];
                     $row['metric_max'] = $lowerBetter ? $row['metric_slowest'] : $row['metric_fastest'];
                     $row['metric_min'] = $lowerBetter ? $row['metric_fastest'] : $row['metric_slowest'];
                     $row['metric_stdev'] = get_std_dev($row['metrics']);
                     $row['metric_rstdev'] = round($row['metric_stdev'] / $row['metric'] * 100, 4);
                     $row['metric_sum'] = array_sum($row['metrics']);
                     $row['metric_sum_squares'] = get_sum_squares($row['metrics']);
                     $row['metric_unit'] = $lowerBetter ? 'ms' : 'Mb/s';
                     $row['metric_unit_long'] = $lowerBetter ? 'milliseconds' : 'megabits per second';
                     $row['samples'] = count($row['metrics']);
                     $row['metrics'] = implode(',', $row['metrics']);
                     $row['status'] = $status;
                     print_msg(sprintf('%s test for endpoint %s completed successfully', $test, $endpoint), $this->verbose, __FILE__, __LINE__);
                     print_msg(sprintf('status: %s; samples: %d; median: %s; mean: %s; 10th: %s; 90th: %s; fastest: %s; slowest: %s; min: %s; max: %s; sum/squares: %s/%s; stdev: %s', $row['status'], $row['samples'], $row['metric'], $row['metric_mean'], $row['metric_10'], $row['metric_90'], $row['metric_fastest'], $row['metric_slowest'], $row['metric_min'], $row['metric_max'], $row['metric_sum'], $row['metric_sum_squares'], $row['metric_stdev']), $this->verbose, __FILE__, __LINE__);
                     print_msg(sprintf('metrics: [%s]', $row['metrics']), $this->verbose, __FILE__, __LINE__);
                     $this->results[] = $row;
                     // add inverse throughput record
                     if (isset($this->options['throughput_inverse']) && !$isThroughput && isset($this->options['meta_compute_service_id']) && isset($row['test_service_id']) && isset($row['test_service_type']) && $row['test_service_type'] == 'compute') {
                         print_msg(sprintf('Generating %s inverse throughput record from [%s] [%s]', $row['test'], implode(', ', array_keys($row)), implode(', ', $row)), $this->verbose, __FILE__, __LINE__);
                         $nrow = array();
                         foreach ($row as $k => $v) {
                             $nrow[$k] = $v;
                         }
                         $nrow['test'] = $row['test'] == 'uplink' ? 'downlink' : 'uplink';
                         // meta attributes that should not be set
                         foreach (array('meta_cpu', 'meta_memory', 'meta_memory_gb', 'meta_memory_mb', 'meta_os_info', 'meta_resource_id') as $k) {
                             $nrow[$k] = FALSE;
                         }
                         foreach (array('meta_compute_service' => 'test_service', 'meta_compute_service_id' => 'test_service_id', 'meta_geo_region' => 'test_geo_region', 'meta_instance_id' => 'test_instance_id', 'meta_hostname' => 'test_endpoint', 'meta_location' => 'test_location', 'meta_location_country' => 'test_location_country', 'meta_location_state' => 'test_location_state', 'meta_provider' => 'test_provider', 'meta_provider_id' => 'test_provider_id', 'meta_region' => 'test_region') as $meta => $attr) {
                             if (isset($nrow[$attr])) {
                                 $nrow[$meta] = $meta == 'meta_hostname' ? get_hostname($nrow[$attr]) : $nrow[$attr];
                             } else {
                                 $nrow[$meta] = FALSE;
                             }
                             $v = isset($this->options[$meta]) ? $this->options[$meta] : ($meta == 'meta_hostname' ? trim(shell_exec('hostname')) : NULL);
                             if ($v) {
                                 $nrow[$attr] = $v;
                             } else {
                                 if (isset($nrow[$attr])) {
                                     unset($nrow[$attr]);
                                 }
                             }
                         }
                         if (!isset($this->myip)) {
                             $this->myip = trim(shell_exec(sprintf('curl -q http://app%d.cloudharmony.com/myip 2>/dev/null', rand(1, 2))));
                         }
                         if ($this->myip) {
                             $nrow['test_ip'] = $this->myip;
                         } else {
                             if (isset($nrow['test_ip'])) {
                                 unset($nrow['test_ip']);
                             }
                         }
                         $this->results[] = $nrow;
                         print_msg(sprintf('Added %s inverse throughput record [%s] [%s]', $nrow['test'], implode(', ', array_keys($nrow)), implode(', ', $nrow)), $this->verbose, __FILE__, __LINE__);
                     }
                 } else {
                     print_msg(sprintf('%s test for endpoint %s failed', $test, $endpoint), $this->verbose, __FILE__, __LINE__, TRUE);
                     $testsFailed++;
                     if (!isset($this->options['suppress_failed']) || !$this->options['suppress_failed']) {
                         if (isset($this->options['traceroute'])) {
                             $hostname = get_hostname($endpoint);
                             $file = sprintf('%s/traceroute.log', $this->options['output']);
                             if (!isset($this->traceroutes[$hostname])) {
                                 $this->traceroutes[$hostname] = TRUE;
                                 print_msg(sprintf('Initiating traceroute to host %s - results to be written to %s', $hostname, $file), $this->verbose, __FILE__, __LINE__);
                                 exec(sprintf('traceroute %s >> %s 2>/dev/null', $hostname, $file));
                             }
                         }
                         $row['status'] = 'failed';
                         $this->results[] = $row;
                     }
                 }
             }
             // stop testing due to various constraints
             if (isset($this->options['max_runtime']) && $testStarted + $this->options['max_runtime'] <= time()) {
                 break;
             } else {
                 if (isset($this->options['max_tests']) && $testsCompleted >= $this->options['max_tests']) {
                     break;
                 } else {
                     if (isset($this->options['abort_threshold']) && $testsFailed >= $this->options['abort_threshold']) {
                         break;
                     }
                 }
             }
         }
     }
     if ($success) {
         if ($rrdStarted) {
             ch_collectd_rrd_stop($this->options['collectd_rrd_dir'], $this->options['output'], isset($this->options['verbose']));
         }
         $this->endTest();
         if (isset($this->options['min_runtime']) && !isset($this->options['min_runtime_in_save']) && $testStarted + $this->options['min_runtime'] > time()) {
             $sleep = $testStarted + $this->options['min_runtime'] - time();
             print_msg(sprintf('Testing complete and --min_runtime %d has not been acheived. Sleeping for %d seconds', $this->options['min_runtime'], $sleep), $this->verbose, __FILE__, __LINE__);
             sleep($sleep);
         }
     }
     return $success;
 }