/**
* invoked before starting tests
* @return void
*/
public function start()
{
if (isset($this->options['collectd_rrd'])) {
ch_collectd_rrd_start($this->options['collectd_rrd_dir'], isset($this->options['verbose']));
}
}
/**
* initiates stream scaling testing. returns TRUE on success, FALSE otherwise
* @return boolean
*/
public function test()
{
$rrdStarted = isset($this->options['collectd_rrd']) ? ch_collectd_rrd_start($this->options['collectd_rrd_dir'], isset($this->options['verbose'])) : FALSE;
$success = FALSE;
$testsCompleted = 0;
$testsFailed = 0;
$testStarted = time();
// apply sleep period before starting testing
if (isset($this->options['sleep_before_start'])) {
$pieces = explode('-', $this->options['sleep_before_start']);
$min = trim($pieces[0]);
$max = isset($pieces[1]) ? trim($pieces[1]) : NULL;
if (is_numeric($min) && (!$max || is_numeric($max) && $max > $min)) {
$sleep = $min && $max ? rand($min, $max) : $min;
print_msg(sprintf('Sleeping fore %d seconds before starting testing due to --sleep_before_start %s', $sleep, $this->options['sleep_before_start']), $this->verbose, __FILE__, __LINE__);
sleep($sleep);
} else {
print_msg(sprintf('--sleep_before_start %s is not valid', $this->options['sleep_before_start']), $this->verbose, __FILE__, __LINE__, TRUE);
}
}
print_msg(sprintf('Initiating testing for %d test endpoints', count($this->options['test_endpoint'])), $this->verbose, __FILE__, __LINE__);
// randomize testing order
$keys = array_keys($this->options['test_endpoint']);
if (isset($this->options['randomize']) && $this->options['randomize']) {
print_msg(sprintf('Randomizing test order'), $this->verbose, __FILE__, __LINE__);
shuffle($keys);
}
foreach ($keys as $testNum => $i) {
$endpoints = $this->options['test_endpoint'][$i];
// max test time
if (isset($this->options['max_runtime']) && $testStarted + $this->options['max_runtime'] <= time()) {
print_msg(sprintf('--max_time %d seconds reached - aborting testing', $this->options['max_runtime']), $this->verbose, __FILE__, __LINE__);
break;
} else {
if (isset($this->options['max_tests']) && $testsCompleted >= $this->options['max_tests']) {
print_msg(sprintf('--max_tests %d reached - aborting testing', $this->options['max_tests']), $this->verbose, __FILE__, __LINE__);
break;
} else {
if (isset($this->options['abort_threshold']) && $testsFailed >= $this->options['abort_threshold']) {
$this->aborted = TRUE;
print_msg(sprintf('--abort_threshold %d reached - aborting testing', $this->options['abort_threshold']), $this->verbose, __FILE__, __LINE__, TRUE);
break;
} else {
if (!$this->validateSameConstraints($i)) {
print_msg(sprintf('Skipping testing for endpoint %s because one or more --same* constraints did not match', $endpoints[0]), $this->verbose, __FILE__, __LINE__);
continue;
}
}
}
}
$tests = array_key_exists($i, $this->options['test']) ? $this->options['test'][$i] : $this->options['test'][0];
$isThroughput = FALSE;
// replace throughput with downlink + uplink
if (in_array('throughput', $tests)) {
$isThroughput = TRUE;
if (!in_array('downlink', $tests)) {
$tests[] = 'downlink';
}
if (!in_array('uplink', $tests)) {
$tests[] = 'uplink';
}
unset($tests[array_search('throughput', $tests)]);
}
$serviceId = isset($this->options['test_service_id']) ? array_key_exists($i, $this->options['test_service_id']) ? $this->options['test_service_id'][$i] : $this->options['test_service_id'][0] : NULL;
$providerId = isset($this->options['test_provider_id']) ? array_key_exists($i, $this->options['test_provider_id']) ? $this->options['test_provider_id'][$i] : $this->options['test_provider_id'][0] : NULL;
$serviceType = isset($this->options['test_service_type']) ? array_key_exists($i, $this->options['test_service_type']) ? $this->options['test_service_type'][$i] : $this->options['test_service_type'][0] : NULL;
if ($serviceId && !$serviceType && count($pieces = explode(':', $serviceId)) == 2 && in_array($pieces[1], array('servers', 'vps', 'compute', 'storage', 'cdn', 'dns'))) {
$serviceType = $pieces[1] == 'servers' || $pieces[1] == 'vps' ? 'compute' : $pieces[1];
}
if ($serviceType) {
$supportedTests = array();
switch ($serviceType) {
case 'compute':
case 'paas':
$supportedTests[] = 'uplink';
$supportedTests[] = 'downlink';
$supportedTests[] = 'latency';
break;
case 'storage':
case 'cdn':
$supportedTests[] = 'downlink';
$supportedTests[] = 'latency';
break;
case 'dns':
$supportedTests[] = 'dns';
break;
}
// no tests to run
$btests = array();
foreach ($tests as $test) {
$btests[] = $test;
}
$tests = array_intersect($tests, $supportedTests);
if (!count($tests)) {
print_msg(sprintf('Skipping testing for endpoint %s because supported tests [%s] are not included in --test [%s]. providerId: %s; serviceId: %s; serviceType: %s', $endpoints[0], implode(', ', $supportedTests), implode(', ', $btests), $providerId, $serviceId, $serviceType), $this->verbose, __FILE__, __LINE__);
continue;
}
}
if (isset($this->options['randomize']) && $this->options['randomize']) {
shuffle($tests);
}
print_msg(sprintf('Starting [%s] testing of endpoint %s [%d of %d]. providerId: %s; serviceId: %s; serviceType: %s', implode(', ', $tests), $endpoints[0], $testNum + 1, count($keys), $providerId, $serviceId, $serviceType), $this->verbose, __FILE__, __LINE__);
foreach ($tests as $test) {
// check for endpoints/service/providers to skip latency testing for
if ($test == 'latency' && isset($this->options['latency_skip'])) {
$hostname = get_hostname($endpoints[0]);
if (in_array($hostname, $this->options['latency_skip']) || in_array($serviceId, $this->options['latency_skip']) || in_array($providerId, $this->options['latency_skip'])) {
print_msg(sprintf('Skipping latency test for endpoint %s; service %s; provider %s; due to --latency_skip constraint', $endpoints[0], $serviceId, $providerId), $this->verbose, __FILE__, __LINE__);
continue;
}
}
// spacing
if ($testNum > 0 && isset($this->options['spacing'])) {
usleep($this->options['spacing'] * 1000);
print_msg(sprintf('Applied test spacing of %d ms', $this->options['spacing']), $this->verbose, __FILE__, __LINE__);
}
$results = array();
$privateEndpoint = FALSE;
print_msg(sprintf('Starting %s test against endpoint %s', $test, $endpoints[0]), $this->verbose, __FILE__, __LINE__);
// iterate through both private and public endpoint addresses
$testStart = NULL;
$testStop = NULL;
foreach (array_reverse($endpoints) as $n => $endpoint) {
if ($test != 'dns' && count($endpoints) > 1 && $n == 0 && !$this->usePrivateNetwork($i)) {
print_msg(sprintf('Skipping private network endpoint %s because services are not related', $endpoint), $this->verbose, __FILE__, __LINE__);
continue;
}
$testStart = date('Y-m-d H:i:s');
switch ($test) {
case 'latency':
$results['latency'] = $this->testLatency($endpoint);
break;
case 'downlink':
$results['downlink'] = $this->testThroughput($endpoint, $i);
break;
case 'uplink':
$results['uplink'] = $this->testThroughput($endpoint, $i, TRUE);
break;
case 'dns':
$results['dns'] = $this->testDns($endpoints);
break;
}
$testStop = date('Y-m-d H:i:s');
// check if test completed
$done = $test == 'dns' ? TRUE : FALSE;
foreach (array_keys($results) as $key) {
if (is_array($results[$key])) {
$done = TRUE;
}
}
if ($done && $test != 'dns') {
if (count($endpoints) > 1 && $n == 0) {
$privateEndpoint = TRUE;
}
break;
}
}
foreach ($results as $test => $metrics) {
$success = TRUE;
$row = array('test' => $test, 'test_endpoint' => $endpoint, 'test_ip' => gethostbyname(get_hostname(str_replace('*', rand(), $endpoint))), 'test_started' => $testStart, 'test_stopped' => $testStop);
if ($country = isset($this->options['test_location_country'][$i]) ? $this->options['test_location_country'][$i] : NULL) {
$state = isset($this->options['test_location_state'][$i]) ? $this->options['test_location_state'][$i] : NULL;
if ($geoRegion = $this->getGeoRegion($country, $state)) {
$row['test_geo_region'] = $geoRegion;
}
}
// add additional test_* result attributes
foreach (array('test_instance_id', 'test_location', 'test_location_country', 'test_location_state', 'test_provider', 'test_provider_id', 'test_region', 'test_service', 'test_service_id', 'test_service_type') as $param) {
if (isset($this->options[$param]) && (array_key_exists($i, $this->options[$param]) || isset($this->options[$param][0]))) {
$row[$param] = array_key_exists($i, $this->options[$param]) ? $this->options[$param][$i] : $this->options[$param][0];
}
}
// private endpoint?
if ($privateEndpoint) {
$row['test_private_endpoint'] = TRUE;
if (isset($this->options['test_private_network_type'])) {
$row['test_private_network_type'] = array_key_exists($i, $this->options['test_private_network_type']) ? $this->options['test_private_network_type'][$i] : $this->options['test_private_network_type'][0];
}
}
$row['timeout'] = $this->options[sprintf('%s_timeout', $test == 'dns' || $test == 'latency' ? $test : 'throughput')];
if (isset($metrics) && is_array($metrics)) {
$testsCompleted++;
if (isset($metrics['metrics'])) {
$row = array_merge($row, $metrics);
} else {
$row['metrics'] = $metrics;
}
// determine status from tests_failed and tests_success
$status = 'fail';
foreach ($row['metrics'] as $n => $metric) {
if (is_numeric($metric)) {
$status = 'success';
}
}
if (isset($row['tests_failed']) && $row['tests_failed'] > 0) {
$status = isset($row['tests_success']) && !$row['tests_success'] ? 'fail' : 'partial';
} else {
if (isset($row['tests_success']) && $row['tests_success'] > 0) {
$status = 'success';
}
}
// calculate metric statistical values
$lowerBetter = $test == 'latency' || $test == 'dns' || isset($this->options['throughput_time']);
$lowerBetter ? rsort($row['metrics']) : sort($row['metrics']);
$discardSlowest = isset($this->options['discard_slowest']) ? $this->options['discard_slowest'] : 0;
$discardFastest = isset($this->options['discard_fastest']) ? $this->options['discard_fastest'] : 0;
if ($discardSlowest || $discardFastest) {
print_msg(sprintf('Trimming %s of slowest and %s of fastest metrics [%s]; Lower better: %s', $discardSlowest . '%', $discardFastest . '%', implode(', ', $row['metrics']), $lowerBetter), $this->verbose, __FILE__, __LINE__);
$row['metrics'] = trim_points($row['metrics'], $discardSlowest, $discardFastest);
print_msg(sprintf('Trimmed metrics [%s]', implode(', ', $row['metrics'])), $this->verbose, __FILE__, __LINE__);
}
$row['metric'] = get_median($row['metrics']);
$row['metric_10'] = get_percentile($row['metrics'], 10, $lowerBetter);
$row['metric_25'] = get_percentile($row['metrics'], 25, $lowerBetter);
$row['metric_75'] = get_percentile($row['metrics'], 75, $lowerBetter);
$row['metric_90'] = get_percentile($row['metrics'], 90, $lowerBetter);
$row['metric_fastest'] = $row['metrics'][count($row['metrics']) - 1];
$row['metric_mean'] = get_mean($row['metrics']);
$row['metric_slowest'] = $row['metrics'][0];
$row['metric_max'] = $lowerBetter ? $row['metric_slowest'] : $row['metric_fastest'];
$row['metric_min'] = $lowerBetter ? $row['metric_fastest'] : $row['metric_slowest'];
$row['metric_stdev'] = get_std_dev($row['metrics']);
$row['metric_rstdev'] = round($row['metric_stdev'] / $row['metric'] * 100, 4);
$row['metric_sum'] = array_sum($row['metrics']);
$row['metric_sum_squares'] = get_sum_squares($row['metrics']);
$row['metric_unit'] = $lowerBetter ? 'ms' : 'Mb/s';
$row['metric_unit_long'] = $lowerBetter ? 'milliseconds' : 'megabits per second';
$row['samples'] = count($row['metrics']);
$row['metrics'] = implode(',', $row['metrics']);
$row['status'] = $status;
print_msg(sprintf('%s test for endpoint %s completed successfully', $test, $endpoint), $this->verbose, __FILE__, __LINE__);
print_msg(sprintf('status: %s; samples: %d; median: %s; mean: %s; 10th: %s; 90th: %s; fastest: %s; slowest: %s; min: %s; max: %s; sum/squares: %s/%s; stdev: %s', $row['status'], $row['samples'], $row['metric'], $row['metric_mean'], $row['metric_10'], $row['metric_90'], $row['metric_fastest'], $row['metric_slowest'], $row['metric_min'], $row['metric_max'], $row['metric_sum'], $row['metric_sum_squares'], $row['metric_stdev']), $this->verbose, __FILE__, __LINE__);
print_msg(sprintf('metrics: [%s]', $row['metrics']), $this->verbose, __FILE__, __LINE__);
$this->results[] = $row;
// add inverse throughput record
if (isset($this->options['throughput_inverse']) && !$isThroughput && isset($this->options['meta_compute_service_id']) && isset($row['test_service_id']) && isset($row['test_service_type']) && $row['test_service_type'] == 'compute') {
print_msg(sprintf('Generating %s inverse throughput record from [%s] [%s]', $row['test'], implode(', ', array_keys($row)), implode(', ', $row)), $this->verbose, __FILE__, __LINE__);
$nrow = array();
foreach ($row as $k => $v) {
$nrow[$k] = $v;
}
$nrow['test'] = $row['test'] == 'uplink' ? 'downlink' : 'uplink';
// meta attributes that should not be set
foreach (array('meta_cpu', 'meta_memory', 'meta_memory_gb', 'meta_memory_mb', 'meta_os_info', 'meta_resource_id') as $k) {
$nrow[$k] = FALSE;
}
foreach (array('meta_compute_service' => 'test_service', 'meta_compute_service_id' => 'test_service_id', 'meta_geo_region' => 'test_geo_region', 'meta_instance_id' => 'test_instance_id', 'meta_hostname' => 'test_endpoint', 'meta_location' => 'test_location', 'meta_location_country' => 'test_location_country', 'meta_location_state' => 'test_location_state', 'meta_provider' => 'test_provider', 'meta_provider_id' => 'test_provider_id', 'meta_region' => 'test_region') as $meta => $attr) {
if (isset($nrow[$attr])) {
$nrow[$meta] = $meta == 'meta_hostname' ? get_hostname($nrow[$attr]) : $nrow[$attr];
} else {
$nrow[$meta] = FALSE;
}
$v = isset($this->options[$meta]) ? $this->options[$meta] : ($meta == 'meta_hostname' ? trim(shell_exec('hostname')) : NULL);
if ($v) {
$nrow[$attr] = $v;
} else {
if (isset($nrow[$attr])) {
unset($nrow[$attr]);
}
}
}
if (!isset($this->myip)) {
$this->myip = trim(shell_exec(sprintf('curl -q http://app%d.cloudharmony.com/myip 2>/dev/null', rand(1, 2))));
}
if ($this->myip) {
$nrow['test_ip'] = $this->myip;
} else {
if (isset($nrow['test_ip'])) {
unset($nrow['test_ip']);
}
}
$this->results[] = $nrow;
print_msg(sprintf('Added %s inverse throughput record [%s] [%s]', $nrow['test'], implode(', ', array_keys($nrow)), implode(', ', $nrow)), $this->verbose, __FILE__, __LINE__);
}
} else {
print_msg(sprintf('%s test for endpoint %s failed', $test, $endpoint), $this->verbose, __FILE__, __LINE__, TRUE);
$testsFailed++;
if (!isset($this->options['suppress_failed']) || !$this->options['suppress_failed']) {
if (isset($this->options['traceroute'])) {
$hostname = get_hostname($endpoint);
$file = sprintf('%s/traceroute.log', $this->options['output']);
if (!isset($this->traceroutes[$hostname])) {
$this->traceroutes[$hostname] = TRUE;
print_msg(sprintf('Initiating traceroute to host %s - results to be written to %s', $hostname, $file), $this->verbose, __FILE__, __LINE__);
exec(sprintf('traceroute %s >> %s 2>/dev/null', $hostname, $file));
}
}
$row['status'] = 'failed';
$this->results[] = $row;
}
}
}
// stop testing due to various constraints
if (isset($this->options['max_runtime']) && $testStarted + $this->options['max_runtime'] <= time()) {
break;
} else {
if (isset($this->options['max_tests']) && $testsCompleted >= $this->options['max_tests']) {
break;
} else {
if (isset($this->options['abort_threshold']) && $testsFailed >= $this->options['abort_threshold']) {
break;
}
}
}
}
}
if ($success) {
if ($rrdStarted) {
ch_collectd_rrd_stop($this->options['collectd_rrd_dir'], $this->options['output'], isset($this->options['verbose']));
}
$this->endTest();
if (isset($this->options['min_runtime']) && !isset($this->options['min_runtime_in_save']) && $testStarted + $this->options['min_runtime'] > time()) {
$sleep = $testStarted + $this->options['min_runtime'] - time();
print_msg(sprintf('Testing complete and --min_runtime %d has not been acheived. Sleeping for %d seconds', $this->options['min_runtime'], $sleep), $this->verbose, __FILE__, __LINE__);
sleep($sleep);
}
}
return $success;
}