/**
* Return the average relative standard deviation of all the iteration sets.
*
* @return float
*/
public function getMeanRelStDev()
{
$rStDevs = array();
foreach ($this->query('//stats') as $stats) {
$rStDevs[] = $stats->getAttribute('rstdev');
}
return Statistics::mean($rStDevs);
}
/**
* Calculate and set the deviation from the mean time for each iteration. If
* the deviation is greater than the rejection threshold, then mark the iteration as
* rejected.
*/
public function computeStats()
{
$this->rejects = array();
if (0 === count($this->iterations)) {
return;
}
$times = array();
foreach ($this->iterations as $iteration) {
$times[] = $iteration->getResult()->getTime() / $iteration->getRevolutions();
}
// standard deviation for T Distribution
$this->stats['stdev'] = Statistics::stdev($times);
// mean of the times
$this->stats['mean'] = Statistics::mean($times);
// standard error
$this->stats['rstdev'] = $this->stats['stdev'] / $this->stats['mean'] * 100;
// variance
$this->stats['variance'] = Statistics::variance($times);
// min and max
$this->stats['min'] = min($times);
$this->stats['max'] = max($times);
foreach ($this->iterations as $iteration) {
// deviation is the percentage different of the value from the mean of the set.
$deviation = 100 / $this->stats['mean'] * ($iteration->getResult()->getTime() / $iteration->getRevolutions() - $this->stats['mean']);
$iteration->setDeviation($deviation);
// the Z-Value repreents the number of standard deviations this
// value is away from the mean.
$revTime = $iteration->getResult()->getTime() / $iteration->getRevolutions();
$zValue = $this->stats['stdev'] ? ($revTime - $this->stats['mean']) / $this->stats['stdev'] : 0;
$iteration->setZValue($zValue);
if (null !== $this->rejectionThreshold) {
if (abs($deviation) >= $this->rejectionThreshold) {
$this->rejects[] = $iteration;
}
}
}
$this->computed = true;
}
/**
* Construct the initial table from the SuiteCollection.
*
* @param SuiteCollection $suiteCollection
* @param Config $config
*
* @return array
*/
private function buildTable(SuiteCollection $suiteCollection, Config $config)
{
$paramJsonFlags = null;
if (true === $config['pretty_params']) {
$paramJsonFlags = JSON_PRETTY_PRINT;
}
$table = [];
$columnNames = [];
foreach ($suiteCollection->getSuites() as $suite) {
$env = $suite->getEnvInformations();
foreach ($suite->getBenchmarks() as $benchmark) {
foreach ($benchmark->getSubjects() as $subject) {
foreach ($subject->getVariants() as $variant) {
$row = new Row(['suite' => $suite->getUuid(), 'date' => $suite->getDate()->format('Y-m-d'), 'stime' => $suite->getDate()->format('H:i:s'), 'benchmark' => $this->getClassShortName($benchmark->getClass()), 'benchmark_full' => $benchmark->getClass(), 'subject' => $subject->getName(), 'groups' => implode(',', $subject->getGroups()), 'params' => json_encode($variant->getParameterSet()->getArrayCopy(), $paramJsonFlags), 'revs' => $variant->getRevolutions(), 'its' => count($variant->getIterations()), 'mem_real' => Statistics::mean($variant->getMetricValues(MemoryResult::class, 'real')), 'mem_final' => Statistics::mean($variant->getMetricValues(MemoryResult::class, 'final')), 'mem_peak' => Statistics::mean($variant->getMetricValues(MemoryResult::class, 'peak'))]);
// the formatter params are passed to the Formatter and
// allow the formatter configurations to use tokens --
// in other words we can override formatting on a
// per-row basis.
$formatParams = [];
if ($timeUnit = $subject->getOutputTimeUnit()) {
$formatParams['output_time_unit'] = $timeUnit;
}
if ($mode = $subject->getOutputMode()) {
$formatParams['output_mode'] = $mode;
}
if ($precision = $subject->getOutputTimePrecision()) {
$formatParams['output_time_precision'] = $precision;
}
$row->setFormatParams($formatParams);
$stats = $variant->getStats()->getStats();
$stats['best'] = $stats['min'];
$stats['worst'] = $stats['max'];
// save on duplication and lazily evaluate the
// available statistics.
if (null === $this->statKeys) {
$this->statKeys = array_keys($stats);
}
$row = $row->merge($stats);
// generate the environment parameters.
// TODO: should we crash here if an attempt is made to
// override a row? it could happen.
foreach ($env as $providerName => $information) {
foreach ($information as $key => $value) {
$row[$providerName . '_' . $key] = $value;
}
}
foreach ($row->getNames() as $columnName) {
if (!isset($columnNames[$columnName])) {
$columnNames[$columnName] = true;
}
}
// if the iterations option is specified then we add a row for each iteration, otherwise
// we continue.
if (false === $config['iterations']) {
$table[] = $row;
continue;
}
foreach ($variant->getIterations() as $index => $iteration) {
$row = clone $row;
$row['iter'] = $index;
foreach ($iteration->getResults() as $result) {
$metrics = $result->getMetrics();
// otherwise prefix the metric key with the result key.
foreach ($metrics as $key => $value) {
// TODO: this is a hack to add the rev time to the report.
if ($result instanceof TimeResult && $key === 'net') {
$row[$result->getKey() . '_rev'] = $result->getRevTime($iteration->getVariant()->getRevolutions());
}
$row[$result->getKey() . '_' . $key] = $value;
}
}
$table[] = $row;
}
}
}
}
}
// multiple suites may have different column names, for example the
// number of environment columns may differ. here we iterate over all
// the rows to ensure they all have all of the columns which have been
// defined.
foreach ($table as $row) {
foreach (array_keys($columnNames) as $columnName) {
if (!isset($row[$columnName])) {
$row[$columnName] = null;
}
}
}
return $table;
}
/**
* It should return the average.
*/
public function testMean()
{
$expected = 33 / 7;
$this->assertEquals($expected, Statistics::mean(array(2, 2, 2, 2, 2, 20, 3)));
}
public function getMeanRelStDev()
{
return Statistics::mean($this->stats['rstdev']);
}
