public function generate(SuiteDocument $results, Config $config)
{
$document = new Document();
$reportEl = $document->createRoot('reports');
$reportEl->setAttribute('name', $config->getName());
$reportEl = $reportEl->appendElement('report');
$descriptionEl = $reportEl->appendElement('description');
$descriptionEl->nodeValue = <<<EOT
Warning: The histogram report is experimental, it may change or be removed without warning in
future versions of PHPBench.
EOT;
$tableEl = $reportEl->appendElement('table');
foreach ($results->query('//subject') as $subjectEl) {
foreach ($subjectEl->query('.//variant') as $variantEl) {
$times = array();
foreach ($variantEl->query('.//iteration') as $iterationEl) {
$times[] = $iterationEl->getAttribute('rev-time');
}
if (count($times) > 1) {
$histogram = Statistics::histogram($times, $config['bins']);
$kde = new Kde($times);
$kdeX = Statistics::linspace(min($times), max($times), $config['bins'] + 1);
$kdeY = $kde->evaluate($kdeX);
} else {
$histogram = array((string) current($times) => 1);
$kdeY = array(null);
}
$counter = 0;
foreach ($histogram as $xValue => $frequency) {
$kdeVal = $kdeY[$counter++];
$rowEl = $tableEl->appendElement('row');
$cellEl = $rowEl->appendElement('cell');
$cellEl->setAttribute('name', 'benchmark');
$cellEl->nodeValue = functions\class_name($subjectEl->evaluate('string(ancestor::benchmark/@class)'));
$cellEl = $rowEl->appendElement('cell');
$cellEl->setAttribute('name', 'subject');
$cellEl->nodeValue = $subjectEl->evaluate('string(./@name)');
$cellEl = $rowEl->appendElement('cell');
$cellEl->setAttribute('name', 'index');
$cellEl->nodeValue = $counter;
$cellEl = $rowEl->appendElement('cell');
$cellEl->setAttribute('name', 'time');
$cellEl->nodeValue = $xValue;
$cellEl = $rowEl->appendElement('cell');
$cellEl->setAttribute('name', 'freq');
$cellEl->nodeValue = $frequency;
$cellEl = $rowEl->appendElement('cell');
$cellEl->setAttribute('name', 'kde');
$cellEl->nodeValue = $kdeVal;
}
}
}
return $document;
}
/**
* Return the average relative standard deviation of all the iteration sets.
*
* @return float
*/
public function getMeanRelStDev()
{
$rStDevs = array();
foreach ($this->query('//stats') as $stats) {
$rStDevs[] = $stats->getAttribute('rstdev');
}
return Statistics::mean($rStDevs);
}
/**
* Calculate and set the deviation from the mean time for each iteration. If
* the deviation is greater than the rejection threshold, then mark the iteration as
* rejected.
*/
public function computeStats()
{
$this->rejects = array();
if (0 === count($this->iterations)) {
return;
}
$times = array();
foreach ($this->iterations as $iteration) {
$times[] = $iteration->getResult()->getTime() / $iteration->getRevolutions();
}
// standard deviation for T Distribution
$this->stats['stdev'] = Statistics::stdev($times);
// mean of the times
$this->stats['mean'] = Statistics::mean($times);
// standard error
$this->stats['rstdev'] = $this->stats['stdev'] / $this->stats['mean'] * 100;
// variance
$this->stats['variance'] = Statistics::variance($times);
// min and max
$this->stats['min'] = min($times);
$this->stats['max'] = max($times);
foreach ($this->iterations as $iteration) {
// deviation is the percentage different of the value from the mean of the set.
$deviation = 100 / $this->stats['mean'] * ($iteration->getResult()->getTime() / $iteration->getRevolutions() - $this->stats['mean']);
$iteration->setDeviation($deviation);
// the Z-Value repreents the number of standard deviations this
// value is away from the mean.
$revTime = $iteration->getResult()->getTime() / $iteration->getRevolutions();
$zValue = $this->stats['stdev'] ? ($revTime - $this->stats['mean']) / $this->stats['stdev'] : 0;
$iteration->setZValue($zValue);
if (null !== $this->rejectionThreshold) {
if (abs($deviation) >= $this->rejectionThreshold) {
$this->rejects[] = $iteration;
}
}
}
$this->computed = true;
}
/**
* It should return the average.
*/
public function testMean()
{
$expected = 33 / 7;
$this->assertEquals($expected, Statistics::mean(array(2, 2, 2, 2, 2, 20, 3)));
}
/**
* It should return histogram data.
*
* @dataProvider provideHistogram
*/
public function testHistogram(array $data, $steps, $lower, $upper, array $expected)
{
$result = Statistics::histogram($data, $steps, $lower, $upper);
$this->assertEquals($expected, $result);
}
private function drawIterations(Variant $variant)
{
$subject = $variant->getSubject();
$this->output->write("[2K");
// clear the whole line
$this->output->write(PHP_EOL);
$this->output->write("[2K");
// clear the whole line
$this->output->write("[1A");
$sigma = 2;
$bins = 16;
if ($variant->isComputed()) {
$times = $variant->getMetricValues(ComputedResult::class, 'z_value');
$stats = $variant->getStats();
$freqs = Statistics::histogram($times, $bins, -$sigma, $sigma);
} else {
$stats = new Distribution([0]);
$freqs = array_fill(0, $bins + 1, null);
}
$this->output->write(sprintf('#%-2d (σ = %s ) -%sσ [', $subject->getIndex(), $this->timeUnit->format($stats->getStdev()), $sigma));
$this->drawBlocks($freqs, $stats);
$this->output->write(sprintf('] +%sσ <comment>%s</comment>', $sigma, $variant->isComputed() ? $this->formatIterationsShortSummary($variant) : ''));
$this->output->write(PHP_EOL);
}
public function provideEvaluate()
{
return array(array(array(10, 20, 15, 5), Statistics::linspace(0, 9, 10), 'silverman', array(0.01537595, 0.0190706, 0.02299592, 0.02700068, 0.03092369, 0.0346125, 0.03794007, 0.0408159, 0.04318983, 0.04504829)), array(array(10, 20, 15, 5), Statistics::linspace(0, 3, 4), 'scott', array(0.01480612, 0.01869787, 0.02286675, 0.02713209)), array(array(10, 20, 15, 5), Statistics::linspace(0, 3, 4), 'silverman', array(0.01537595, 0.0190706, 0.02299592, 0.02700068)));
}
/**
* @dataProvider provideKdeMode
*/
public function testKdeMode($population, $space, $bandwidth, $expected)
{
$result = Statistics::kdeMode($population, $space, $bandwidth);
$this->assertEquals($expected, round($result, 2));
}
/**
* Construct the initial table from the SuiteCollection.
*
* @param SuiteCollection $suiteCollection
* @param Config $config
*
* @return array
*/
private function buildTable(SuiteCollection $suiteCollection, Config $config)
{
$paramJsonFlags = null;
if (true === $config['pretty_params']) {
$paramJsonFlags = JSON_PRETTY_PRINT;
}
$table = [];
$columnNames = [];
foreach ($suiteCollection->getSuites() as $suite) {
$env = $suite->getEnvInformations();
foreach ($suite->getBenchmarks() as $benchmark) {
foreach ($benchmark->getSubjects() as $subject) {
foreach ($subject->getVariants() as $variant) {
$row = new Row(['suite' => $suite->getUuid(), 'date' => $suite->getDate()->format('Y-m-d'), 'stime' => $suite->getDate()->format('H:i:s'), 'benchmark' => $this->getClassShortName($benchmark->getClass()), 'benchmark_full' => $benchmark->getClass(), 'subject' => $subject->getName(), 'groups' => implode(',', $subject->getGroups()), 'params' => json_encode($variant->getParameterSet()->getArrayCopy(), $paramJsonFlags), 'revs' => $variant->getRevolutions(), 'its' => count($variant->getIterations()), 'mem_real' => Statistics::mean($variant->getMetricValues(MemoryResult::class, 'real')), 'mem_final' => Statistics::mean($variant->getMetricValues(MemoryResult::class, 'final')), 'mem_peak' => Statistics::mean($variant->getMetricValues(MemoryResult::class, 'peak'))]);
// the formatter params are passed to the Formatter and
// allow the formatter configurations to use tokens --
// in other words we can override formatting on a
// per-row basis.
$formatParams = [];
if ($timeUnit = $subject->getOutputTimeUnit()) {
$formatParams['output_time_unit'] = $timeUnit;
}
if ($mode = $subject->getOutputMode()) {
$formatParams['output_mode'] = $mode;
}
if ($precision = $subject->getOutputTimePrecision()) {
$formatParams['output_time_precision'] = $precision;
}
$row->setFormatParams($formatParams);
$stats = $variant->getStats()->getStats();
$stats['best'] = $stats['min'];
$stats['worst'] = $stats['max'];
// save on duplication and lazily evaluate the
// available statistics.
if (null === $this->statKeys) {
$this->statKeys = array_keys($stats);
}
$row = $row->merge($stats);
// generate the environment parameters.
// TODO: should we crash here if an attempt is made to
// override a row? it could happen.
foreach ($env as $providerName => $information) {
foreach ($information as $key => $value) {
$row[$providerName . '_' . $key] = $value;
}
}
foreach ($row->getNames() as $columnName) {
if (!isset($columnNames[$columnName])) {
$columnNames[$columnName] = true;
}
}
// if the iterations option is specified then we add a row for each iteration, otherwise
// we continue.
if (false === $config['iterations']) {
$table[] = $row;
continue;
}
foreach ($variant->getIterations() as $index => $iteration) {
$row = clone $row;
$row['iter'] = $index;
foreach ($iteration->getResults() as $result) {
$metrics = $result->getMetrics();
// otherwise prefix the metric key with the result key.
foreach ($metrics as $key => $value) {
// TODO: this is a hack to add the rev time to the report.
if ($result instanceof TimeResult && $key === 'net') {
$row[$result->getKey() . '_rev'] = $result->getRevTime($iteration->getVariant()->getRevolutions());
}
$row[$result->getKey() . '_' . $key] = $value;
}
}
$table[] = $row;
}
}
}
}
}
// multiple suites may have different column names, for example the
// number of environment columns may differ. here we iterate over all
// the rows to ensure they all have all of the columns which have been
// defined.
foreach ($table as $row) {
foreach (array_keys($columnNames) as $columnName) {
if (!isset($row[$columnName])) {
$row[$columnName] = null;
}
}
}
return $table;
}
public function benchStDev()
{
Statistics::stdev([10, 100, 42, 84, 11, 12, 9, 6]);
}
/**
* Return the mode time.
*
* @return float
*/
public function getModeTime()
{
return Statistics::kdeMode($this->getTimes());
}
public function getMeanRelStDev()
{
return Statistics::mean($this->stats['rstdev']);
}
public function generatePoints($params)
{
$this->points = Statistics::linspace(1, 10, $params['points']);
}
private function drawIterations(IterationCollection $iterations)
{
$subject = $iterations->getSubject();
$this->output->write("[2K");
// clear the whole line
$this->output->write(PHP_EOL);
$this->output->write("[2K");
// clear the whole line
$this->output->write("[1A");
$sigma = 2;
$bins = 16;
if ($iterations->isComputed()) {
$times = $iterations->getZValues();
$stats = $iterations->getStats();
$freqs = Statistics::histogram($times, $bins, -$sigma, $sigma);
} else {
$stats = array('stdev' => 0, 'mean' => 0, 'max' => 0);
$freqs = array_fill(0, $bins + 1, null);
}
$this->output->write(sprintf('#%-2d (σ = %s ) -%sσ [', $subject->getIndex(), $this->timeUnit->format($stats['stdev']), $sigma));
$this->drawBlocks($freqs, $stats);
$this->output->write(sprintf('] +%sσ <comment>%s</comment>', $sigma, $iterations->isComputed() ? $this->formatIterationsShortSummary($iterations) : ''));
$this->output->write(PHP_EOL);
}
