public function significance($variantKey, $controlKey, $eventKey)
{
$pControl = $this->conversionRate($controlKey, $eventKey);
$pTreatment = $this->conversionRate($variantKey, $eventKey);
$nControl = $this->variantTotal($controlKey);
$nTreatment = $this->variantTotal($variantKey);
# convert to a z score
$sigmaCombined = sqrt($pTreatment * (1 - $pTreatment) / $nTreatment + $pControl * (1 - $pControl) / $nControl);
# add 1e-8 to denominator to avoid divide by zero
$z = ($pTreatment - $pControl) / ($sigmaCombined + 1.0E-8);
# return the likelihood of a value this extreme or greater under
# the null hypothesis
$p = 2 * (1 - stats_cdf_normal(abs($z), 0, 1, 1));
return 100 - round(100 * $p, 2);
}
/**
* @param CM_Model_SplittestVariation $variationWorse
* @return float|null P-value with Šidák correction
*/
public function getSignificance(CM_Model_SplittestVariation $variationWorse)
{
$fixturesA = $this->getFixtureCount();
$fixturesB = $variationWorse->getFixtureCount();
if (!$fixturesA || !$fixturesB) {
return null;
}
$conversionsA = $this->getConversionCount();
$conversionsB = $variationWorse->getConversionCount();
if (!$conversionsA || !$conversionsB) {
return null;
}
$weightA = $this->getConversionWeight();
$weightB = $variationWorse->getConversionWeight();
if (!$weightA || !$weightB) {
return null;
}
$rateA = $weightA / $fixturesA;
$rateB = $weightB / $fixturesB;
$netRateA = $weightA / $conversionsA;
$netRateB = $weightB / $conversionsB;
$fixturesTotal = $fixturesA + $fixturesB;
$weightTotal = $weightA + $weightB;
$rateTotal = $weightTotal / $fixturesTotal;
$conversionsExpectedA = $rateTotal * $fixturesA / $netRateA;
$conversionsExpectedB = $rateTotal * $fixturesB / $netRateB;
$varianceExpectedA = $conversionsExpectedA * (1 - $conversionsExpectedA / $fixturesA);
$varianceExpectedB = $conversionsExpectedB * (1 - $conversionsExpectedB / $fixturesB);
if ($varianceExpectedA < 9 || $varianceExpectedB < 9) {
return null;
}
$sigmaExpectedA = sqrt($varianceExpectedA);
$sigmaExpectedB = sqrt($varianceExpectedB);
if ($conversionsExpectedA - 3 * $sigmaExpectedA < 0 || $conversionsExpectedB - 3 * $sigmaExpectedB < 0) {
return null;
}
if ($conversionsExpectedA + 3 * $sigmaExpectedA > $fixturesA || $conversionsExpectedB + 3 * $sigmaExpectedB > $fixturesB) {
return null;
}
$rateDeviation = abs($rateA - $rateB);
$sigmaExpectedRateA = $sigmaExpectedA * $netRateA / $fixturesA;
$sigmaExpectedRateB = $sigmaExpectedB * $netRateB / $fixturesB;
$sigmaExpectedRateDeviation = sqrt($sigmaExpectedRateA * $sigmaExpectedRateA + $sigmaExpectedRateB * $sigmaExpectedRateB);
$pValue = 2 * stats_cdf_normal(-$rateDeviation, 0, $sigmaExpectedRateDeviation, 1);
$variationCount = $this->getSplittest()->getVariations()->getCount();
$independentExperimentCount = max(1, $variationCount - 1);
$pValueSidak = 1 - pow(1 - $pValue, $independentExperimentCount);
return $pValueSidak;
}
public static function sdFromPercentBelowX($prob, $x, $mean)
{
$prob = self::formatPercent($prob);
return stats_cdf_normal($prob, $x, $mean, 4);
}
