public function significance($variantKey, $controlKey, $eventKey) { $pControl = $this->conversionRate($controlKey, $eventKey); $pTreatment = $this->conversionRate($variantKey, $eventKey); $nControl = $this->variantTotal($controlKey); $nTreatment = $this->variantTotal($variantKey); # convert to a z score $sigmaCombined = sqrt($pTreatment * (1 - $pTreatment) / $nTreatment + $pControl * (1 - $pControl) / $nControl); # add 1e-8 to denominator to avoid divide by zero $z = ($pTreatment - $pControl) / ($sigmaCombined + 1.0E-8); # return the likelihood of a value this extreme or greater under # the null hypothesis $p = 2 * (1 - stats_cdf_normal(abs($z), 0, 1, 1)); return 100 - round(100 * $p, 2); }
/** * @param CM_Model_SplittestVariation $variationWorse * @return float|null P-value with Šidák correction */ public function getSignificance(CM_Model_SplittestVariation $variationWorse) { $fixturesA = $this->getFixtureCount(); $fixturesB = $variationWorse->getFixtureCount(); if (!$fixturesA || !$fixturesB) { return null; } $conversionsA = $this->getConversionCount(); $conversionsB = $variationWorse->getConversionCount(); if (!$conversionsA || !$conversionsB) { return null; } $weightA = $this->getConversionWeight(); $weightB = $variationWorse->getConversionWeight(); if (!$weightA || !$weightB) { return null; } $rateA = $weightA / $fixturesA; $rateB = $weightB / $fixturesB; $netRateA = $weightA / $conversionsA; $netRateB = $weightB / $conversionsB; $fixturesTotal = $fixturesA + $fixturesB; $weightTotal = $weightA + $weightB; $rateTotal = $weightTotal / $fixturesTotal; $conversionsExpectedA = $rateTotal * $fixturesA / $netRateA; $conversionsExpectedB = $rateTotal * $fixturesB / $netRateB; $varianceExpectedA = $conversionsExpectedA * (1 - $conversionsExpectedA / $fixturesA); $varianceExpectedB = $conversionsExpectedB * (1 - $conversionsExpectedB / $fixturesB); if ($varianceExpectedA < 9 || $varianceExpectedB < 9) { return null; } $sigmaExpectedA = sqrt($varianceExpectedA); $sigmaExpectedB = sqrt($varianceExpectedB); if ($conversionsExpectedA - 3 * $sigmaExpectedA < 0 || $conversionsExpectedB - 3 * $sigmaExpectedB < 0) { return null; } if ($conversionsExpectedA + 3 * $sigmaExpectedA > $fixturesA || $conversionsExpectedB + 3 * $sigmaExpectedB > $fixturesB) { return null; } $rateDeviation = abs($rateA - $rateB); $sigmaExpectedRateA = $sigmaExpectedA * $netRateA / $fixturesA; $sigmaExpectedRateB = $sigmaExpectedB * $netRateB / $fixturesB; $sigmaExpectedRateDeviation = sqrt($sigmaExpectedRateA * $sigmaExpectedRateA + $sigmaExpectedRateB * $sigmaExpectedRateB); $pValue = 2 * stats_cdf_normal(-$rateDeviation, 0, $sigmaExpectedRateDeviation, 1); $variationCount = $this->getSplittest()->getVariations()->getCount(); $independentExperimentCount = max(1, $variationCount - 1); $pValueSidak = 1 - pow(1 - $pValue, $independentExperimentCount); return $pValueSidak; }
public static function sdFromPercentBelowX($prob, $x, $mean) { $prob = self::formatPercent($prob); return stats_cdf_normal($prob, $x, $mean, 4); }