/**
* @param CM_Model_SplittestVariation $variationWorse
* @return float|null P-value with Šidák correction
*/
public function getSignificance(CM_Model_SplittestVariation $variationWorse)
{
$fixturesA = $this->getFixtureCount();
$fixturesB = $variationWorse->getFixtureCount();
if (!$fixturesA || !$fixturesB) {
return null;
}
$conversionsA = $this->getConversionCount();
$conversionsB = $variationWorse->getConversionCount();
if (!$conversionsA || !$conversionsB) {
return null;
}
$weightA = $this->getConversionWeight();
$weightB = $variationWorse->getConversionWeight();
if (!$weightA || !$weightB) {
return null;
}
$rateA = $weightA / $fixturesA;
$rateB = $weightB / $fixturesB;
$netRateA = $weightA / $conversionsA;
$netRateB = $weightB / $conversionsB;
$fixturesTotal = $fixturesA + $fixturesB;
$weightTotal = $weightA + $weightB;
$rateTotal = $weightTotal / $fixturesTotal;
$conversionsExpectedA = $rateTotal * $fixturesA / $netRateA;
$conversionsExpectedB = $rateTotal * $fixturesB / $netRateB;
$varianceExpectedA = $conversionsExpectedA * (1 - $conversionsExpectedA / $fixturesA);
$varianceExpectedB = $conversionsExpectedB * (1 - $conversionsExpectedB / $fixturesB);
if ($varianceExpectedA < 9 || $varianceExpectedB < 9) {
return null;
}
$sigmaExpectedA = sqrt($varianceExpectedA);
$sigmaExpectedB = sqrt($varianceExpectedB);
if ($conversionsExpectedA - 3 * $sigmaExpectedA < 0 || $conversionsExpectedB - 3 * $sigmaExpectedB < 0) {
return null;
}
if ($conversionsExpectedA + 3 * $sigmaExpectedA > $fixturesA || $conversionsExpectedB + 3 * $sigmaExpectedB > $fixturesB) {
return null;
}
$rateDeviation = abs($rateA - $rateB);
$sigmaExpectedRateA = $sigmaExpectedA * $netRateA / $fixturesA;
$sigmaExpectedRateB = $sigmaExpectedB * $netRateB / $fixturesB;
$sigmaExpectedRateDeviation = sqrt($sigmaExpectedRateA * $sigmaExpectedRateA + $sigmaExpectedRateB * $sigmaExpectedRateB);
$pValue = 2 * stats_cdf_normal(-$rateDeviation, 0, $sigmaExpectedRateDeviation, 1);
$variationCount = $this->getSplittest()->getVariations()->getCount();
$independentExperimentCount = max(1, $variationCount - 1);
$pValueSidak = 1 - pow(1 - $pValue, $independentExperimentCount);
return $pValueSidak;
}
public function testFlushVariationCache()
{
$test = CM_Model_Splittest::create('foo', ['v1', 'v2']);
$variation1 = new CM_Model_SplittestVariation(CM_Db_Db::select('cm_splittestVariation', 'id', ['name' => 'v1'])->fetchColumn());
$variation2 = new CM_Model_SplittestVariation(CM_Db_Db::select('cm_splittestVariation', 'id', ['name' => 'v2'])->fetchColumn());
$variation2->setEnabled(false);
$fixture = $this->mockClass('CM_Splittest_Fixture')->newInstanceWithoutConstructor();
$fixture->mockMethod('getId')->set(1);
$fixture->mockMethod('getFixtureType')->set(1);
CMTest_TH::timeForward(1);
$variation = CMTest_TH::callProtectedMethod($test, '_getVariationFixture', [$fixture]);
$this->assertSame('v1', $variation);
$test->flush();
$variation2->setEnabled(true);
$variation1->setEnabled(false);
$variation = CMTest_TH::callProtectedMethod($test, '_getVariationFixture', [$fixture]);
$this->assertSame('v2', $variation);
}
public function trackSplittest(CM_Splittest_Fixture $fixture, CM_Model_SplittestVariation $variation)
{
$nameSplittest = $variation->getSplittest()->getName();
$nameVariation = $variation->getName();
switch ($fixture->getFixtureType()) {
case CM_Splittest_Fixture::TYPE_REQUEST_CLIENT:
$this->setRequestClientId($fixture->getId());
break;
case CM_Splittest_Fixture::TYPE_USER:
$this->setUserId($fixture->getId());
break;
}
$trackEventJob = new CMService_KissMetrics_TrackPropertyListJob();
$trackEventJob->queue(array('code' => $this->_getCode(), 'identityList' => $this->_getIdentityList(), 'propertyList' => array('Splittest ' . $nameSplittest => $nameVariation)));
}