public function testNotWithstandingCrashesJobsAreEventuallyPerformed()
{
$this->limitTo(100)->forAll(Generator\bind(Generator\choose(1, 4), function ($workers) {
return Generator\tuple(Generator\constant($workers), Generator\seq(Generator\oneOf(Generator\map(function ($durationAndTag) {
list($duration, $tag) = $durationAndTag;
return ['enqueueJob', $duration, $tag];
}, Generator\tuple(Generator\nat(), Generator\elements(['generic', 'fast-lane']))), Generator\map(function ($workerIndex) {
return ['restartWorkerGracefully', $workerIndex];
}, Generator\choose(0, $workers - 1)), Generator\map(function ($workerIndex) {
return ['restartWorkerByKilling', $workerIndex];
}, Generator\choose(0, $workers - 1)), Generator\constant('restartRecruiterGracefully'), Generator\constant('restartRecruiterByKilling'), Generator\map(function ($milliseconds) {
return ['sleep', $milliseconds];
}, Generator\choose(1, 1000)))));
}))->hook(Listener\log('/tmp/recruiter-test-iterations.log'))->hook(Listener\collectFrequencies())->disableShrinking()->then(function ($tuple) {
list($workers, $actions) = $tuple;
$this->clean();
$this->start($workers);
foreach ($actions as $action) {
$this->logAction($action);
if (is_array($action)) {
$arguments = $action;
$method = array_shift($arguments);
call_user_func_array([$this, $method], $arguments);
} else {
$this->{$action}();
}
}
$estimatedTime = max(count($actions) * 4, 60);
Timeout::inSeconds($estimatedTime, function () {
return "all {$this->jobs} jobs to be performed. Now is " . date('c') . " Logs: " . $this->files();
})->until(function () {
return $this->jobRepository->countArchived() === $this->jobs;
});
$at = T\now();
$statistics = $this->recruiter->statistics($tag = null, $at);
$this->assertInvariantsOnStatistics($statistics);
// TODO: remove duplication
$statisticsByTag = [];
$cumulativeThroughput = 0;
foreach (['generic', 'fast-lane'] as $tag) {
$statisticsByTag[$tag] = $this->recruiter->statistics($tag, $at);
$this->assertInvariantsOnStatistics($statisticsByTag[$tag]);
$cumulativeThroughput += $statisticsByTag[$tag]['throughput']['value'];
}
var_dump($statistics, $statisticsByTag);
// TODO: add tolerance
$this->assertEquals($statistics['throughput']['value'], $cumulativeThroughput);
});
}
protected function stopProcessWithSignal(array $processAndPipes, $signal)
{
list($process, $pipes, $name) = $processAndPipes;
proc_terminate($process, $signal);
$this->lastStatus = proc_get_status($process);
Timeout::inSeconds(30, function () {
return 'termination of process: ' . var_export($this->lastStatus, true);
})->until(function () use($process) {
$this->lastStatus = proc_get_status($process);
return $this->lastStatus['running'] == false;
});
}
