/**
* @covers HashRing
*/
public function testHashRing()
{
$ring = new HashRing(array('s1' => 1, 's2' => 1, 's3' => 2, 's4' => 2, 's5' => 2, 's6' => 3));
$locations = array();
for ($i = 0; $i < 20; $i++) {
$locations["hello{$i}"] = $ring->getLocation("hello{$i}");
}
$expectedLocations = array("hello0" => "s5", "hello1" => "s6", "hello2" => "s2", "hello3" => "s5", "hello4" => "s6", "hello5" => "s4", "hello6" => "s5", "hello7" => "s4", "hello8" => "s5", "hello9" => "s5", "hello10" => "s3", "hello11" => "s6", "hello12" => "s1", "hello13" => "s3", "hello14" => "s3", "hello15" => "s5", "hello16" => "s4", "hello17" => "s6", "hello18" => "s6", "hello19" => "s3");
$this->assertEquals($expectedLocations, $locations, 'Items placed at proper locations');
$locations = array();
for ($i = 0; $i < 5; $i++) {
$locations["hello{$i}"] = $ring->getLocations("hello{$i}", 2);
}
$expectedLocations = array("hello0" => array("s5", "s6"), "hello1" => array("s6", "s4"), "hello2" => array("s2", "s1"), "hello3" => array("s5", "s6"), "hello4" => array("s6", "s4"));
$this->assertEquals($expectedLocations, $locations, 'Items placed at proper locations');
}
/**
* @param array $jobs
* @param HashRing $partitionRing
* @param integer $flags
* @return array List of Job object that could not be inserted
*/
protected function tryJobInsertions( array $jobs, HashRing &$partitionRing, $flags ) {
$jobsLeft = array();
// Because jobs are spread across partitions, per-job de-duplication needs
// to use a consistent hash to avoid allowing duplicate jobs per partition.
// When inserting a batch of de-duplicated jobs, QOS_ATOMIC is disregarded.
$uJobsByPartition = array(); // (partition name => job list)
foreach ( $jobs as $key => $job ) {
if ( $job->ignoreDuplicates() ) {
$sha1 = sha1( serialize( $job->getDeduplicationInfo() ) );
$uJobsByPartition[$partitionRing->getLocation( $sha1 )][] = $job;
unset( $jobs[$key] );
}
}
// Get the batches of jobs that are not de-duplicated
if ( $flags & self::QOS_ATOMIC ) {
$nuJobBatches = array( $jobs ); // all or nothing
} else {
// Split the jobs into batches and spread them out over servers if there
// are many jobs. This helps keep the partitions even. Otherwise, send all
// the jobs to a single partition queue to avoids the extra connections.
$nuJobBatches = array_chunk( $jobs, 300 );
}
// Insert the de-duplicated jobs into the queues...
foreach ( $uJobsByPartition as $partition => $jobBatch ) {
$queue = $this->partitionQueues[$partition];
try {
$ok = $queue->doBatchPush( $jobBatch, $flags | self::QOS_ATOMIC );
} catch ( JobQueueError $e ) {
$ok = false;
MWExceptionHandler::logException( $e );
}
if ( $ok ) {
$key = $this->getCacheKey( 'empty' );
$this->cache->set( $key, 'false', JobQueueDB::CACHE_TTL_LONG );
} else {
$partitionRing = $partitionRing->newWithoutLocation( $partition ); // blacklist
if ( !$partitionRing ) {
throw new JobQueueError( "Could not insert job(s), all partitions are down." );
}
$jobsLeft = array_merge( $jobsLeft, $jobBatch ); // not inserted
}
}
// Insert the jobs that are not de-duplicated into the queues...
foreach ( $nuJobBatches as $jobBatch ) {
$partition = ArrayUtils::pickRandom( $partitionRing->getLocationWeights() );
$queue = $this->partitionQueues[$partition];
try {
$ok = $queue->doBatchPush( $jobBatch, $flags | self::QOS_ATOMIC );
} catch ( JobQueueError $e ) {
$ok = false;
MWExceptionHandler::logException( $e );
}
if ( $ok ) {
$key = $this->getCacheKey( 'empty' );
$this->cache->set( $key, 'false', JobQueueDB::CACHE_TTL_LONG );
} else {
$partitionRing = $partitionRing->newWithoutLocation( $partition ); // blacklist
if ( !$partitionRing ) {
throw new JobQueueError( "Could not insert job(s), all partitions are down." );
}
$jobsLeft = array_merge( $jobsLeft, $jobBatch ); // not inserted
}
}
return $jobsLeft;
}
/**
* @param array $jobs
* @param array $partitionsTry
* @param integer $flags
* @return array List of Job object that could not be inserted
*/
protected function tryJobInsertions(array $jobs, array &$partitionsTry, $flags)
{
if (!count($partitionsTry)) {
return $jobs;
// can't insert anything
}
$jobsLeft = array();
$partitionRing = new HashRing($partitionsTry);
// Because jobs are spread across partitions, per-job de-duplication needs
// to use a consistent hash to avoid allowing duplicate jobs per partition.
// When inserting a batch of de-duplicated jobs, QOS_ATOMIC is disregarded.
$uJobsByPartition = array();
// (partition name => job list)
foreach ($jobs as $key => $job) {
if ($job->ignoreDuplicates()) {
$sha1 = sha1(serialize($job->getDeduplicationInfo()));
$uJobsByPartition[$partitionRing->getLocation($sha1)][] = $job;
unset($jobs[$key]);
}
}
// Get the batches of jobs that are not de-duplicated
if ($flags & self::QOS_ATOMIC) {
$nuJobBatches = array($jobs);
// all or nothing
} else {
// Split the jobs into batches and spread them out over servers if there
// are many jobs. This helps keep the partitions even. Otherwise, send all
// the jobs to a single partition queue to avoids the extra connections.
$nuJobBatches = array_chunk($jobs, 300);
}
// Insert the de-duplicated jobs into the queues...
foreach ($uJobsByPartition as $partition => $jobBatch) {
$queue = $this->partitionQueues[$partition];
if ($queue->doBatchPush($jobBatch, $flags)) {
$key = $this->getCacheKey('empty');
$this->cache->set($key, 'false', JobQueueDB::CACHE_TTL_LONG);
} else {
unset($partitionsTry[$partition]);
// blacklist partition
$jobsLeft = array_merge($jobsLeft, $jobBatch);
// not inserted
}
}
// Insert the jobs that are not de-duplicated into the queues...
foreach ($nuJobBatches as $jobBatch) {
$partition = ArrayUtils::pickRandom($partitionsTry);
if ($partition === false) {
// all partitions at 0 weight?
$jobsLeft = array_merge($jobsLeft, $jobBatch);
// not inserted
} else {
$queue = $this->partitionQueues[$partition];
if ($queue->doBatchPush($jobBatch, $flags)) {
$key = $this->getCacheKey('empty');
$this->cache->set($key, 'false', JobQueueDB::CACHE_TTL_LONG);
} else {
unset($partitionsTry[$partition]);
// blacklist partition
$jobsLeft = array_merge($jobsLeft, $jobBatch);
// not inserted
}
}
}
return $jobsLeft;
}
