/**
*
*/
public function testGenerate()
{
if (extension_loaded('xdebug')) {
$this->assertTrue(true);
return;
}
$gameVersion = 'tw';
$pool = PdoFactory::makePool($gameVersion);
$groupedUidList = ['db3' => [474000, 474001, 474002]];
$payload = [];
$provider = new UserDetailProvider($gameVersion, $pool);
$batchReader = $provider->generate($groupedUidList);
foreach ($batchReader as $batch) {
$shardId = $batch['shardId'];
$this->assertStringStartsWith('db', $shardId);
$shardDataSet = $batch['dataSet'];
$this->assertTrue(is_array($shardDataSet));
foreach ($shardDataSet as $uid => $userInfo) {
$this->assertArrayHasKey('language', $userInfo);
$payload[$shardId][$uid] = $userInfo;
}
}
$foundUidList = [];
foreach ($payload as $shardId => $userList) {
$foundUidList[$shardId] = array_keys($userList);
foreach ($userList as $user) {
static::assertTrue(is_array($user));
static::assertArrayHasKey('uid', $user);
static::assertArrayHasKey('snsid', $user);
}
}
sort($groupedUidList);
sort($foundUidList);
static::assertEquals($groupedUidList, $foundUidList);
}
/**
* @param array $groupedUidList
*
* @return int
*/
public function updateES(array $groupedUidList)
{
appendLog(sprintf('%s start with memory usage %s', __METHOD__, PHP_Timer::resourceUsage()));
$count = array_sum(array_map(function (array $list) {
return count($list);
}, $groupedUidList));
if ($count === 0) {
appendLog(sprintf('%s: have 0 user to sync', __METHOD__));
return 0;
}
$start = microtime(true);
$groupedDetail = $this->dataProvider->generate($groupedUidList);
$delta = microtime(true) - $start;
$totalUidCount = array_sum(array_map(function (array $uidList) {
return count($uidList);
}, $groupedUidList));
appendLog(sprintf('Total %d uids, read detail cost %s', $totalUidCount, PHP_Timer::secondsToTimeString($delta)));
$bufferSyncAction = function (array $users) {
$count = count($users);
appendLog(sprintf('ES updater have %d user to sync', $count));
$deltaList = $this->indexer->batchUpdate($users, function ($userCount, $delta) {
appendLog(sprintf('on ES batch update of %d users cost %s', $userCount, PHP_Timer::secondsToTimeString($delta)));
});
$totalDelta = array_sum($deltaList);
appendLog(sprintf('Sync %d users to ES cost %s with average cost %s', $count, PHP_Timer::secondsToTimeString($totalDelta), PHP_Timer::secondsToTimeString($totalDelta / $count)));
};
$buffer = new WriteBuffer($bufferSyncAction, 500);
foreach ($groupedDetail as $payload) {
$shardId = $payload['shardId'];
$shardUserList = $payload['dataSet'];
$count = count($shardUserList);
if ($count === 0) {
continue;
}
appendLog(sprintf('%s have %d user to sync', $shardId, $count));
foreach ($shardUserList as $userInfo) {
$buffer->add($userInfo);
}
}
$buffer->sync();
}
$msg = basename(__FILE__) . ': process for ' . $calendarDay . ' run with ts ' . date('c');
appendLog($msg);
$groupedUidList = $installUidProvider->generate($calendarDay, function ($shardId, $userCount, $delta) {
if ($userCount === 0) {
return;
}
appendLog(sprintf('%s install(%d) cost %s', $shardId, $userCount, PHP_Timer::secondsToTimeString($delta)));
});
$distribution = array_map(function (array $uidList) {
return count($uidList);
}, $groupedUidList);
$newInstallCount = array_sum($distribution);
appendLog(sprintf('Total %d new install on %s', $newInstallCount, $calendarDay));
$totalUser += $newInstallCount;
$start = microtime(true);
$groupedDetail = $userDetailProvider->generate($groupedUidList);
$delta = microtime(true) - $start;
appendLog(sprintf('Total %d new install on %s, read detail cost %s', $newInstallCount, $calendarDay, PHP_Timer::secondsToTimeString($delta)));
$esUpdateQueue = [];
foreach ($groupedDetail as $payload) {
$shardId = $payload['shardId'];
$shardUserList = $payload['dataSet'];
$count = count($shardUserList);
if ($count === 0) {
continue;
}
appendLog(sprintf('%s have %d user to sync', $shardId, $count));
$esUpdateQueue = array_merge($esUpdateQueue, $shardUserList);
$queueLength = count($esUpdateQueue);
if ($queueLength >= $magicNumber) {
appendLog(sprintf('%s: flush ES update queue: %d user on date %s to sync %s', date('c'), $queueLength, $calendarDay, PHP_Timer::resourceUsage()));
