/** * */ public function testGenerate() { if (extension_loaded('xdebug')) { $this->assertTrue(true); return; } $gameVersion = 'tw'; $pool = PdoFactory::makePool($gameVersion); $groupedUidList = ['db3' => [474000, 474001, 474002]]; $payload = []; $provider = new UserDetailProvider($gameVersion, $pool); $batchReader = $provider->generate($groupedUidList); foreach ($batchReader as $batch) { $shardId = $batch['shardId']; $this->assertStringStartsWith('db', $shardId); $shardDataSet = $batch['dataSet']; $this->assertTrue(is_array($shardDataSet)); foreach ($shardDataSet as $uid => $userInfo) { $this->assertArrayHasKey('language', $userInfo); $payload[$shardId][$uid] = $userInfo; } } $foundUidList = []; foreach ($payload as $shardId => $userList) { $foundUidList[$shardId] = array_keys($userList); foreach ($userList as $user) { static::assertTrue(is_array($user)); static::assertArrayHasKey('uid', $user); static::assertArrayHasKey('snsid', $user); } } sort($groupedUidList); sort($foundUidList); static::assertEquals($groupedUidList, $foundUidList); }
/** * @param array $groupedUidList * * @return int */ public function updateES(array $groupedUidList) { appendLog(sprintf('%s start with memory usage %s', __METHOD__, PHP_Timer::resourceUsage())); $count = array_sum(array_map(function (array $list) { return count($list); }, $groupedUidList)); if ($count === 0) { appendLog(sprintf('%s: have 0 user to sync', __METHOD__)); return 0; } $start = microtime(true); $groupedDetail = $this->dataProvider->generate($groupedUidList); $delta = microtime(true) - $start; $totalUidCount = array_sum(array_map(function (array $uidList) { return count($uidList); }, $groupedUidList)); appendLog(sprintf('Total %d uids, read detail cost %s', $totalUidCount, PHP_Timer::secondsToTimeString($delta))); $bufferSyncAction = function (array $users) { $count = count($users); appendLog(sprintf('ES updater have %d user to sync', $count)); $deltaList = $this->indexer->batchUpdate($users, function ($userCount, $delta) { appendLog(sprintf('on ES batch update of %d users cost %s', $userCount, PHP_Timer::secondsToTimeString($delta))); }); $totalDelta = array_sum($deltaList); appendLog(sprintf('Sync %d users to ES cost %s with average cost %s', $count, PHP_Timer::secondsToTimeString($totalDelta), PHP_Timer::secondsToTimeString($totalDelta / $count))); }; $buffer = new WriteBuffer($bufferSyncAction, 500); foreach ($groupedDetail as $payload) { $shardId = $payload['shardId']; $shardUserList = $payload['dataSet']; $count = count($shardUserList); if ($count === 0) { continue; } appendLog(sprintf('%s have %d user to sync', $shardId, $count)); foreach ($shardUserList as $userInfo) { $buffer->add($userInfo); } } $buffer->sync(); }
$msg = basename(__FILE__) . ': process for ' . $calendarDay . ' run with ts ' . date('c'); appendLog($msg); $groupedUidList = $installUidProvider->generate($calendarDay, function ($shardId, $userCount, $delta) { if ($userCount === 0) { return; } appendLog(sprintf('%s install(%d) cost %s', $shardId, $userCount, PHP_Timer::secondsToTimeString($delta))); }); $distribution = array_map(function (array $uidList) { return count($uidList); }, $groupedUidList); $newInstallCount = array_sum($distribution); appendLog(sprintf('Total %d new install on %s', $newInstallCount, $calendarDay)); $totalUser += $newInstallCount; $start = microtime(true); $groupedDetail = $userDetailProvider->generate($groupedUidList); $delta = microtime(true) - $start; appendLog(sprintf('Total %d new install on %s, read detail cost %s', $newInstallCount, $calendarDay, PHP_Timer::secondsToTimeString($delta))); $esUpdateQueue = []; foreach ($groupedDetail as $payload) { $shardId = $payload['shardId']; $shardUserList = $payload['dataSet']; $count = count($shardUserList); if ($count === 0) { continue; } appendLog(sprintf('%s have %d user to sync', $shardId, $count)); $esUpdateQueue = array_merge($esUpdateQueue, $shardUserList); $queueLength = count($esUpdateQueue); if ($queueLength >= $magicNumber) { appendLog(sprintf('%s: flush ES update queue: %d user on date %s to sync %s', date('c'), $queueLength, $calendarDay, PHP_Timer::resourceUsage()));