function ParseAuctionData($house, $snapshot, &$json)
{
global $maxPacketSize;
global $houseRegionCache;
global $equipBaseItemLevel;
global $usefulBonusesCache, $enoughBonusesSeenCache;
global $TIMELEFT_ENUM;
$snapshotString = date('Y-m-d H:i:s', $snapshot);
$startTimer = microtime(true);
$ourDb = DBConnect(true);
$region = $houseRegionCache[$house]['region'];
$existingIds = [];
$stmt = $ourDb->prepare(EXISTING_SQL);
$stmt->bind_param('i', $house);
$stmt->execute();
$id = $bid = $buy = $timeLeft = $infoKey = null;
$stmt->bind_result($id, $bid, $buy, $timeLeft, $infoKey);
while ($stmt->fetch()) {
$existingIds[$id] = [$bid, $buy, $timeLeft, $infoKey];
}
$stmt->close();
$stmt = $ourDb->prepare('SELECT id, species, breed FROM tblAuctionPet WHERE house = ?');
$stmt->bind_param('i', $house);
$stmt->execute();
$result = $stmt->get_result();
$existingPetIds = DBMapArray($result);
$stmt->close();
$naiveMax = 0;
$lowMax = -1;
$highMax = -1;
$hasRollOver = false;
$jsonAuctions = [];
if (isset($json['auctions']['auctions'])) {
$jsonAuctions =& $json['auctions']['auctions'];
} elseif (isset($json['auctions']) && count($json['auctions']) > 5) {
$jsonAuctions =& $json['auctions'];
}
if ($jsonAuctions) {
$auctionCount = count($jsonAuctions);
for ($x = 0; $x < $auctionCount; $x++) {
$auctionId = $jsonAuctions[$x]['auc'];
$naiveMax = max($naiveMax, $auctionId);
if ($auctionId < 0x20000000) {
$lowMax = max($lowMax, $auctionId);
}
if ($auctionId > 0x60000000) {
$highMax = max($highMax, $auctionId);
}
}
}
if ($lowMax != -1 && $highMax != -1) {
$hasRollOver = true;
$max = $lowMax;
// rolled over
} else {
$max = $naiveMax;
}
unset($naiveMax, $lowMax, $highMax);
$stmt = $ourDb->prepare('SELECT ifnull(maxid,0) FROM tblSnapshot s WHERE house = ? AND updated = (SELECT max(s2.updated) FROM tblSnapshot s2 WHERE s2.house = ? AND s2.updated < ?)');
$stmt->bind_param('iis', $house, $house, $snapshotString);
$stmt->execute();
$stmt->bind_result($lastMax);
if ($stmt->fetch() !== true) {
$lastMax = 0;
}
$stmt->close();
$stmt = $ourDb->prepare('UPDATE tblSnapshot SET maxid = ? WHERE house = ? AND updated = ?');
$stmt->bind_param('iis', $max, $house, $snapshotString);
$stmt->execute();
$stmt->close();
$stmt = $ourDb->prepare('SELECT unix_timestamp(updated) updated, maxid FROM tblSnapshot WHERE house = ? AND updated BETWEEN timestampadd(HOUR, -49, ?) AND ? ORDER BY updated ASC');
$stmt->bind_param('iss', $house, $snapshotString, $snapshotString);
$stmt->execute();
$result = $stmt->get_result();
$snapshotList = DBMapArray($result, null);
$stmt->close();
$prevSnapshot = $snapshot;
if (count($snapshotList)) {
$prevSnapshot = intval($snapshotList[count($snapshotList) - 1]['updated'], 10);
}
$snapshotWindow = $snapshot - $prevSnapshot;
$expiredLength = 1;
// any missing shorts can be considered expired
if ($snapshotWindow > 1800) {
// 30 mins
$expiredLength = 2;
// any missing shorts or mediums can be expired
}
$sqlStart = 'REPLACE INTO tblAuction (house, id, item, quantity, bid, buy, seller, timeleft) VALUES ';
$sqlStartPet = 'REPLACE INTO tblAuctionPet (house, id, species, breed, `level`, quality) VALUES ';
$sqlStartExtra = 'REPLACE INTO tblAuctionExtra (house, id, `rand`, `seed`, `context`, `lootedlevel`, `level`, `bonusset`';
for ($x = 1; $x <= MAX_BONUSES; $x++) {
$sqlStartExtra .= ", bonus{$x}";
}
$sqlStartExtra .= ') VALUES ';
$sqlStartBonusesSeen = 'INSERT INTO tblItemBonusesSeen (item, bonusset, bonus1, bonus2, bonus3, bonus4, observed) VALUES ';
$sqlEndBonusesSeen = ' ON DUPLICATE KEY UPDATE observed = observed + 1';
$sqlStartLevelsSeen = 'INSERT IGNORE INTO tblItemLevelsSeen (item, bonusset, `level`) VALUES ';
$totalAuctions = 0;
$itemInfo = array();
$petInfo = array();
$sellerInfo = array();
$expiredItemInfo = array();
if ($jsonAuctions) {
$auctionCount = count($jsonAuctions);
DebugMessage("House " . str_pad($house, 5, ' ', STR_PAD_LEFT) . " prepping {$auctionCount} auctions");
$sellerCount = 0;
for ($x = 0; $x < $auctionCount; $x++) {
$auction =& $jsonAuctions[$x];
if ($auction['owner'] == '???') {
continue;
}
if (!isset($sellerInfo[$auction['ownerRealm']])) {
$sellerInfo[$auction['ownerRealm']] = array();
}
if (!isset($sellerInfo[$auction['ownerRealm']][$auction['owner']])) {
$sellerCount++;
$sellerInfo[$auction['ownerRealm']][$auction['owner']] = array('new' => 0, 'total' => 0, 'id' => 0, 'items' => []);
}
$sellerInfo[$auction['ownerRealm']][$auction['owner']]['total']++;
if ((!$hasRollOver || $auction['auc'] < 0x20000000) && $auction['auc'] > $lastMax) {
$sellerInfo[$auction['ownerRealm']][$auction['owner']]['new']++;
$itemId = intval($auction['item'], 10);
if (!isset($sellerInfo[$auction['ownerRealm']][$auction['owner']]['items'][$itemId])) {
$sellerInfo[$auction['ownerRealm']][$auction['owner']]['items'][$itemId] = [0, 0];
}
$sellerInfo[$auction['ownerRealm']][$auction['owner']]['items'][$itemId][0]++;
$sellerInfo[$auction['ownerRealm']][$auction['owner']]['items'][$itemId][1] += $auction['quantity'];
}
}
DebugMessage("House " . str_pad($house, 5, ' ', STR_PAD_LEFT) . " getting {$sellerCount} seller IDs");
GetSellerIds($region, $sellerInfo, $snapshot);
$sql = $sqlPet = $sqlExtra = $sqlBonusesSeen = $sqlLevelsSeen = '';
$delayedAuctionSql = [];
DebugMessage("House " . str_pad($house, 5, ' ', STR_PAD_LEFT) . " parsing {$auctionCount} auctions");
while ($auction = array_pop($jsonAuctions)) {
if (isset($auction['petBreedId'])) {
$auction['petBreedId'] = ($auction['petBreedId'] - 3) % 10 + 3;
// squash gender
}
$auction['timeLeft'] = isset($TIMELEFT_ENUM[$auction['timeLeft']]) ? $TIMELEFT_ENUM[$auction['timeLeft']] : 0;
$auction['lootedLevel'] = null;
if (isset($auction['modifiers'])) {
foreach ($auction['modifiers'] as $modObj) {
if (isset($modObj['type']) && $modObj['type'] == 9) {
$auction['lootedLevel'] = intval($modObj['value']);
}
}
}
$totalAuctions++;
$itemInfoKey = false;
$bonusSet = 0;
$bonuses = [];
$bonusItemLevel = null;
$priceScaling = null;
if (!isset($auction['petSpeciesId']) && isset($equipBaseItemLevel[$auction['item']]) && isset($auction['bonusLists'])) {
for ($y = 0; $y < count($auction['bonusLists']); $y++) {
if (isset($auction['bonusLists'][$y]['bonusListId']) && $auction['bonusLists'][$y]['bonusListId']) {
$bonuses[] = intval($auction['bonusLists'][$y]['bonusListId'], 10);
}
}
$bonuses = array_unique($bonuses, SORT_NUMERIC);
sort($bonuses, SORT_NUMERIC);
$bonusSet = $bonuses ? GetBonusSet($bonuses) : 0;
$bonusItemLevel = GetBonusItemLevel($bonuses, $equipBaseItemLevel[$auction['item']], $auction['lootedLevel']);
if ($bonusItemLevel - $equipBaseItemLevel[$auction['item']] != 0) {
$priceScaling = pow(1.15, ($bonusItemLevel - $equipBaseItemLevel[$auction['item']]) / 15);
}
}
if ($auction['buyout'] != 0) {
if (isset($auction['petSpeciesId'])) {
if (!isset($petInfo[$auction['petSpeciesId']][$auction['petBreedId']])) {
$petInfo[$auction['petSpeciesId']][$auction['petBreedId']] = array('a' => array(), 'tq' => 0);
}
$petInfo[$auction['petSpeciesId']][$auction['petBreedId']]['a'][] = array('q' => $auction['quantity'], 'p' => $auction['buyout']);
$petInfo[$auction['petSpeciesId']][$auction['petBreedId']]['tq'] += $auction['quantity'];
} else {
$itemInfoKey = str_pad($auction['item'], ITEM_ID_PAD, '0', STR_PAD_LEFT) . ":{$bonusSet}";
if (!isset($itemInfo[$itemInfoKey])) {
$itemInfo[$itemInfoKey] = array('a' => array(), 'tq' => 0);
}
$itemInfo[$itemInfoKey]['a'][] = array('q' => $auction['quantity'], 'p' => isset($priceScaling) ? $auction['buyout'] / $priceScaling : $auction['buyout']);
$itemInfo[$itemInfoKey]['tq'] += $auction['quantity'];
}
}
if (isset($existingIds[$auction['auc']])) {
$needUpdate = $auction['bid'] != $existingIds[$auction['auc']][EXISTING_COL_BID];
$needUpdate |= $auction['timeLeft'] != $existingIds[$auction['auc']][EXISTING_COL_TIMELEFT];
unset($existingIds[$auction['auc']]);
unset($existingPetIds[$auction['auc']]);
if (!$needUpdate) {
continue;
}
} else {
// new auction
if ($auction['buyout'] != 0) {
if ($itemInfoKey !== false) {
if (!isset($expiredItemInfo['n'][$itemInfoKey])) {
$expiredItemInfo['n'][$itemInfoKey] = 0;
}
$expiredItemInfo['n'][$itemInfoKey]++;
}
}
if (isset($equipBaseItemLevel[$auction['item']])) {
if (!isset($enoughBonusesSeenCache["{$auction['item']}:{$bonusSet}"])) {
$usefulBonuses = [];
foreach ($bonuses as $bonus) {
if (isset($usefulBonusesCache[$bonus])) {
$usefulBonuses[$bonus] = $bonus;
}
}
sort($usefulBonuses, SORT_NUMERIC);
switch (count($usefulBonuses)) {
case 0:
$usefulBonuses[] = 0;
case 1:
$usefulBonuses[] = 0;
case 2:
$usefulBonuses[] = 0;
case 3:
$usefulBonuses[] = 0;
}
$thisSql = sprintf('(%u,%u,%u,%u,%u,%u,1)', $auction['item'], $bonusSet, $usefulBonuses[0], $usefulBonuses[1], $usefulBonuses[2], $usefulBonuses[3]);
if (strlen($sqlBonusesSeen) + 5 + strlen($thisSql) + strlen($sqlEndBonusesSeen) > $maxPacketSize) {
if (GetDBLock($ourDb, DB_LOCK_SEEN_BONUSES)) {
DebugMessage("House " . str_pad($house, 5, ' ', STR_PAD_LEFT) . " updating seen bonuses (" . round($totalAuctions / $auctionCount * 100) . '%)');
DBQueryWithError($ourDb, $sqlBonusesSeen . $sqlEndBonusesSeen);
ReleaseDBLock($ourDb, DB_LOCK_SEEN_BONUSES);
} else {
DebugMessage("Could not obtain " . DB_LOCK_SEEN_BONUSES . " DB lock, skipping update of seen bonuses.", E_USER_WARNING);
}
$sqlBonusesSeen = '';
}
$sqlBonusesSeen .= ($sqlBonusesSeen ? ',' : $sqlStartBonusesSeen) . $thisSql;
}
if (!is_null($bonusItemLevel)) {
$thisSql = sprintf('(%u,%u,%u)', $auction['item'], $bonusSet, $bonusItemLevel);
if (strlen($sqlLevelsSeen) + 5 + strlen($thisSql) > $maxPacketSize) {
if (GetDBLock($ourDb, DB_LOCK_SEEN_ILVLS)) {
DebugMessage("House " . str_pad($house, 5, ' ', STR_PAD_LEFT) . " updating seen levels (" . round($totalAuctions / $auctionCount * 100) . '%)');
DBQueryWithError($ourDb, $sqlLevelsSeen);
ReleaseDBLock($ourDb, DB_LOCK_SEEN_ILVLS);
} else {
DebugMessage("Could not obtain " . DB_LOCK_SEEN_ILVLS . " DB lock, skipping update of seen levels.", E_USER_WARNING);
}
$sqlLevelsSeen = '';
}
$sqlLevelsSeen .= ($sqlLevelsSeen ? ',' : $sqlStartLevelsSeen) . $thisSql;
}
}
}
$thisSql = sprintf('(%u, %u, %u, %u, %u, %u, %u, %u)', $house, $auction['auc'], $auction['item'], $auction['quantity'], $auction['bid'], $auction['buyout'], $auction['owner'] == '???' ? 0 : $sellerInfo[$auction['ownerRealm']][$auction['owner']]['id'], $auction['timeLeft']);
if (strlen($sql) + 5 + strlen($thisSql) > $maxPacketSize) {
DebugMessage("House " . str_pad($house, 5, ' ', STR_PAD_LEFT) . " updating tblAuction (" . round($totalAuctions / $auctionCount * 100) . '%)');
DBQueryWithError($ourDb, $sql);
$sql = '';
}
$sql .= ($sql == '' ? $sqlStart : ',') . $thisSql;
if (isset($auction['petSpeciesId'])) {
$thisSql = sprintf('(%u, %u, %u, %u, %u, %u)', $house, $auction['auc'], $auction['petSpeciesId'], $auction['petBreedId'], $auction['petLevel'], $auction['petQualityId']);
if (strlen($sqlPet) + 5 + strlen($thisSql) > $maxPacketSize) {
$delayedAuctionSql[] = $sqlPet;
// delayed since tblAuction row must be inserted first for foreign key
$sqlPet = '';
}
$sqlPet .= ($sqlPet == '' ? $sqlStartPet : ',') . $thisSql;
} else {
if (isset($equipBaseItemLevel[$auction['item']])) {
if (count($bonuses) || $auction['rand'] || $auction['context']) {
for ($y = count($bonuses); $y < MAX_BONUSES; $y++) {
$bonuses[] = 'null';
}
$bonuses = implode(',', $bonuses);
$thisSql = sprintf('(%u,%u,%d,%d,%u,%s,%s,%u,%s)', $house, $auction['auc'], $auction['rand'], $auction['seed'], $auction['context'], isset($auction['lootedLevel']) ? $auction['lootedLevel'] : 'null', isset($bonusItemLevel) ? $bonusItemLevel : 'null', $bonusSet, $bonuses);
if (strlen($sqlExtra) + 5 + strlen($thisSql) > $maxPacketSize) {
$delayedAuctionSql[] = $sqlExtra;
// delayed since tblAuction row must be inserted first for foreign key
$sqlExtra = '';
}
$sqlExtra .= ($sqlExtra == '' ? $sqlStartExtra : ',') . $thisSql;
}
}
}
}
if ($sqlBonusesSeen != '') {
if (GetDBLock($ourDb, DB_LOCK_SEEN_BONUSES)) {
DebugMessage("House " . str_pad($house, 5, ' ', STR_PAD_LEFT) . " updating seen bonuses");
DBQueryWithError($ourDb, $sqlBonusesSeen . $sqlEndBonusesSeen);
ReleaseDBLock($ourDb, DB_LOCK_SEEN_BONUSES);
} else {
DebugMessage("Could not obtain " . DB_LOCK_SEEN_BONUSES . " DB lock, skipping update of seen bonuses.", E_USER_WARNING);
}
}
if ($sqlLevelsSeen != '') {
if (GetDBLock($ourDb, DB_LOCK_SEEN_ILVLS)) {
DebugMessage("House " . str_pad($house, 5, ' ', STR_PAD_LEFT) . " updating seen levels");
DBQueryWithError($ourDb, $sqlLevelsSeen);
ReleaseDBLock($ourDb, DB_LOCK_SEEN_ILVLS);
} else {
DebugMessage("Could not obtain " . DB_LOCK_SEEN_ILVLS . " DB lock, skipping update of seen levels.", E_USER_WARNING);
}
}
if ($sql != '') {
DebugMessage("House " . str_pad($house, 5, ' ', STR_PAD_LEFT) . " updating tblAuction");
DBQueryWithError($ourDb, $sql);
}
if ($sqlPet != '') {
$delayedAuctionSql[] = $sqlPet;
}
if ($sqlExtra != '') {
$delayedAuctionSql[] = $sqlExtra;
}
if (count($delayedAuctionSql)) {
DebugMessage("House " . str_pad($house, 5, ' ', STR_PAD_LEFT) . " updating tblAuctionExtra, tblAuctionPet");
}
while (count($delayedAuctionSql)) {
DBQueryWithError($ourDb, array_pop($delayedAuctionSql));
}
unset($sqlBonusesSeen, $sqlLevelsSeen, $sqlPet, $sqlExtra, $delayedAuctionSql);
$sql = <<<EOF
insert ignore into tblAuctionRare (house, id, prevseen) (
select a.house, a.id, tis.lastseen
from tblAuction a
left join tblAuctionExtra ae on ae.house=a.house and ae.id=a.id
left join tblItemSummary tis on tis.house=a.house and tis.item=a.item and tis.bonusset=ifnull(ae.bonusset,0)
where a.house = %d
and a.id > %d
and a.item not in (82800)
%s
and ifnull(tis.lastseen, '2000-01-01') < timestampadd(day,-14,'%s'))
EOF;
$sql = sprintf($sql, $house, $lastMax, $hasRollOver ? ' and a.id < 0x20000000 ' : '', $snapshotString);
DebugMessage("House " . str_pad($house, 5, ' ', STR_PAD_LEFT) . " updating tblAuctionRare");
DBQueryWithError($ourDb, $sql);
}
foreach ($existingIds as $existingId => &$oldRow) {
// all missing auctions
if (!isset($existingPetIds[$existingId])) {
// missing item auction
if ($oldRow[EXISTING_COL_BUY] > 0 && $oldRow[EXISTING_COL_TIMELEFT] > 0 && $oldRow[EXISTING_COL_TIMELEFT] <= $expiredLength) {
// probably expired item with buyout
$expiredPosted = date('Y-m-d', $snapshot - GetAuctionAge($existingId, $snapshot, $snapshotList));
if (!isset($expiredItemInfo[$expiredPosted][$oldRow[EXISTING_COL_INFOKEY]])) {
$expiredItemInfo[$expiredPosted][$oldRow[EXISTING_COL_INFOKEY]] = 0;
}
$expiredItemInfo[$expiredPosted][$oldRow[EXISTING_COL_INFOKEY]]++;
}
}
}
unset($oldRow);
$rareDeletes = [];
$preDeleted = count($itemInfo);
foreach ($existingIds as $existingId => &$oldRow) {
if (!isset($existingPetIds[$existingId]) && !isset($itemInfo[$oldRow[EXISTING_COL_INFOKEY]])) {
list($itemId, $bonusSet) = explode(':', $oldRow[EXISTING_COL_INFOKEY]);
$rareDeletes[$bonusSet][] = $itemId;
$itemInfo[$oldRow[EXISTING_COL_INFOKEY]] = array('tq' => 0, 'a' => array());
}
}
unset($oldRow);
DebugMessage("House " . str_pad($house, 5, ' ', STR_PAD_LEFT) . " updating " . count($itemInfo) . " item info (including " . (count($itemInfo) - $preDeleted) . " no longer available)");
UpdateItemInfo($house, $itemInfo, $snapshot);
$sql = 'delete from tblUserRareReport where house = %d and bonusset = %d and item in (%s)';
foreach ($rareDeletes as $bonusSet => $itemIds) {
$chunked = array_chunk($itemIds, 200);
foreach ($chunked as $chunk) {
DBQueryWithError($ourDb, sprintf($sql, $house, $bonusSet, implode(',', $chunk)));
}
}
$preDeleted = count($petInfo);
foreach ($existingPetIds as &$oldRow) {
if (!isset($petInfo[$oldRow['species']][$oldRow['breed']])) {
$petInfo[$oldRow['species']][$oldRow['breed']] = array('tq' => 0, 'a' => array());
}
}
unset($oldRow);
DebugMessage("House " . str_pad($house, 5, ' ', STR_PAD_LEFT) . " updating " . count($petInfo) . " pet info (including " . (count($petInfo) - $preDeleted) . " no longer available)");
UpdatePetInfo($house, $petInfo, $snapshot);
DebugMessage("House " . str_pad($house, 5, ' ', STR_PAD_LEFT) . " updating seller history");
UpdateSellerInfo($sellerInfo, $house, $snapshot);
if (count($expiredItemInfo) > 0) {
$sqlStart = 'INSERT INTO tblItemExpired (item, bonusset, house, `when`, created, expired) VALUES ';
$sqlEnd = ' ON DUPLICATE KEY UPDATE created=created+values(created), expired=expired+values(expired)';
$sql = '';
if (isset($expiredItemInfo['n'])) {
DebugMessage("House " . str_pad($house, 5, ' ', STR_PAD_LEFT) . " adding new auctions for " . count($expiredItemInfo['n']) . " items");
$snapshotDay = date('Y-m-d', $snapshot);
$expiredCount = 0;
foreach ($expiredItemInfo['n'] as $infoKey => $createdCount) {
$keyParts = explode(':', $infoKey);
$sqlPart = sprintf('(%u, %u, %u, \'%s\', %u, %u)', $keyParts[0], $keyParts[1], $house, $snapshotDay, $createdCount, $expiredCount);
if (strlen($sql) + 10 + strlen($sqlPart) + strlen($sqlEnd) > $maxPacketSize) {
DBQueryWithError($ourDb, $sql . $sqlEnd);
$sql = '';
}
$sql .= ($sql == '' ? $sqlStart : ',') . $sqlPart;
}
unset($expiredItemInfo['n']);
}
if ($sql != '') {
DBQueryWithError($ourDb, $sql . $sqlEnd);
$sql = '';
}
$createdCount = 0;
$snapshotDays = array_keys($expiredItemInfo);
foreach ($snapshotDays as $snapshotDay) {
//DebugMessage("House " . str_pad($house, 5, ' ', STR_PAD_LEFT) . " adding expired auctions from $snapshotDay for ".count($expiredItemInfo[$snapshotDay])." items");
foreach ($expiredItemInfo[$snapshotDay] as $infoKey => $expiredCount) {
$keyParts = explode(':', $infoKey);
$sqlPart = sprintf('(%u, %u, %u, \'%s\', %u, %u)', $keyParts[0], $keyParts[1], $house, $snapshotDay, $createdCount, $expiredCount);
if (strlen($sql) + 10 + strlen($sqlPart) + strlen($sqlEnd) > $maxPacketSize) {
DBQueryWithError($ourDb, $sql . $sqlEnd);
$sql = '';
}
$sql .= ($sql == '' ? $sqlStart : ',') . $sqlPart;
}
}
if ($sql != '') {
DBQueryWithError($ourDb, $sql . $sqlEnd);
}
}
if (count($existingIds) > 0) {
DebugMessage("House " . str_pad($house, 5, ' ', STR_PAD_LEFT) . " deleting " . count($existingIds) . " auctions");
$sqlStart = sprintf('DELETE FROM tblAuction WHERE house = %d AND id IN (', $house);
$sql = '';
foreach ($existingIds as $lostId => &$lostRow) {
if (strlen($sql) + 10 + strlen($lostId) > $maxPacketSize) {
DBQueryWithError($ourDb, $sql . ')');
$sql = '';
}
$sql .= ($sql == '' ? $sqlStart : ',') . $lostId;
}
unset($lostRow);
if ($sql != '') {
DBQueryWithError($ourDb, $sql . ')');
}
}
$snapshotHourStart = date('Y-m-d H', $snapshot) . ':00:00';
$stmt = $ourDb->prepare('UPDATE tblSnapshot SET flags = flags | 1 WHERE house = ? AND updated between ? and timestampadd(second, 3599, ?) AND updated != ?');
$stmt->bind_param('isss', $house, $snapshotHourStart, $snapshotHourStart, $snapshotString);
$stmt->execute();
$stmt->close();
$ourDb->close();
MCSetHouse($house, 'ts', $snapshot);
DebugMessage("House " . str_pad($house, 5, ' ', STR_PAD_LEFT) . " finished with {$totalAuctions} auctions in " . round(microtime(true) - $startTimer, 2) . " sec");
}
function FetchSnapshot()
{
global $db, $region;
$lockName = "fetchsnapshot_{$region}";
$stmt = $db->prepare('select get_lock(?, 30)');
$stmt->bind_param('s', $lockName);
$stmt->execute();
$lockSuccess = null;
$stmt->bind_result($lockSuccess);
if (!$stmt->fetch()) {
$lockSuccess = null;
}
$stmt->close();
if ($lockSuccess != '1') {
DebugMessage("Could not get mysql lock for {$lockName}.");
return 30;
}
$earlyCheckSeconds = EARLY_CHECK_SECONDS;
$nextRealmSql = <<<ENDSQL
select r.house, min(r.canonical), count(*) c, ifnull(hc.nextcheck, s.nextcheck) upd, s.lastupdate, s.mindelta, hc.lastchecksuccessresult
from tblRealm r
left join (
select deltas.house, timestampadd(second, least(ifnull(min(delta)-{$earlyCheckSeconds}, 45*60), 150*60), max(deltas.updated)) nextcheck, max(deltas.updated) lastupdate, min(delta) mindelta
from (
select sn.updated,
if(@prevhouse = sn.house and sn.updated > timestampadd(hour, -72, now()), unix_timestamp(sn.updated) - @prevdate, null) delta,
@prevdate := unix_timestamp(sn.updated) updated_ts,
@prevhouse := sn.house house
from (select @prevhouse := null, @prevdate := null) setup, tblSnapshot sn
order by sn.house, sn.updated) deltas
group by deltas.house
) s on s.house = r.house
left join tblHouseCheck hc on hc.house = r.house
where r.region = ?
and r.house is not null
and r.canonical is not null
group by r.house
order by ifnull(upd, '2000-01-01') asc, c desc, r.house asc
limit 1
ENDSQL;
$house = $slug = $realmCount = $nextDate = $lastDate = $minDelta = $lastSuccessJson = null;
$stmt = $db->prepare($nextRealmSql);
$stmt->bind_param('s', $region);
$stmt->execute();
$stmt->bind_result($house, $slug, $realmCount, $nextDate, $lastDate, $minDelta, $lastSuccessJson);
$gotRealm = $stmt->fetch() === true;
$stmt->close();
if (!$gotRealm) {
DebugMessage("No {$region} realms to fetch!");
ReleaseDBLock($lockName);
return 30;
}
if (strtotime($nextDate) > time() && strtotime($nextDate) < time() + 3.5 * 60 * 60) {
$delay = strtotime($nextDate) - time();
DebugMessage("No {$region} realms ready yet, waiting " . SecondsOrMinutes($delay) . ".");
ReleaseDBLock($lockName);
return $delay;
}
SetHouseNextCheck($house, time() + 600, null);
ReleaseDBLock($lockName);
DebugMessage("{$region} {$slug} fetch for house {$house} to update {$realmCount} realms, due since " . (is_null($nextDate) ? 'unknown' : SecondsOrMinutes(time() - strtotime($nextDate)) . ' ago'));
$url = GetBattleNetURL($region, "wow/auction/data/{$slug}");
$outHeaders = [];
$dta = [];
$json = \Newsstand\HTTP::Get($url, [], $outHeaders);
if ($json === false && isset($outHeaders['body'])) {
// happens if server returns non-200 code, but we'll want that json anyway
$json = $outHeaders['body'];
}
if ($json !== false) {
$dta = json_decode($json, true);
if (json_last_error() != JSON_ERROR_NONE) {
$dta = [];
}
}
if (!isset($dta['files']) && !is_null($lastSuccessJson)) {
// no files in current status json, probably "internal server error"
// check the headers on our last known good data url
$lastGoodDta = json_decode($lastSuccessJson, true);
if (json_last_error() != JSON_ERROR_NONE) {
DebugMessage("{$region} {$slug} invalid JSON for last successful json\n" . $lastSuccessJson, E_USER_WARNING);
} elseif (!isset($lastGoodDta['files'])) {
DebugMessage("{$region} {$slug} no files in the last success json?!", E_USER_WARNING);
} else {
usort($lastGoodDta['files'], 'AuctionFileSort');
$fileInfo = end($lastGoodDta['files']);
$oldModified = ceil(intval($fileInfo['lastModified'], 10) / 1000);
DebugMessage("{$region} {$slug} returned no files. Checking headers on URL from " . date('Y-m-d H:i:s', $oldModified));
$headers = \Newsstand\HTTP::Head(preg_replace('/^http:/', 'https:', $fileInfo['url']));
if (isset($headers['Last-Modified'])) {
$newModified = strtotime($headers['Last-Modified']);
$fileInfo['lastModified'] = $newModified * 1000;
$dta['files'] = [$fileInfo];
if (abs($oldModified - $newModified) < 10) {
DebugMessage("{$region} {$slug} data file has unchanged last modified date from last successful parse.");
} else {
DebugMessage("{$region} {$slug} data file modified " . date('Y-m-d H:i:s', $newModified) . ".");
}
} else {
DebugMessage("{$region} {$slug} data file failed fetching last modified date via HEAD method.");
}
}
}
if (!isset($dta['files'])) {
$delay = GetCheckDelay(strtotime($lastDate));
DebugMessage("{$region} {$slug} returned no files. Waiting " . SecondsOrMinutes($delay) . ".", E_USER_WARNING);
SetHouseNextCheck($house, time() + $delay, $json);
\Newsstand\HTTP::AbandonConnections();
return 0;
}
usort($dta['files'], 'AuctionFileSort');
$fileInfo = end($dta['files']);
$modified = ceil(intval($fileInfo['lastModified'], 10) / 1000);
$lastDateUnix = is_null($lastDate) ? $modified - 1 : strtotime($lastDate);
$delay = 0;
if (!is_null($minDelta) && $modified <= $lastDateUnix) {
if ($lastDateUnix + $minDelta > time()) {
// we checked for an earlier-than-expected snapshot, didn't see one
$delay = $lastDateUnix + $minDelta - time() + 8;
// next check will be 8 seconds after expected update
} else {
if ($lastDateUnix + $minDelta + 45 > time()) {
// this is the first check after we expected a new snapshot, but didn't see one.
// don't trust api, assume data file URL won't change, and check last-modified time on data file
$headers = \Newsstand\HTTP::Head(preg_replace('/^http:/', 'https:', $fileInfo['url']));
if (isset($headers['Last-Modified'])) {
$newModified = strtotime($headers['Last-Modified']);
if ($newModified > $modified) {
DebugMessage("{$region} {$slug} data file indicates last modified {$newModified} " . date('H:i:s', $newModified) . ", ignoring API result.");
$modified = $newModified;
} else {
if ($newModified == $modified) {
DebugMessage("{$region} {$slug} data file has last modified date matching API result.");
} else {
DebugMessage("{$region} {$slug} data file has last modified date earlier than API result: {$newModified} " . date('H:i:s', $newModified) . ".");
}
}
} else {
DebugMessage("{$region} {$slug} data file failed fetching last modified date via HEAD method.");
}
}
}
}
if ($modified <= $lastDateUnix) {
if ($delay <= 0) {
$delay = GetCheckDelay($modified);
}
DebugMessage("{$region} {$slug} still not updated since {$modified} " . date('H:i:s', $modified) . " (" . SecondsOrMinutes(time() - $modified) . " ago). Waiting " . SecondsOrMinutes($delay) . ".");
SetHouseNextCheck($house, time() + $delay, $json);
return 0;
}
DebugMessage("{$region} {$slug} updated {$modified} " . date('H:i:s', $modified) . " (" . SecondsOrMinutes(time() - $modified) . " ago), fetching auction data file");
$dlStart = microtime(true);
$data = \Newsstand\HTTP::Get(preg_replace('/^http:/', 'https:', $fileInfo['url']), [], $outHeaders);
$dlDuration = microtime(true) - $dlStart;
if (!$data || substr($data, -4) != "]\r\n}") {
if (!$data) {
DebugMessage("{$region} {$slug} data file empty. Waiting 5 seconds and trying again.");
sleep(5);
} else {
DebugMessage("{$region} {$slug} data file malformed. Waiting 10 seconds and trying again.");
sleep(10);
}
$dlStart = microtime(true);
$data = \Newsstand\HTTP::Get($fileInfo['url'] . (parse_url($fileInfo['url'], PHP_URL_QUERY) ? '&' : '?') . 'please', [], $outHeaders);
$dlDuration = microtime(true) - $dlStart;
}
if (!$data) {
DebugMessage("{$region} {$slug} data file empty. Will try again in 30 seconds.");
SetHouseNextCheck($house, time() + 30, $json);
\Newsstand\HTTP::AbandonConnections();
return 10;
}
if (substr($data, -4) != "]\r\n}") {
$delay = GetCheckDelay($modified);
DebugMessage("{$region} {$slug} data file still probably malformed. Waiting " . SecondsOrMinutes($delay) . ".");
SetHouseNextCheck($house, time() + $delay, $json);
return 0;
}
$xferBytes = isset($outHeaders['X-Original-Content-Length']) ? $outHeaders['X-Original-Content-Length'] : strlen($data);
DebugMessage("{$region} {$slug} data file " . strlen($data) . " bytes" . ($xferBytes != strlen($data) ? ' (transfer length ' . $xferBytes . ', ' . round($xferBytes / strlen($data) * 100, 1) . '%)' : '') . ", " . round($dlDuration, 2) . "sec, " . round($xferBytes / 1000 / $dlDuration) . "KBps");
if ($xferBytes >= strlen($data) && strlen($data) > 65536) {
DebugMessage('No compression? ' . print_r($outHeaders, true));
}
if ($xferBytes / 1000 / $dlDuration < 200 && in_array($region, ['US', 'EU'])) {
DebugMessage("Speed under 200KBps, closing persistent connections");
\Newsstand\HTTP::AbandonConnections();
}
$successJson = json_encode($dta);
// will include any updates from using lastSuccessJson
$stmt = $db->prepare('INSERT INTO tblHouseCheck (house, nextcheck, lastcheck, lastcheckresult, lastchecksuccess, lastchecksuccessresult) VALUES (?, NULL, now(), ?, now(), ?) ON DUPLICATE KEY UPDATE nextcheck=values(nextcheck), lastcheck=values(lastcheck), lastcheckresult=values(lastcheckresult), lastchecksuccess=values(lastchecksuccess), lastchecksuccessresult=values(lastchecksuccessresult)');
$stmt->bind_param('iss', $house, $json, $successJson);
$stmt->execute();
$stmt->close();
$stmt = $db->prepare('INSERT INTO tblSnapshot (house, updated) VALUES (?, from_unixtime(?))');
$stmt->bind_param('ii', $house, $modified);
$stmt->execute();
$stmt->close();
MCSet('housecheck_' . $house, time(), 0);
$fileName = "{$modified}-" . str_pad($house, 5, '0', STR_PAD_LEFT) . ".json";
file_put_contents(SNAPSHOT_PATH . $fileName, $data, LOCK_EX);
link(SNAPSHOT_PATH . $fileName, SNAPSHOT_PATH . 'parse/' . $fileName);
if (in_array($region, ['US', 'EU'])) {
link(SNAPSHOT_PATH . $fileName, SNAPSHOT_PATH . 'watch/' . $fileName);
}
unlink(SNAPSHOT_PATH . $fileName);
return 0;
}