Пример #1
0
function ParseAuctionData($house, $snapshot, &$json)
{
    global $maxPacketSize;
    global $houseRegionCache;
    global $equipBaseItemLevel;
    global $usefulBonusesCache, $enoughBonusesSeenCache;
    global $TIMELEFT_ENUM;
    $snapshotString = date('Y-m-d H:i:s', $snapshot);
    $startTimer = microtime(true);
    $ourDb = DBConnect(true);
    $region = $houseRegionCache[$house]['region'];
    $existingIds = [];
    $stmt = $ourDb->prepare(EXISTING_SQL);
    $stmt->bind_param('i', $house);
    $stmt->execute();
    $id = $bid = $buy = $timeLeft = $infoKey = null;
    $stmt->bind_result($id, $bid, $buy, $timeLeft, $infoKey);
    while ($stmt->fetch()) {
        $existingIds[$id] = [$bid, $buy, $timeLeft, $infoKey];
    }
    $stmt->close();
    $stmt = $ourDb->prepare('SELECT id, species, breed FROM tblAuctionPet WHERE house = ?');
    $stmt->bind_param('i', $house);
    $stmt->execute();
    $result = $stmt->get_result();
    $existingPetIds = DBMapArray($result);
    $stmt->close();
    $naiveMax = 0;
    $lowMax = -1;
    $highMax = -1;
    $hasRollOver = false;
    $jsonAuctions = [];
    if (isset($json['auctions']['auctions'])) {
        $jsonAuctions =& $json['auctions']['auctions'];
    } elseif (isset($json['auctions']) && count($json['auctions']) > 5) {
        $jsonAuctions =& $json['auctions'];
    }
    if ($jsonAuctions) {
        $auctionCount = count($jsonAuctions);
        for ($x = 0; $x < $auctionCount; $x++) {
            $auctionId = $jsonAuctions[$x]['auc'];
            $naiveMax = max($naiveMax, $auctionId);
            if ($auctionId < 0x20000000) {
                $lowMax = max($lowMax, $auctionId);
            }
            if ($auctionId > 0x60000000) {
                $highMax = max($highMax, $auctionId);
            }
        }
    }
    if ($lowMax != -1 && $highMax != -1) {
        $hasRollOver = true;
        $max = $lowMax;
        // rolled over
    } else {
        $max = $naiveMax;
    }
    unset($naiveMax, $lowMax, $highMax);
    $stmt = $ourDb->prepare('SELECT ifnull(maxid,0) FROM tblSnapshot s WHERE house = ? AND updated = (SELECT max(s2.updated) FROM tblSnapshot s2 WHERE s2.house = ? AND s2.updated < ?)');
    $stmt->bind_param('iis', $house, $house, $snapshotString);
    $stmt->execute();
    $stmt->bind_result($lastMax);
    if ($stmt->fetch() !== true) {
        $lastMax = 0;
    }
    $stmt->close();
    $stmt = $ourDb->prepare('UPDATE tblSnapshot SET maxid = ? WHERE house = ? AND updated = ?');
    $stmt->bind_param('iis', $max, $house, $snapshotString);
    $stmt->execute();
    $stmt->close();
    $stmt = $ourDb->prepare('SELECT unix_timestamp(updated) updated, maxid FROM tblSnapshot WHERE house = ? AND updated BETWEEN timestampadd(HOUR, -49, ?) AND ? ORDER BY updated ASC');
    $stmt->bind_param('iss', $house, $snapshotString, $snapshotString);
    $stmt->execute();
    $result = $stmt->get_result();
    $snapshotList = DBMapArray($result, null);
    $stmt->close();
    $prevSnapshot = $snapshot;
    if (count($snapshotList)) {
        $prevSnapshot = intval($snapshotList[count($snapshotList) - 1]['updated'], 10);
    }
    $snapshotWindow = $snapshot - $prevSnapshot;
    $expiredLength = 1;
    // any missing shorts can be considered expired
    if ($snapshotWindow > 1800) {
        // 30 mins
        $expiredLength = 2;
        // any missing shorts or mediums can be expired
    }
    $sqlStart = 'REPLACE INTO tblAuction (house, id, item, quantity, bid, buy, seller, timeleft) VALUES ';
    $sqlStartPet = 'REPLACE INTO tblAuctionPet (house, id, species, breed, `level`, quality) VALUES ';
    $sqlStartExtra = 'REPLACE INTO tblAuctionExtra (house, id, `rand`, `seed`, `context`, `lootedlevel`, `level`, `bonusset`';
    for ($x = 1; $x <= MAX_BONUSES; $x++) {
        $sqlStartExtra .= ", bonus{$x}";
    }
    $sqlStartExtra .= ') VALUES ';
    $sqlStartBonusesSeen = 'INSERT INTO tblItemBonusesSeen (item, bonusset, bonus1, bonus2, bonus3, bonus4, observed) VALUES ';
    $sqlEndBonusesSeen = ' ON DUPLICATE KEY UPDATE observed = observed + 1';
    $sqlStartLevelsSeen = 'INSERT IGNORE INTO tblItemLevelsSeen (item, bonusset, `level`) VALUES ';
    $totalAuctions = 0;
    $itemInfo = array();
    $petInfo = array();
    $sellerInfo = array();
    $expiredItemInfo = array();
    if ($jsonAuctions) {
        $auctionCount = count($jsonAuctions);
        DebugMessage("House " . str_pad($house, 5, ' ', STR_PAD_LEFT) . " prepping {$auctionCount} auctions");
        $sellerCount = 0;
        for ($x = 0; $x < $auctionCount; $x++) {
            $auction =& $jsonAuctions[$x];
            if ($auction['owner'] == '???') {
                continue;
            }
            if (!isset($sellerInfo[$auction['ownerRealm']])) {
                $sellerInfo[$auction['ownerRealm']] = array();
            }
            if (!isset($sellerInfo[$auction['ownerRealm']][$auction['owner']])) {
                $sellerCount++;
                $sellerInfo[$auction['ownerRealm']][$auction['owner']] = array('new' => 0, 'total' => 0, 'id' => 0, 'items' => []);
            }
            $sellerInfo[$auction['ownerRealm']][$auction['owner']]['total']++;
            if ((!$hasRollOver || $auction['auc'] < 0x20000000) && $auction['auc'] > $lastMax) {
                $sellerInfo[$auction['ownerRealm']][$auction['owner']]['new']++;
                $itemId = intval($auction['item'], 10);
                if (!isset($sellerInfo[$auction['ownerRealm']][$auction['owner']]['items'][$itemId])) {
                    $sellerInfo[$auction['ownerRealm']][$auction['owner']]['items'][$itemId] = [0, 0];
                }
                $sellerInfo[$auction['ownerRealm']][$auction['owner']]['items'][$itemId][0]++;
                $sellerInfo[$auction['ownerRealm']][$auction['owner']]['items'][$itemId][1] += $auction['quantity'];
            }
        }
        DebugMessage("House " . str_pad($house, 5, ' ', STR_PAD_LEFT) . " getting {$sellerCount} seller IDs");
        GetSellerIds($region, $sellerInfo, $snapshot);
        $sql = $sqlPet = $sqlExtra = $sqlBonusesSeen = $sqlLevelsSeen = '';
        $delayedAuctionSql = [];
        DebugMessage("House " . str_pad($house, 5, ' ', STR_PAD_LEFT) . " parsing {$auctionCount} auctions");
        while ($auction = array_pop($jsonAuctions)) {
            if (isset($auction['petBreedId'])) {
                $auction['petBreedId'] = ($auction['petBreedId'] - 3) % 10 + 3;
                // squash gender
            }
            $auction['timeLeft'] = isset($TIMELEFT_ENUM[$auction['timeLeft']]) ? $TIMELEFT_ENUM[$auction['timeLeft']] : 0;
            $auction['lootedLevel'] = null;
            if (isset($auction['modifiers'])) {
                foreach ($auction['modifiers'] as $modObj) {
                    if (isset($modObj['type']) && $modObj['type'] == 9) {
                        $auction['lootedLevel'] = intval($modObj['value']);
                    }
                }
            }
            $totalAuctions++;
            $itemInfoKey = false;
            $bonusSet = 0;
            $bonuses = [];
            $bonusItemLevel = null;
            $priceScaling = null;
            if (!isset($auction['petSpeciesId']) && isset($equipBaseItemLevel[$auction['item']]) && isset($auction['bonusLists'])) {
                for ($y = 0; $y < count($auction['bonusLists']); $y++) {
                    if (isset($auction['bonusLists'][$y]['bonusListId']) && $auction['bonusLists'][$y]['bonusListId']) {
                        $bonuses[] = intval($auction['bonusLists'][$y]['bonusListId'], 10);
                    }
                }
                $bonuses = array_unique($bonuses, SORT_NUMERIC);
                sort($bonuses, SORT_NUMERIC);
                $bonusSet = $bonuses ? GetBonusSet($bonuses) : 0;
                $bonusItemLevel = GetBonusItemLevel($bonuses, $equipBaseItemLevel[$auction['item']], $auction['lootedLevel']);
                if ($bonusItemLevel - $equipBaseItemLevel[$auction['item']] != 0) {
                    $priceScaling = pow(1.15, ($bonusItemLevel - $equipBaseItemLevel[$auction['item']]) / 15);
                }
            }
            if ($auction['buyout'] != 0) {
                if (isset($auction['petSpeciesId'])) {
                    if (!isset($petInfo[$auction['petSpeciesId']][$auction['petBreedId']])) {
                        $petInfo[$auction['petSpeciesId']][$auction['petBreedId']] = array('a' => array(), 'tq' => 0);
                    }
                    $petInfo[$auction['petSpeciesId']][$auction['petBreedId']]['a'][] = array('q' => $auction['quantity'], 'p' => $auction['buyout']);
                    $petInfo[$auction['petSpeciesId']][$auction['petBreedId']]['tq'] += $auction['quantity'];
                } else {
                    $itemInfoKey = str_pad($auction['item'], ITEM_ID_PAD, '0', STR_PAD_LEFT) . ":{$bonusSet}";
                    if (!isset($itemInfo[$itemInfoKey])) {
                        $itemInfo[$itemInfoKey] = array('a' => array(), 'tq' => 0);
                    }
                    $itemInfo[$itemInfoKey]['a'][] = array('q' => $auction['quantity'], 'p' => isset($priceScaling) ? $auction['buyout'] / $priceScaling : $auction['buyout']);
                    $itemInfo[$itemInfoKey]['tq'] += $auction['quantity'];
                }
            }
            if (isset($existingIds[$auction['auc']])) {
                $needUpdate = $auction['bid'] != $existingIds[$auction['auc']][EXISTING_COL_BID];
                $needUpdate |= $auction['timeLeft'] != $existingIds[$auction['auc']][EXISTING_COL_TIMELEFT];
                unset($existingIds[$auction['auc']]);
                unset($existingPetIds[$auction['auc']]);
                if (!$needUpdate) {
                    continue;
                }
            } else {
                // new auction
                if ($auction['buyout'] != 0) {
                    if ($itemInfoKey !== false) {
                        if (!isset($expiredItemInfo['n'][$itemInfoKey])) {
                            $expiredItemInfo['n'][$itemInfoKey] = 0;
                        }
                        $expiredItemInfo['n'][$itemInfoKey]++;
                    }
                }
                if (isset($equipBaseItemLevel[$auction['item']])) {
                    if (!isset($enoughBonusesSeenCache["{$auction['item']}:{$bonusSet}"])) {
                        $usefulBonuses = [];
                        foreach ($bonuses as $bonus) {
                            if (isset($usefulBonusesCache[$bonus])) {
                                $usefulBonuses[$bonus] = $bonus;
                            }
                        }
                        sort($usefulBonuses, SORT_NUMERIC);
                        switch (count($usefulBonuses)) {
                            case 0:
                                $usefulBonuses[] = 0;
                            case 1:
                                $usefulBonuses[] = 0;
                            case 2:
                                $usefulBonuses[] = 0;
                            case 3:
                                $usefulBonuses[] = 0;
                        }
                        $thisSql = sprintf('(%u,%u,%u,%u,%u,%u,1)', $auction['item'], $bonusSet, $usefulBonuses[0], $usefulBonuses[1], $usefulBonuses[2], $usefulBonuses[3]);
                        if (strlen($sqlBonusesSeen) + 5 + strlen($thisSql) + strlen($sqlEndBonusesSeen) > $maxPacketSize) {
                            if (GetDBLock($ourDb, DB_LOCK_SEEN_BONUSES)) {
                                DebugMessage("House " . str_pad($house, 5, ' ', STR_PAD_LEFT) . " updating seen bonuses (" . round($totalAuctions / $auctionCount * 100) . '%)');
                                DBQueryWithError($ourDb, $sqlBonusesSeen . $sqlEndBonusesSeen);
                                ReleaseDBLock($ourDb, DB_LOCK_SEEN_BONUSES);
                            } else {
                                DebugMessage("Could not obtain " . DB_LOCK_SEEN_BONUSES . " DB lock, skipping update of seen bonuses.", E_USER_WARNING);
                            }
                            $sqlBonusesSeen = '';
                        }
                        $sqlBonusesSeen .= ($sqlBonusesSeen ? ',' : $sqlStartBonusesSeen) . $thisSql;
                    }
                    if (!is_null($bonusItemLevel)) {
                        $thisSql = sprintf('(%u,%u,%u)', $auction['item'], $bonusSet, $bonusItemLevel);
                        if (strlen($sqlLevelsSeen) + 5 + strlen($thisSql) > $maxPacketSize) {
                            if (GetDBLock($ourDb, DB_LOCK_SEEN_ILVLS)) {
                                DebugMessage("House " . str_pad($house, 5, ' ', STR_PAD_LEFT) . " updating seen levels (" . round($totalAuctions / $auctionCount * 100) . '%)');
                                DBQueryWithError($ourDb, $sqlLevelsSeen);
                                ReleaseDBLock($ourDb, DB_LOCK_SEEN_ILVLS);
                            } else {
                                DebugMessage("Could not obtain " . DB_LOCK_SEEN_ILVLS . " DB lock, skipping update of seen levels.", E_USER_WARNING);
                            }
                            $sqlLevelsSeen = '';
                        }
                        $sqlLevelsSeen .= ($sqlLevelsSeen ? ',' : $sqlStartLevelsSeen) . $thisSql;
                    }
                }
            }
            $thisSql = sprintf('(%u, %u, %u, %u, %u, %u, %u, %u)', $house, $auction['auc'], $auction['item'], $auction['quantity'], $auction['bid'], $auction['buyout'], $auction['owner'] == '???' ? 0 : $sellerInfo[$auction['ownerRealm']][$auction['owner']]['id'], $auction['timeLeft']);
            if (strlen($sql) + 5 + strlen($thisSql) > $maxPacketSize) {
                DebugMessage("House " . str_pad($house, 5, ' ', STR_PAD_LEFT) . " updating tblAuction (" . round($totalAuctions / $auctionCount * 100) . '%)');
                DBQueryWithError($ourDb, $sql);
                $sql = '';
            }
            $sql .= ($sql == '' ? $sqlStart : ',') . $thisSql;
            if (isset($auction['petSpeciesId'])) {
                $thisSql = sprintf('(%u, %u, %u, %u, %u, %u)', $house, $auction['auc'], $auction['petSpeciesId'], $auction['petBreedId'], $auction['petLevel'], $auction['petQualityId']);
                if (strlen($sqlPet) + 5 + strlen($thisSql) > $maxPacketSize) {
                    $delayedAuctionSql[] = $sqlPet;
                    // delayed since tblAuction row must be inserted first for foreign key
                    $sqlPet = '';
                }
                $sqlPet .= ($sqlPet == '' ? $sqlStartPet : ',') . $thisSql;
            } else {
                if (isset($equipBaseItemLevel[$auction['item']])) {
                    if (count($bonuses) || $auction['rand'] || $auction['context']) {
                        for ($y = count($bonuses); $y < MAX_BONUSES; $y++) {
                            $bonuses[] = 'null';
                        }
                        $bonuses = implode(',', $bonuses);
                        $thisSql = sprintf('(%u,%u,%d,%d,%u,%s,%s,%u,%s)', $house, $auction['auc'], $auction['rand'], $auction['seed'], $auction['context'], isset($auction['lootedLevel']) ? $auction['lootedLevel'] : 'null', isset($bonusItemLevel) ? $bonusItemLevel : 'null', $bonusSet, $bonuses);
                        if (strlen($sqlExtra) + 5 + strlen($thisSql) > $maxPacketSize) {
                            $delayedAuctionSql[] = $sqlExtra;
                            // delayed since tblAuction row must be inserted first for foreign key
                            $sqlExtra = '';
                        }
                        $sqlExtra .= ($sqlExtra == '' ? $sqlStartExtra : ',') . $thisSql;
                    }
                }
            }
        }
        if ($sqlBonusesSeen != '') {
            if (GetDBLock($ourDb, DB_LOCK_SEEN_BONUSES)) {
                DebugMessage("House " . str_pad($house, 5, ' ', STR_PAD_LEFT) . " updating seen bonuses");
                DBQueryWithError($ourDb, $sqlBonusesSeen . $sqlEndBonusesSeen);
                ReleaseDBLock($ourDb, DB_LOCK_SEEN_BONUSES);
            } else {
                DebugMessage("Could not obtain " . DB_LOCK_SEEN_BONUSES . " DB lock, skipping update of seen bonuses.", E_USER_WARNING);
            }
        }
        if ($sqlLevelsSeen != '') {
            if (GetDBLock($ourDb, DB_LOCK_SEEN_ILVLS)) {
                DebugMessage("House " . str_pad($house, 5, ' ', STR_PAD_LEFT) . " updating seen levels");
                DBQueryWithError($ourDb, $sqlLevelsSeen);
                ReleaseDBLock($ourDb, DB_LOCK_SEEN_ILVLS);
            } else {
                DebugMessage("Could not obtain " . DB_LOCK_SEEN_ILVLS . " DB lock, skipping update of seen levels.", E_USER_WARNING);
            }
        }
        if ($sql != '') {
            DebugMessage("House " . str_pad($house, 5, ' ', STR_PAD_LEFT) . " updating tblAuction");
            DBQueryWithError($ourDb, $sql);
        }
        if ($sqlPet != '') {
            $delayedAuctionSql[] = $sqlPet;
        }
        if ($sqlExtra != '') {
            $delayedAuctionSql[] = $sqlExtra;
        }
        if (count($delayedAuctionSql)) {
            DebugMessage("House " . str_pad($house, 5, ' ', STR_PAD_LEFT) . " updating tblAuctionExtra, tblAuctionPet");
        }
        while (count($delayedAuctionSql)) {
            DBQueryWithError($ourDb, array_pop($delayedAuctionSql));
        }
        unset($sqlBonusesSeen, $sqlLevelsSeen, $sqlPet, $sqlExtra, $delayedAuctionSql);
        $sql = <<<EOF
insert ignore into tblAuctionRare (house, id, prevseen) (
select a.house, a.id, tis.lastseen
from tblAuction a
left join tblAuctionExtra ae on ae.house=a.house and ae.id=a.id
left join tblItemSummary tis on tis.house=a.house and tis.item=a.item and tis.bonusset=ifnull(ae.bonusset,0)
where a.house = %d
and a.id > %d
and a.item not in (82800)
%s
and ifnull(tis.lastseen, '2000-01-01') < timestampadd(day,-14,'%s'))
EOF;
        $sql = sprintf($sql, $house, $lastMax, $hasRollOver ? ' and a.id < 0x20000000 ' : '', $snapshotString);
        DebugMessage("House " . str_pad($house, 5, ' ', STR_PAD_LEFT) . " updating tblAuctionRare");
        DBQueryWithError($ourDb, $sql);
    }
    foreach ($existingIds as $existingId => &$oldRow) {
        // all missing auctions
        if (!isset($existingPetIds[$existingId])) {
            // missing item auction
            if ($oldRow[EXISTING_COL_BUY] > 0 && $oldRow[EXISTING_COL_TIMELEFT] > 0 && $oldRow[EXISTING_COL_TIMELEFT] <= $expiredLength) {
                // probably expired item with buyout
                $expiredPosted = date('Y-m-d', $snapshot - GetAuctionAge($existingId, $snapshot, $snapshotList));
                if (!isset($expiredItemInfo[$expiredPosted][$oldRow[EXISTING_COL_INFOKEY]])) {
                    $expiredItemInfo[$expiredPosted][$oldRow[EXISTING_COL_INFOKEY]] = 0;
                }
                $expiredItemInfo[$expiredPosted][$oldRow[EXISTING_COL_INFOKEY]]++;
            }
        }
    }
    unset($oldRow);
    $rareDeletes = [];
    $preDeleted = count($itemInfo);
    foreach ($existingIds as $existingId => &$oldRow) {
        if (!isset($existingPetIds[$existingId]) && !isset($itemInfo[$oldRow[EXISTING_COL_INFOKEY]])) {
            list($itemId, $bonusSet) = explode(':', $oldRow[EXISTING_COL_INFOKEY]);
            $rareDeletes[$bonusSet][] = $itemId;
            $itemInfo[$oldRow[EXISTING_COL_INFOKEY]] = array('tq' => 0, 'a' => array());
        }
    }
    unset($oldRow);
    DebugMessage("House " . str_pad($house, 5, ' ', STR_PAD_LEFT) . " updating " . count($itemInfo) . " item info (including " . (count($itemInfo) - $preDeleted) . " no longer available)");
    UpdateItemInfo($house, $itemInfo, $snapshot);
    $sql = 'delete from tblUserRareReport where house = %d and bonusset = %d and item in (%s)';
    foreach ($rareDeletes as $bonusSet => $itemIds) {
        $chunked = array_chunk($itemIds, 200);
        foreach ($chunked as $chunk) {
            DBQueryWithError($ourDb, sprintf($sql, $house, $bonusSet, implode(',', $chunk)));
        }
    }
    $preDeleted = count($petInfo);
    foreach ($existingPetIds as &$oldRow) {
        if (!isset($petInfo[$oldRow['species']][$oldRow['breed']])) {
            $petInfo[$oldRow['species']][$oldRow['breed']] = array('tq' => 0, 'a' => array());
        }
    }
    unset($oldRow);
    DebugMessage("House " . str_pad($house, 5, ' ', STR_PAD_LEFT) . " updating " . count($petInfo) . " pet info (including " . (count($petInfo) - $preDeleted) . " no longer available)");
    UpdatePetInfo($house, $petInfo, $snapshot);
    DebugMessage("House " . str_pad($house, 5, ' ', STR_PAD_LEFT) . " updating seller history");
    UpdateSellerInfo($sellerInfo, $house, $snapshot);
    if (count($expiredItemInfo) > 0) {
        $sqlStart = 'INSERT INTO tblItemExpired (item, bonusset, house, `when`, created, expired) VALUES ';
        $sqlEnd = ' ON DUPLICATE KEY UPDATE created=created+values(created), expired=expired+values(expired)';
        $sql = '';
        if (isset($expiredItemInfo['n'])) {
            DebugMessage("House " . str_pad($house, 5, ' ', STR_PAD_LEFT) . " adding new auctions for " . count($expiredItemInfo['n']) . " items");
            $snapshotDay = date('Y-m-d', $snapshot);
            $expiredCount = 0;
            foreach ($expiredItemInfo['n'] as $infoKey => $createdCount) {
                $keyParts = explode(':', $infoKey);
                $sqlPart = sprintf('(%u, %u, %u, \'%s\', %u, %u)', $keyParts[0], $keyParts[1], $house, $snapshotDay, $createdCount, $expiredCount);
                if (strlen($sql) + 10 + strlen($sqlPart) + strlen($sqlEnd) > $maxPacketSize) {
                    DBQueryWithError($ourDb, $sql . $sqlEnd);
                    $sql = '';
                }
                $sql .= ($sql == '' ? $sqlStart : ',') . $sqlPart;
            }
            unset($expiredItemInfo['n']);
        }
        if ($sql != '') {
            DBQueryWithError($ourDb, $sql . $sqlEnd);
            $sql = '';
        }
        $createdCount = 0;
        $snapshotDays = array_keys($expiredItemInfo);
        foreach ($snapshotDays as $snapshotDay) {
            //DebugMessage("House " . str_pad($house, 5, ' ', STR_PAD_LEFT) . " adding expired auctions from $snapshotDay for ".count($expiredItemInfo[$snapshotDay])." items");
            foreach ($expiredItemInfo[$snapshotDay] as $infoKey => $expiredCount) {
                $keyParts = explode(':', $infoKey);
                $sqlPart = sprintf('(%u, %u, %u, \'%s\', %u, %u)', $keyParts[0], $keyParts[1], $house, $snapshotDay, $createdCount, $expiredCount);
                if (strlen($sql) + 10 + strlen($sqlPart) + strlen($sqlEnd) > $maxPacketSize) {
                    DBQueryWithError($ourDb, $sql . $sqlEnd);
                    $sql = '';
                }
                $sql .= ($sql == '' ? $sqlStart : ',') . $sqlPart;
            }
        }
        if ($sql != '') {
            DBQueryWithError($ourDb, $sql . $sqlEnd);
        }
    }
    if (count($existingIds) > 0) {
        DebugMessage("House " . str_pad($house, 5, ' ', STR_PAD_LEFT) . " deleting " . count($existingIds) . " auctions");
        $sqlStart = sprintf('DELETE FROM tblAuction WHERE house = %d AND id IN (', $house);
        $sql = '';
        foreach ($existingIds as $lostId => &$lostRow) {
            if (strlen($sql) + 10 + strlen($lostId) > $maxPacketSize) {
                DBQueryWithError($ourDb, $sql . ')');
                $sql = '';
            }
            $sql .= ($sql == '' ? $sqlStart : ',') . $lostId;
        }
        unset($lostRow);
        if ($sql != '') {
            DBQueryWithError($ourDb, $sql . ')');
        }
    }
    $snapshotHourStart = date('Y-m-d H', $snapshot) . ':00:00';
    $stmt = $ourDb->prepare('UPDATE tblSnapshot SET flags = flags | 1 WHERE house = ? AND updated between ? and timestampadd(second, 3599, ?) AND updated != ?');
    $stmt->bind_param('isss', $house, $snapshotHourStart, $snapshotHourStart, $snapshotString);
    $stmt->execute();
    $stmt->close();
    $ourDb->close();
    MCSetHouse($house, 'ts', $snapshot);
    DebugMessage("House " . str_pad($house, 5, ' ', STR_PAD_LEFT) . " finished with {$totalAuctions} auctions in " . round(microtime(true) - $startTimer, 2) . " sec");
}
Пример #2
0
function FetchSnapshot()
{
    global $db, $region;
    $lockName = "fetchsnapshot_{$region}";
    $stmt = $db->prepare('select get_lock(?, 30)');
    $stmt->bind_param('s', $lockName);
    $stmt->execute();
    $lockSuccess = null;
    $stmt->bind_result($lockSuccess);
    if (!$stmt->fetch()) {
        $lockSuccess = null;
    }
    $stmt->close();
    if ($lockSuccess != '1') {
        DebugMessage("Could not get mysql lock for {$lockName}.");
        return 30;
    }
    $earlyCheckSeconds = EARLY_CHECK_SECONDS;
    $nextRealmSql = <<<ENDSQL
    select r.house, min(r.canonical), count(*) c, ifnull(hc.nextcheck, s.nextcheck) upd, s.lastupdate, s.mindelta, hc.lastchecksuccessresult
    from tblRealm r
    left join (
        select deltas.house, timestampadd(second, least(ifnull(min(delta)-{$earlyCheckSeconds}, 45*60), 150*60), max(deltas.updated)) nextcheck, max(deltas.updated) lastupdate, min(delta) mindelta
        from (
            select sn.updated,
            if(@prevhouse = sn.house and sn.updated > timestampadd(hour, -72, now()), unix_timestamp(sn.updated) - @prevdate, null) delta,
            @prevdate := unix_timestamp(sn.updated) updated_ts,
            @prevhouse := sn.house house
            from (select @prevhouse := null, @prevdate := null) setup, tblSnapshot sn
            order by sn.house, sn.updated) deltas
        group by deltas.house
        ) s on s.house = r.house
    left join tblHouseCheck hc on hc.house = r.house
    where r.region = ?
    and r.house is not null
    and r.canonical is not null
    group by r.house
    order by ifnull(upd, '2000-01-01') asc, c desc, r.house asc
    limit 1
ENDSQL;
    $house = $slug = $realmCount = $nextDate = $lastDate = $minDelta = $lastSuccessJson = null;
    $stmt = $db->prepare($nextRealmSql);
    $stmt->bind_param('s', $region);
    $stmt->execute();
    $stmt->bind_result($house, $slug, $realmCount, $nextDate, $lastDate, $minDelta, $lastSuccessJson);
    $gotRealm = $stmt->fetch() === true;
    $stmt->close();
    if (!$gotRealm) {
        DebugMessage("No {$region} realms to fetch!");
        ReleaseDBLock($lockName);
        return 30;
    }
    if (strtotime($nextDate) > time() && strtotime($nextDate) < time() + 3.5 * 60 * 60) {
        $delay = strtotime($nextDate) - time();
        DebugMessage("No {$region} realms ready yet, waiting " . SecondsOrMinutes($delay) . ".");
        ReleaseDBLock($lockName);
        return $delay;
    }
    SetHouseNextCheck($house, time() + 600, null);
    ReleaseDBLock($lockName);
    DebugMessage("{$region} {$slug} fetch for house {$house} to update {$realmCount} realms, due since " . (is_null($nextDate) ? 'unknown' : SecondsOrMinutes(time() - strtotime($nextDate)) . ' ago'));
    $url = GetBattleNetURL($region, "wow/auction/data/{$slug}");
    $outHeaders = [];
    $dta = [];
    $json = \Newsstand\HTTP::Get($url, [], $outHeaders);
    if ($json === false && isset($outHeaders['body'])) {
        // happens if server returns non-200 code, but we'll want that json anyway
        $json = $outHeaders['body'];
    }
    if ($json !== false) {
        $dta = json_decode($json, true);
        if (json_last_error() != JSON_ERROR_NONE) {
            $dta = [];
        }
    }
    if (!isset($dta['files']) && !is_null($lastSuccessJson)) {
        // no files in current status json, probably "internal server error"
        // check the headers on our last known good data url
        $lastGoodDta = json_decode($lastSuccessJson, true);
        if (json_last_error() != JSON_ERROR_NONE) {
            DebugMessage("{$region} {$slug} invalid JSON for last successful json\n" . $lastSuccessJson, E_USER_WARNING);
        } elseif (!isset($lastGoodDta['files'])) {
            DebugMessage("{$region} {$slug} no files in the last success json?!", E_USER_WARNING);
        } else {
            usort($lastGoodDta['files'], 'AuctionFileSort');
            $fileInfo = end($lastGoodDta['files']);
            $oldModified = ceil(intval($fileInfo['lastModified'], 10) / 1000);
            DebugMessage("{$region} {$slug} returned no files. Checking headers on URL from " . date('Y-m-d H:i:s', $oldModified));
            $headers = \Newsstand\HTTP::Head(preg_replace('/^http:/', 'https:', $fileInfo['url']));
            if (isset($headers['Last-Modified'])) {
                $newModified = strtotime($headers['Last-Modified']);
                $fileInfo['lastModified'] = $newModified * 1000;
                $dta['files'] = [$fileInfo];
                if (abs($oldModified - $newModified) < 10) {
                    DebugMessage("{$region} {$slug} data file has unchanged last modified date from last successful parse.");
                } else {
                    DebugMessage("{$region} {$slug} data file modified " . date('Y-m-d H:i:s', $newModified) . ".");
                }
            } else {
                DebugMessage("{$region} {$slug} data file failed fetching last modified date via HEAD method.");
            }
        }
    }
    if (!isset($dta['files'])) {
        $delay = GetCheckDelay(strtotime($lastDate));
        DebugMessage("{$region} {$slug} returned no files. Waiting " . SecondsOrMinutes($delay) . ".", E_USER_WARNING);
        SetHouseNextCheck($house, time() + $delay, $json);
        \Newsstand\HTTP::AbandonConnections();
        return 0;
    }
    usort($dta['files'], 'AuctionFileSort');
    $fileInfo = end($dta['files']);
    $modified = ceil(intval($fileInfo['lastModified'], 10) / 1000);
    $lastDateUnix = is_null($lastDate) ? $modified - 1 : strtotime($lastDate);
    $delay = 0;
    if (!is_null($minDelta) && $modified <= $lastDateUnix) {
        if ($lastDateUnix + $minDelta > time()) {
            // we checked for an earlier-than-expected snapshot, didn't see one
            $delay = $lastDateUnix + $minDelta - time() + 8;
            // next check will be 8 seconds after expected update
        } else {
            if ($lastDateUnix + $minDelta + 45 > time()) {
                // this is the first check after we expected a new snapshot, but didn't see one.
                // don't trust api, assume data file URL won't change, and check last-modified time on data file
                $headers = \Newsstand\HTTP::Head(preg_replace('/^http:/', 'https:', $fileInfo['url']));
                if (isset($headers['Last-Modified'])) {
                    $newModified = strtotime($headers['Last-Modified']);
                    if ($newModified > $modified) {
                        DebugMessage("{$region} {$slug} data file indicates last modified {$newModified} " . date('H:i:s', $newModified) . ", ignoring API result.");
                        $modified = $newModified;
                    } else {
                        if ($newModified == $modified) {
                            DebugMessage("{$region} {$slug} data file has last modified date matching API result.");
                        } else {
                            DebugMessage("{$region} {$slug} data file has last modified date earlier than API result: {$newModified} " . date('H:i:s', $newModified) . ".");
                        }
                    }
                } else {
                    DebugMessage("{$region} {$slug} data file failed fetching last modified date via HEAD method.");
                }
            }
        }
    }
    if ($modified <= $lastDateUnix) {
        if ($delay <= 0) {
            $delay = GetCheckDelay($modified);
        }
        DebugMessage("{$region} {$slug} still not updated since {$modified} " . date('H:i:s', $modified) . " (" . SecondsOrMinutes(time() - $modified) . " ago). Waiting " . SecondsOrMinutes($delay) . ".");
        SetHouseNextCheck($house, time() + $delay, $json);
        return 0;
    }
    DebugMessage("{$region} {$slug} updated {$modified} " . date('H:i:s', $modified) . " (" . SecondsOrMinutes(time() - $modified) . " ago), fetching auction data file");
    $dlStart = microtime(true);
    $data = \Newsstand\HTTP::Get(preg_replace('/^http:/', 'https:', $fileInfo['url']), [], $outHeaders);
    $dlDuration = microtime(true) - $dlStart;
    if (!$data || substr($data, -4) != "]\r\n}") {
        if (!$data) {
            DebugMessage("{$region} {$slug} data file empty. Waiting 5 seconds and trying again.");
            sleep(5);
        } else {
            DebugMessage("{$region} {$slug} data file malformed. Waiting 10 seconds and trying again.");
            sleep(10);
        }
        $dlStart = microtime(true);
        $data = \Newsstand\HTTP::Get($fileInfo['url'] . (parse_url($fileInfo['url'], PHP_URL_QUERY) ? '&' : '?') . 'please', [], $outHeaders);
        $dlDuration = microtime(true) - $dlStart;
    }
    if (!$data) {
        DebugMessage("{$region} {$slug} data file empty. Will try again in 30 seconds.");
        SetHouseNextCheck($house, time() + 30, $json);
        \Newsstand\HTTP::AbandonConnections();
        return 10;
    }
    if (substr($data, -4) != "]\r\n}") {
        $delay = GetCheckDelay($modified);
        DebugMessage("{$region} {$slug} data file still probably malformed. Waiting " . SecondsOrMinutes($delay) . ".");
        SetHouseNextCheck($house, time() + $delay, $json);
        return 0;
    }
    $xferBytes = isset($outHeaders['X-Original-Content-Length']) ? $outHeaders['X-Original-Content-Length'] : strlen($data);
    DebugMessage("{$region} {$slug} data file " . strlen($data) . " bytes" . ($xferBytes != strlen($data) ? ' (transfer length ' . $xferBytes . ', ' . round($xferBytes / strlen($data) * 100, 1) . '%)' : '') . ", " . round($dlDuration, 2) . "sec, " . round($xferBytes / 1000 / $dlDuration) . "KBps");
    if ($xferBytes >= strlen($data) && strlen($data) > 65536) {
        DebugMessage('No compression? ' . print_r($outHeaders, true));
    }
    if ($xferBytes / 1000 / $dlDuration < 200 && in_array($region, ['US', 'EU'])) {
        DebugMessage("Speed under 200KBps, closing persistent connections");
        \Newsstand\HTTP::AbandonConnections();
    }
    $successJson = json_encode($dta);
    // will include any updates from using lastSuccessJson
    $stmt = $db->prepare('INSERT INTO tblHouseCheck (house, nextcheck, lastcheck, lastcheckresult, lastchecksuccess, lastchecksuccessresult) VALUES (?, NULL, now(), ?, now(), ?) ON DUPLICATE KEY UPDATE nextcheck=values(nextcheck), lastcheck=values(lastcheck), lastcheckresult=values(lastcheckresult), lastchecksuccess=values(lastchecksuccess), lastchecksuccessresult=values(lastchecksuccessresult)');
    $stmt->bind_param('iss', $house, $json, $successJson);
    $stmt->execute();
    $stmt->close();
    $stmt = $db->prepare('INSERT INTO tblSnapshot (house, updated) VALUES (?, from_unixtime(?))');
    $stmt->bind_param('ii', $house, $modified);
    $stmt->execute();
    $stmt->close();
    MCSet('housecheck_' . $house, time(), 0);
    $fileName = "{$modified}-" . str_pad($house, 5, '0', STR_PAD_LEFT) . ".json";
    file_put_contents(SNAPSHOT_PATH . $fileName, $data, LOCK_EX);
    link(SNAPSHOT_PATH . $fileName, SNAPSHOT_PATH . 'parse/' . $fileName);
    if (in_array($region, ['US', 'EU'])) {
        link(SNAPSHOT_PATH . $fileName, SNAPSHOT_PATH . 'watch/' . $fileName);
    }
    unlink(SNAPSHOT_PATH . $fileName);
    return 0;
}