function BnetGet()
{
$parts = explode('-', $_GET['bnetget'], 2);
if (count($parts) != 2) {
echo 'Not enough parts.';
exit;
}
switch ($parts[0]) {
case 'US':
case 'EU':
break;
default:
echo 'Bad region';
exit;
}
$urlPart = 'wow/auction/data/' . $parts[1];
header('Location: ' . GetBattleNetURL($parts[0], $urlPart));
}
function FetchRegionData($region)
{
global $caughtKill;
$region = trim(strtolower($region));
$results = [];
DebugMessage("Fetching realms for {$region}");
$url = GetBattleNetURL($region, 'wow/realm/status');
$jsonString = HTTP::Get($url);
$json = json_decode($jsonString, true);
if (json_last_error() != JSON_ERROR_NONE) {
DebugMessage("Error decoding " . strlen($jsonString) . " length JSON string for {$region}: " . json_last_error_msg(), E_USER_WARNING);
return $results;
}
if (!isset($json['realms'])) {
DebugMessage("Did not find realms in realm status JSON for {$region}", E_USER_WARNING);
return $results;
}
$slugMap = [];
foreach ($json['realms'] as $realmRow) {
if ($caughtKill) {
break;
}
if (!isset($realmRow['slug'])) {
continue;
}
$slug = $realmRow['slug'];
if (isset($results[$slug])) {
$results[$slug]['name'] = $realmRow['name'];
continue;
}
$resultRow = ['name' => $realmRow['name'], 'canonical' => 1];
$results[$slug] = $resultRow;
$slugMap[$slug] = [$slug];
if (isset($realmRow['connected_realms'])) {
foreach ($realmRow['connected_realms'] as $connectedSlug) {
if ($connectedSlug == $slug) {
continue;
}
$results[$connectedSlug] = ['name' => ''];
$slugMap[$slug][] = $connectedSlug;
}
}
}
$chunks = array_chunk($slugMap, REALM_CHUNK_SIZE, true);
foreach ($chunks as $chunk) {
DebugMessage("Fetching auction data for {$region} " . implode(', ', array_keys($chunk)));
$urls = [];
foreach (array_keys($chunk) as $slug) {
$urls[$slug] = GetBattleNetURL($region, 'wow/auction/data/' . $slug);
}
$started = JSNow();
$dataUrls = [];
$jsons = FetchURLBatch($urls);
foreach ($chunk as $slug => $slugs) {
$json = [];
if (!isset($jsons[$slug])) {
DebugMessage("No HTTP response for {$region} {$slug}", E_USER_WARNING);
} else {
$json = json_decode($jsons[$slug], true);
if (json_last_error() != JSON_ERROR_NONE) {
DebugMessage("Error decoding JSON string for {$region} {$slug}: " . json_last_error_msg(), E_USER_WARNING);
$json = [];
}
}
$modified = isset($json['files'][0]['lastModified']) ? $json['files'][0]['lastModified'] : 0;
$url = isset($json['files'][0]['url']) ? $json['files'][0]['url'] : '';
if ($url) {
$dataUrls[$slug] = $url;
}
foreach ($slugs as $connectedSlug) {
$results[$connectedSlug]['checked'] = $started;
$results[$connectedSlug]['modified'] = $modified;
}
}
$dataHeads = FetchURLBatch($dataUrls, [CURLOPT_HEADER => true, CURLOPT_NOBODY => true]);
foreach ($chunk as $slug => $slugs) {
$fileDate = 0;
if (isset($dataHeads[$slug])) {
if (preg_match('/(?:^|\\n)Last-Modified: ([^\\n]+)/i', $dataHeads[$slug], $res)) {
$fileDate = strtotime($res[1]) * 1000;
} elseif ($dataHeads[$slug]) {
DebugMessage("Found no last-modified header for {$region} {$slug} at " . $dataUrls[$slug] . "\n" . $dataHeads[$slug], E_USER_WARNING);
}
} elseif (isset($dataUrls[$slug])) {
DebugMessage("Fetched no header for {$region} {$slug} at " . $dataUrls[$slug], E_USER_WARNING);
}
foreach ($slugs as $connectedSlug) {
$results[$connectedSlug]['file'] = $fileDate;
}
}
}
ksort($results);
return $results;
}
function GetRussianOwnerRealms($region)
{
global $db, $caughtKill;
$ruID = 0;
$ruSlug = '';
$ruToRun = array();
$stmt = $db->prepare('select id, slug from tblRealm where region=? and locale=\'ru_RU\' and ownerrealm is null');
$stmt->bind_param('s', $region);
$stmt->execute();
$stmt->bind_result($ruID, $ruSlug);
while ($stmt->fetch()) {
heartbeat();
if ($caughtKill) {
return;
}
DebugMessage("Getting ownerrealm for russian slug {$ruSlug}");
$url = GetBattleNetURL($region, 'wow/realm/status?realms=' . $ruSlug . '&locale=ru_RU');
$ruRealm = json_decode(FetchHTTP($url), true, 512, JSON_BIGINT_AS_STRING);
if (json_last_error() != JSON_ERROR_NONE) {
DebugMessage("{$url} did not return valid JSON");
continue;
}
if (!isset($ruRealm['realms']) || count($ruRealm['realms']) == 0) {
DebugMessage("{$url} returned no realms");
continue;
}
$ruOwner = str_replace(' ', '', $ruRealm['realms'][0]['name']);
$ruToRun[] = sprintf('update tblRealm set ownerrealm = \'%s\' where id = %d', $db->escape_string($ruOwner), $ruID);
}
$stmt->close();
if ($caughtKill) {
return;
}
foreach ($ruToRun as $sql) {
heartbeat();
if ($caughtKill) {
return;
}
if (!$db->real_query($sql)) {
DebugMessage(sprintf("%s: %s", $sql, $db->error), E_USER_WARNING);
}
}
}
function GetLocalizedOwnerRealms($region, $locale)
{
global $db, $caughtKill;
$realmId = 0;
$slug = '';
$sqlToRun = array();
$stmt = $db->prepare('SELECT id, slug FROM tblRealm WHERE region=? AND locale=? AND ownerrealm IS NULL');
$stmt->bind_param('ss', $region, $locale);
$stmt->execute();
$stmt->bind_result($realmId, $slug);
while ($stmt->fetch()) {
heartbeat();
if ($caughtKill) {
return;
}
PrintDebugNoise("Getting ownerrealm for {$locale} slug {$slug}");
$url = GetBattleNetURL($region, 'wow/realm/status?realms=' . urlencode($slug) . '&locale=' . $locale);
$realmJson = json_decode(\Newsstand\HTTP::Get($url), true, 512, JSON_BIGINT_AS_STRING);
if (json_last_error() != JSON_ERROR_NONE) {
PrintDebugNoise("{$url} did not return valid JSON");
continue;
}
if (!isset($realmJson['realms']) || count($realmJson['realms']) == 0) {
PrintDebugNoise("{$url} returned no realms");
continue;
}
if (count($realmJson['realms']) > 1) {
PrintImportantMessage("Region {$region} slug {$slug} returned " . count($realmJson['realms']) . " realms. {$url}");
}
$ownerRealm = $realmJson['realms'][0]['name'];
$sqlToRun[] = sprintf('UPDATE tblRealm SET ownerrealm = \'%s\' WHERE id = %d', $db->escape_string($ownerRealm), $realmId);
}
$stmt->close();
if ($caughtKill) {
return;
}
foreach ($sqlToRun as $sql) {
heartbeat();
if ($caughtKill) {
return;
}
if (!$db->real_query($sql)) {
PrintImportantMessage(sprintf("%s: %s", $sql, $db->error), E_USER_WARNING);
}
}
}
function GetGuild(&$characterNames, $guild, $realmName)
{
global $db, $caughtKill, $allRealms;
heartbeat();
if ($caughtKill) {
return;
}
if (!isset($allRealms[$realmName])) {
DebugMessage('Could not find realm ' . $realmName);
return;
}
$guildId = 0;
$scanned = 0;
$stmt = $db->prepare('select id, ifnull(scanned,\'2000-01-01\') from tblGuild where realm = ? and name = ?');
$stmt->bind_param('is', $allRealms[$realmName]['id'], $guild);
$stmt->execute();
$stmt->bind_result($guildId, $scanned);
$hasRow = $stmt->fetch() === true;
$stmt->close();
if ($hasRow) {
if (strtotime($scanned) >= time() - 14 * 24 * 60 * 60) {
return;
}
} else {
$stmt = $db->prepare('insert into tblGuild (realm, name) values (?, ?)');
$stmt->bind_param('is', $allRealms[$realmName]['id'], $guild);
$stmt->execute();
$stmt->close();
$guildId = $db->insert_id;
}
DebugMessage("Getting guild <{$guild}> on {$realmName}");
$url = GetBattleNetURL($allRealms[$realmName]['region'], "wow/guild/" . $allRealms[$realmName]['slug'] . "/" . rawurlencode($guild) . "?fields=members");
$json = FetchHTTP($url);
if (!$json) {
return;
}
heartbeat();
if ($caughtKill) {
return;
}
$dta = json_decode($json, true);
if (json_last_error() != JSON_ERROR_NONE) {
return;
}
if (!isset($dta['members'])) {
return;
}
$charCount = 0;
$side = $dta['side'] + 1;
foreach ($dta['members'] as $member) {
if (!isset($member['character'])) {
continue;
}
if (!isset($member['character']['name'])) {
continue;
}
if (!isset($member['character']['realm'])) {
continue;
}
if (!isset($allRealms[$member['character']['realm']])) {
continue;
}
$charCount++;
$member['character']['gender']++;
// line up with db enum
$stmt = $db->prepare('insert into tblCharacter (name, realm, scanned, race, class, gender, level) values (?, ?, NOW(), ?, ?, ?, ?) on duplicate key update lastmodified=null, scanned=values(scanned), race=values(race), class=values(class), gender=values(gender), level=values(level)');
$stmt->bind_param('siiiii', $member['character']['name'], $allRealms[$member['character']['realm']]['id'], $member['character']['race'], $member['character']['class'], $member['character']['gender'], $member['character']['level']);
$stmt->execute();
$stmt->close();
unset($characterNames[$allRealms[$member['character']['realm']]['ownerrealm']][$member['character']['name']]);
}
$stmt = $db->prepare('update tblGuild set scanned=now(), side=?, members=? where id = ?');
$stmt->bind_param('iii', $side, $charCount, $guildId);
$stmt->execute();
$stmt->close();
}
$json = json_decode(\Newsstand\HTTP::Get($url), true);
if (!isset($json['members'])) {
continue;
}
for ($y = 0; $y < count($json['members']); $y++) {
heartbeat();
if ($caughtKill) {
exit;
}
$c = $json['members'][$y]['character'];
if ($c['level'] < 20) {
continue;
}
$toon = $c['name'];
DebugMessage("Fetching {$region} {$slug} {$toon} of <{$guild}>");
$url = GetBattleNetURL($region, "wow/character/{$slug}/{$toon}?fields=appearance");
$cjson = json_decode(\Newsstand\HTTP::Get($url), true);
if (!isset($cjson['appearance'])) {
continue;
}
$imgUrl = "http://render-{$region}.worldofwarcraft.com/character/" . preg_replace('/-avatar\\.jpg$/', '-inset.jpg', $cjson['thumbnail']);
DebugMessage("Fetching {$imgUrl}");
$img = \Newsstand\HTTP::Get($imgUrl);
if ($img) {
$hits++;
$id = MakeID();
$helm = $cjson['appearance']['showHelm'] ? 1 : 0;
DebugMessage("Saving {$id} as {$c['race']} {$c['gender']} {$helm}");
file_put_contents(CAPTCHA_DIR . '/' . $id . '.jpg', $img);
$sql = 'INSERT INTO tblCaptcha (id, race, gender, helm) VALUES (?, ?, ?, ?)';
$stmt = $db->prepare($sql);
function FetchSnapshot()
{
global $db, $region;
$lockName = "fetchsnapshot_{$region}";
$stmt = $db->prepare('select get_lock(?, 30)');
$stmt->bind_param('s', $lockName);
$stmt->execute();
$lockSuccess = null;
$stmt->bind_result($lockSuccess);
if (!$stmt->fetch()) {
$lockSuccess = null;
}
$stmt->close();
if ($lockSuccess != '1') {
DebugMessage("Could not get mysql lock for {$lockName}.");
return 30;
}
$earlyCheckSeconds = EARLY_CHECK_SECONDS;
$nextRealmSql = <<<ENDSQL
select r.house, min(r.canonical), count(*) c, ifnull(hc.nextcheck, s.nextcheck) upd, s.lastupdate, s.mindelta, hc.lastchecksuccessresult
from tblRealm r
left join (
select deltas.house, timestampadd(second, least(ifnull(min(delta)-{$earlyCheckSeconds}, 45*60), 150*60), max(deltas.updated)) nextcheck, max(deltas.updated) lastupdate, min(delta) mindelta
from (
select sn.updated,
if(@prevhouse = sn.house and sn.updated > timestampadd(hour, -72, now()), unix_timestamp(sn.updated) - @prevdate, null) delta,
@prevdate := unix_timestamp(sn.updated) updated_ts,
@prevhouse := sn.house house
from (select @prevhouse := null, @prevdate := null) setup, tblSnapshot sn
order by sn.house, sn.updated) deltas
group by deltas.house
) s on s.house = r.house
left join tblHouseCheck hc on hc.house = r.house
where r.region = ?
and r.house is not null
and r.canonical is not null
group by r.house
order by ifnull(upd, '2000-01-01') asc, c desc, r.house asc
limit 1
ENDSQL;
$house = $slug = $realmCount = $nextDate = $lastDate = $minDelta = $lastSuccessJson = null;
$stmt = $db->prepare($nextRealmSql);
$stmt->bind_param('s', $region);
$stmt->execute();
$stmt->bind_result($house, $slug, $realmCount, $nextDate, $lastDate, $minDelta, $lastSuccessJson);
$gotRealm = $stmt->fetch() === true;
$stmt->close();
if (!$gotRealm) {
DebugMessage("No {$region} realms to fetch!");
ReleaseDBLock($lockName);
return 30;
}
if (strtotime($nextDate) > time() && strtotime($nextDate) < time() + 3.5 * 60 * 60) {
$delay = strtotime($nextDate) - time();
DebugMessage("No {$region} realms ready yet, waiting " . SecondsOrMinutes($delay) . ".");
ReleaseDBLock($lockName);
return $delay;
}
SetHouseNextCheck($house, time() + 600, null);
ReleaseDBLock($lockName);
DebugMessage("{$region} {$slug} fetch for house {$house} to update {$realmCount} realms, due since " . (is_null($nextDate) ? 'unknown' : SecondsOrMinutes(time() - strtotime($nextDate)) . ' ago'));
$url = GetBattleNetURL($region, "wow/auction/data/{$slug}");
$outHeaders = [];
$dta = [];
$json = \Newsstand\HTTP::Get($url, [], $outHeaders);
if ($json === false && isset($outHeaders['body'])) {
// happens if server returns non-200 code, but we'll want that json anyway
$json = $outHeaders['body'];
}
if ($json !== false) {
$dta = json_decode($json, true);
if (json_last_error() != JSON_ERROR_NONE) {
$dta = [];
}
}
if (!isset($dta['files']) && !is_null($lastSuccessJson)) {
// no files in current status json, probably "internal server error"
// check the headers on our last known good data url
$lastGoodDta = json_decode($lastSuccessJson, true);
if (json_last_error() != JSON_ERROR_NONE) {
DebugMessage("{$region} {$slug} invalid JSON for last successful json\n" . $lastSuccessJson, E_USER_WARNING);
} elseif (!isset($lastGoodDta['files'])) {
DebugMessage("{$region} {$slug} no files in the last success json?!", E_USER_WARNING);
} else {
usort($lastGoodDta['files'], 'AuctionFileSort');
$fileInfo = end($lastGoodDta['files']);
$oldModified = ceil(intval($fileInfo['lastModified'], 10) / 1000);
DebugMessage("{$region} {$slug} returned no files. Checking headers on URL from " . date('Y-m-d H:i:s', $oldModified));
$headers = \Newsstand\HTTP::Head(preg_replace('/^http:/', 'https:', $fileInfo['url']));
if (isset($headers['Last-Modified'])) {
$newModified = strtotime($headers['Last-Modified']);
$fileInfo['lastModified'] = $newModified * 1000;
$dta['files'] = [$fileInfo];
if (abs($oldModified - $newModified) < 10) {
DebugMessage("{$region} {$slug} data file has unchanged last modified date from last successful parse.");
} else {
DebugMessage("{$region} {$slug} data file modified " . date('Y-m-d H:i:s', $newModified) . ".");
}
} else {
DebugMessage("{$region} {$slug} data file failed fetching last modified date via HEAD method.");
}
}
}
if (!isset($dta['files'])) {
$delay = GetCheckDelay(strtotime($lastDate));
DebugMessage("{$region} {$slug} returned no files. Waiting " . SecondsOrMinutes($delay) . ".", E_USER_WARNING);
SetHouseNextCheck($house, time() + $delay, $json);
\Newsstand\HTTP::AbandonConnections();
return 0;
}
usort($dta['files'], 'AuctionFileSort');
$fileInfo = end($dta['files']);
$modified = ceil(intval($fileInfo['lastModified'], 10) / 1000);
$lastDateUnix = is_null($lastDate) ? $modified - 1 : strtotime($lastDate);
$delay = 0;
if (!is_null($minDelta) && $modified <= $lastDateUnix) {
if ($lastDateUnix + $minDelta > time()) {
// we checked for an earlier-than-expected snapshot, didn't see one
$delay = $lastDateUnix + $minDelta - time() + 8;
// next check will be 8 seconds after expected update
} else {
if ($lastDateUnix + $minDelta + 45 > time()) {
// this is the first check after we expected a new snapshot, but didn't see one.
// don't trust api, assume data file URL won't change, and check last-modified time on data file
$headers = \Newsstand\HTTP::Head(preg_replace('/^http:/', 'https:', $fileInfo['url']));
if (isset($headers['Last-Modified'])) {
$newModified = strtotime($headers['Last-Modified']);
if ($newModified > $modified) {
DebugMessage("{$region} {$slug} data file indicates last modified {$newModified} " . date('H:i:s', $newModified) . ", ignoring API result.");
$modified = $newModified;
} else {
if ($newModified == $modified) {
DebugMessage("{$region} {$slug} data file has last modified date matching API result.");
} else {
DebugMessage("{$region} {$slug} data file has last modified date earlier than API result: {$newModified} " . date('H:i:s', $newModified) . ".");
}
}
} else {
DebugMessage("{$region} {$slug} data file failed fetching last modified date via HEAD method.");
}
}
}
}
if ($modified <= $lastDateUnix) {
if ($delay <= 0) {
$delay = GetCheckDelay($modified);
}
DebugMessage("{$region} {$slug} still not updated since {$modified} " . date('H:i:s', $modified) . " (" . SecondsOrMinutes(time() - $modified) . " ago). Waiting " . SecondsOrMinutes($delay) . ".");
SetHouseNextCheck($house, time() + $delay, $json);
return 0;
}
DebugMessage("{$region} {$slug} updated {$modified} " . date('H:i:s', $modified) . " (" . SecondsOrMinutes(time() - $modified) . " ago), fetching auction data file");
$dlStart = microtime(true);
$data = \Newsstand\HTTP::Get(preg_replace('/^http:/', 'https:', $fileInfo['url']), [], $outHeaders);
$dlDuration = microtime(true) - $dlStart;
if (!$data || substr($data, -4) != "]\r\n}") {
if (!$data) {
DebugMessage("{$region} {$slug} data file empty. Waiting 5 seconds and trying again.");
sleep(5);
} else {
DebugMessage("{$region} {$slug} data file malformed. Waiting 10 seconds and trying again.");
sleep(10);
}
$dlStart = microtime(true);
$data = \Newsstand\HTTP::Get($fileInfo['url'] . (parse_url($fileInfo['url'], PHP_URL_QUERY) ? '&' : '?') . 'please', [], $outHeaders);
$dlDuration = microtime(true) - $dlStart;
}
if (!$data) {
DebugMessage("{$region} {$slug} data file empty. Will try again in 30 seconds.");
SetHouseNextCheck($house, time() + 30, $json);
\Newsstand\HTTP::AbandonConnections();
return 10;
}
if (substr($data, -4) != "]\r\n}") {
$delay = GetCheckDelay($modified);
DebugMessage("{$region} {$slug} data file still probably malformed. Waiting " . SecondsOrMinutes($delay) . ".");
SetHouseNextCheck($house, time() + $delay, $json);
return 0;
}
$xferBytes = isset($outHeaders['X-Original-Content-Length']) ? $outHeaders['X-Original-Content-Length'] : strlen($data);
DebugMessage("{$region} {$slug} data file " . strlen($data) . " bytes" . ($xferBytes != strlen($data) ? ' (transfer length ' . $xferBytes . ', ' . round($xferBytes / strlen($data) * 100, 1) . '%)' : '') . ", " . round($dlDuration, 2) . "sec, " . round($xferBytes / 1000 / $dlDuration) . "KBps");
if ($xferBytes >= strlen($data) && strlen($data) > 65536) {
DebugMessage('No compression? ' . print_r($outHeaders, true));
}
if ($xferBytes / 1000 / $dlDuration < 200 && in_array($region, ['US', 'EU'])) {
DebugMessage("Speed under 200KBps, closing persistent connections");
\Newsstand\HTTP::AbandonConnections();
}
$successJson = json_encode($dta);
// will include any updates from using lastSuccessJson
$stmt = $db->prepare('INSERT INTO tblHouseCheck (house, nextcheck, lastcheck, lastcheckresult, lastchecksuccess, lastchecksuccessresult) VALUES (?, NULL, now(), ?, now(), ?) ON DUPLICATE KEY UPDATE nextcheck=values(nextcheck), lastcheck=values(lastcheck), lastcheckresult=values(lastcheckresult), lastchecksuccess=values(lastchecksuccess), lastchecksuccessresult=values(lastchecksuccessresult)');
$stmt->bind_param('iss', $house, $json, $successJson);
$stmt->execute();
$stmt->close();
$stmt = $db->prepare('INSERT INTO tblSnapshot (house, updated) VALUES (?, from_unixtime(?))');
$stmt->bind_param('ii', $house, $modified);
$stmt->execute();
$stmt->close();
MCSet('housecheck_' . $house, time(), 0);
$fileName = "{$modified}-" . str_pad($house, 5, '0', STR_PAD_LEFT) . ".json";
file_put_contents(SNAPSHOT_PATH . $fileName, $data, LOCK_EX);
link(SNAPSHOT_PATH . $fileName, SNAPSHOT_PATH . 'parse/' . $fileName);
if (in_array($region, ['US', 'EU'])) {
link(SNAPSHOT_PATH . $fileName, SNAPSHOT_PATH . 'watch/' . $fileName);
}
unlink(SNAPSHOT_PATH . $fileName);
return 0;
}
function FetchPets($pets)
{
global $petMap;
$results = array();
foreach ($pets as &$id) {
heartbeat();
DebugMessage('Fetching pet ' . $id);
$url = GetBattleNetURL('us', 'wow/battlePet/species/' . $id);
$json = \Newsstand\HTTP::Get($url);
$dta = json_decode($json, true);
if (json_last_error() != JSON_ERROR_NONE || !isset($dta['speciesId'])) {
DebugMessage('Error fetching pet ' . $id . ' from battle.net..');
continue;
}
$results[$dta['speciesId']] = array('json' => $json);
foreach ($petMap as $ours => $details) {
if (!isset($dta[$details['name']])) {
if ($details['required']) {
DebugMessage('Pet ' . $dta['speciesId'] . ' did not have required column ' . $details['name'], E_USER_WARNING);
unset($results[$dta['speciesId']]);
continue 2;
}
$dta[$details['name']] = null;
}
if (is_bool($dta[$details['name']])) {
$results[$dta['speciesId']][$ours] = $dta[$details['name']] ? 1 : 0;
} else {
$results[$dta['speciesId']][$ours] = $dta[$details['name']];
}
}
}
return $results;
}
