function doFetch($type, $id) { global $redis, $debug; $key = "zkb:zkbFeed:{$type}:{$id}"; $fetched = $redis->get($key); if ($fetched == true) { return; } if ($debug) { Util::out("Fetching {$type} {$id}"); } $lastFetchedID = (int) $redis->get("zkb:lastFetchedID:{$type}:{$id}"); do { $url = "https://zkillboard.com/api/{$type}ID/{$id}/orderDirection/asc/afterKillID/{$lastFetchedID}/no-items/no-attackers/"; $raw = Util::getData($url); $json = json_decode($raw, true); foreach ($json as $kill) { $killID = (int) $kill['killID']; $hash = $kill['zkb']['hash']; $lastFetchedID = max($lastFetchedID, $killID); saveKill($killID, $hash); } $newKills = sizeof($json); } while ($newKills > 0); $redis->set("zkb:lastFetchedID:{$type}:{$id}", $lastFetchedID); $redis->setex($key, 3601, true); }
function getCSV($url, $method) { global $mdb, $redis; $file = $redis->get("RC:{$url}"); if ($file == null) { $file = file_get_contents($url); $redis->setex("RC:{$url}", 900, $file); } $csv = bzdecompress($file); Util::out("Parsing {$url}"); parseCSV($csv, $method); }
function calcStats($row) { global $mdb, $debug; $type = $row['type']; $id = $row['id']; $newSequence = $row['sequence']; $key = ['type' => $type, 'id' => $id]; $stats = $mdb->findDoc('statistics', $key); if ($stats === null) { $stats = []; $stats['type'] = $type; $stats['id'] = $id; } $oldSequence = (int) @$stats['sequence']; if ($newSequence <= $oldSequence) { return; } for ($i = 0; $i <= 1; ++$i) { $isVictim = $i == 0; if (($type == 'locationID' || $type == 'regionID' || $type == 'solarSystemID') && $isVictim == true) { continue; } // build the query $query = [$row['type'] => $row['id'], 'isVictim' => $isVictim]; $query = MongoFilter::buildQuery($query); // set the proper sequence values $query = ['$and' => [['sequence' => ['$gt' => $oldSequence]], ['sequence' => ['$lte' => $newSequence]], $query]]; $allTime = $mdb->group('killmails', [], $query, 'killID', ['zkb.points', 'zkb.totalValue']); mergeAllTime($stats, $allTime, $isVictim); $groups = $mdb->group('killmails', 'vGroupID', $query, 'killID', ['zkb.points', 'zkb.totalValue'], ['vGroupID' => 1]); mergeGroups($stats, $groups, $isVictim); $months = $mdb->group('killmails', ['year' => 'dttm', 'month' => 'dttm'], $query, 'killID', ['zkb.points', 'zkb.totalValue'], ['year' => 1, 'month' => 1]); mergeMonths($stats, $months, $isVictim); } // Update the sequence $stats['sequence'] = $newSequence; // save it $mdb->getCollection('statistics')->save($stats); $r = $mdb->getDb()->command(['getLastError' => 1]); if ($r['ok'] != 1) { die('stats update failure'); } if ($debug) { Util::out("Stats completed for: {$type} {$id} {$newSequence}"); } }
function createInvolved($data) { global $mdb; $dataArray = array('character', 'corporation', 'alliance', 'faction', 'shipType'); $array = array(); foreach ($dataArray as $index) { if (isset($data[$index]['id']) && $data[$index]['id'] != 0) { $array["{$index}ID"] = (int) $data[$index]['id']; } } if (isset($array['shipTypeID']) && Info::getGroupID($array['shipTypeID']) == -1) { $mdb->getCollection('information')->update(['type' => 'group'], ['$set' => ['lastCrestUpdate' => new MongoDate(1)]]); Util::out('Bailing on processing a kill, unable to find groupID for ' . $array['shipTypeID']); exit; } if (isset($array['shipTypeID'])) { $array['groupID'] = (int) Info::getGroupID($array['shipTypeID']); } if (isset($data['finalBlow']) && $data['finalBlow'] == true) { $array['finalBlow'] = true; } return $array; }
$name = $group['name']; $exists = $mdb->count('information', ['type' => 'groupID', 'id' => $groupID]); if ($exists == 0) { ++$newGroups; } $mdb->insertUpdate('information', ['type' => 'groupID', 'id' => $groupID], ['name' => $name, 'lastCrestUpdate' => $mdb->now(-86400)]); $types = CrestTools::getJSON($href); if (@$types['types'] != null) { foreach ($types['types'] as $type) { $typeID = (int) getTypeID($type['href']); $name = $type['name']; $exists = $mdb->count('information', ['type' => 'typeID', 'id' => $typeID]); if ($exists > 0) { continue; } Util::out("Discovered item: {$name}"); ++$newItems; $mdb->insertUpdate('information', ['type' => 'typeID', 'id' => $typeID], ['name' => $name, 'groupID' => $groupID, 'lastCrestUpdate' => new MongoDate(1)]); } } } $next = @$groups['next']['href']; if ($next != null) { $groups = CrestTools::getJSON($next); } } while ($next != null); $mdb->insertUpdate('storage', ['locker' => 'groupsPopulated'], ['contents' => true]); if ($newGroups > 0) { Log::irc("Added {$newGroups} new groupIDs"); } if ($newItems > 0) {
continue; } $killID = @$killmail['killID']; if ($killID != 0) { $crestmail['processed'] = true; $crestmails->save($crestmail); $queueProcess->push($killID); ++$counter; $queueShare->push($killID); } else { $crestmails->update($crestmail, array('$set' => array('processed' => null))); } } } if ($debug && $counter > 0) { Util::out('Added ' . number_format($counter, 0) . ' Kills.'); } function validKill(&$kill) { // Show all pod kills $victimShipID = $kill['victim']['shipType']['id']; if ($victimShipID == 670 || $victimShipID == 33328) { return true; } $npcOnly = true; foreach ($kill['attackers'] as $attacker) { if (isset($attacker['shipType']['id'])) { $attackerGroupID = Info::getGroupID($attacker['shipType']['id']); if ($attackerGroupID == 365) { return true; }
<?php require_once "../init.php"; $raw = file_get_contents("{$baseDir}/setup/items.json"); $json = json_decode($raw, true); foreach ($json as $row) { $doc = $mdb->find("information", ['type' => 'typeID', 'id' => (int) $row['typeID']]); if ($doc === null || is_array($doc) && sizeof($doc) == 0) { Util::out("Adding " . $row['name']); $row['id'] = $row['typeID']; unset($row['typeID']); $row['type'] = 'typeID'; $mdb->save("information", $row); } $row = null; } $redis->set("tq:itemsPopulated", true);
<?php require_once '../init.php'; if (date('Hi') != '0001') { exit; } $types = ['allianceID', 'corporationID', 'factionID', 'shipTypeID', 'groupID', 'solarSystemID', 'regionID']; foreach ($types as $type) { Util::out($type); $entities = $mdb->find('statistics', ['type' => $type]); foreach ($entities as $row) { calcTop($row); } } function calcTop($row) { global $mdb; if (date('d') != '01' && isset($row['topAllTime'])) { return; } $parameters = [$row['type'] => $row['id']]; $parameters['limit'] = 10; $parameters['kills'] = true; $topLists[] = array('type' => 'character', 'data' => Stats::getTop('characterID', $parameters)); $topLists[] = array('type' => 'corporation', 'data' => Stats::getTop('corporationID', $parameters, true)); $topLists[] = array('type' => 'alliance', 'data' => Stats::getTop('allianceID', $parameters, true)); $topLists[] = array('type' => 'faction', 'data' => Stats::getTop('factionID', $parameters, true)); $topLists[] = array('type' => 'ship', 'data' => Stats::getTop('shipTypeID', $parameters, true)); $topLists[] = array('type' => 'system', 'data' => Stats::getTop('solarSystemID', $parameters, true)); do { $r = $mdb->set('statistics', $row, ['topAllTime' => $topLists]);
// Don't fetch killmail api for wars with no kill count change if ($prevKills != $currKills) { $kmHref = $war['killmails']; $page = floor($mdb->count('warmails', ['warID' => $id]) / 2000); if ($page == 0) { $page = 1; } elseif ($page > 1) { $kmHref .= "?page={$page}"; } while ($kmHref != null) { $killmails = CrestTools::getJSON($kmHref); foreach ($killmails['items'] as $kill) { $href = $kill['href']; $exploded = explode('/', $href); $killID = (int) $exploded[4]; $hash = $exploded[5]; $mdb->insertUpdate('warmails', ['warID' => $id, 'killID' => $killID]); if (!$mdb->exists('crestmails', ['killID' => $killID, 'hash' => $hash])) { $mdb->insert('crestmails', ['killID' => (int) $killID, 'hash' => $hash], ['processed' => false]); Util::out("New WARmail {$killID}"); } } $next = @$killmails['next']['href']; if ($next != $kmHref) { $kmHref = $next; } else { $kmHref = null; } } } }
$stomp = new Stomp($stompServer, $stompUser, $stompPassword); } catch (Exception $ex) { Util::out("Stomp error: " . $ex->getMessage()); exit; } $stomp->setReadTimeout(1); foreach ($topics as $topic) { $stomp->subscribe($topic, array('id' => 'zkb-' . $baseAddr, 'persistent' => 'true', 'ack' => 'client', 'prefetch-count' => 1)); } $stompCount = 0; $timer = new Timer(); while ($timer->stop() <= 59000) { $frame = $stomp->readFrame(); if (!empty($frame)) { $killdata = json_decode($frame->body, true); $killID = (int) $killdata['killID']; if ($killID == 0) { continue; } $hash = $hash = Killmail::getCrestHash($killID, $killdata); $killdata['killID'] = (int) $killID; if (!$mdb->exists('crestmails', ['killID' => $killID, 'hash' => $hash])) { ++$stompCount; $i = $mdb->getCollection('crestmails')->insert(['killID' => $killID, 'hash' => $hash, 'processed' => false, 'source' => 'stomp', 'added' => $mdb->now()]); } $stomp->ack($frame->headers['message-id']); } } if ($stompCount > 0) { Util::out("New kills from STOMP: {$stompCount}"); }
<?php require_once "../init.php"; if ($redis->get("tq:itemsPopulated") != true) { Util::out("Waiting for items to be populated..."); exit; } if (date('i') != 45) { exit; } $assign = ['capacity', 'name', 'portionSize', 'mass', 'volume', 'description', 'radius', 'published']; $rows = $mdb->find("information", ['type' => 'typeID']); foreach ($rows as $row) { $typeID = (int) $row['id']; $lastCrestUpdate = @$row['lastCrestUpdate']; if ($lastCrestUpdate != null && $lastCrestUpdate->sec > time() - 86400) { continue; } $crest = CrestTools::getJSON("https://public-crest.eveonline.com/types/{$typeID}/"); foreach ($assign as $key) { if (isset($crest[$key])) { $row[$key] = $crest[$key]; } } $row['lastCrestUpdate'] = $mdb->now(); $mdb->save("information", $row); }
if ($rank !== false) { $rank = 1 + $rank; $mdb->set("statistics", $row, ['recentOverallRank' => $rank]); $redis->hSet("tq:ranks:{$type}:recent:{$today}", $rank, $id); } } $keys = $redis->keys("tq:ranks:recent:*"); foreach ($keys as $key) { $redis->del($key); } $keys = $redis->keys("tq:ranks:*:recent:*"); foreach ($keys as $key) { $redis->expire($key, 14 * 86400); } $redis->setex($todaysKey, 87000, true); Util::out("Recent rankings complete"); function getValue(&$array, $index) { $index = ucwords($index); $value = @$array["recent{$index}"]; return $value; } function getRecent($type, $id, $isVictim, $ninetyDayKillID) { global $mdb; // build the query $query = [$type => $id, 'isVictim' => $isVictim]; $query = MongoFilter::buildQuery($query); // set the proper sequence values $query = ['$and' => [['killID' => ['$gte' => $ninetyDayKillID]], $query]]; $result = $mdb->group('killmails', [], $query, 'killID', ['zkb.points', 'zkb.totalValue']);
moveAndExpire($multi, $today, "tq:ranks:weekly:{$type}:{$today}:shipsDestroyed"); moveAndExpire($multi, $today, "tq:ranks:weekly:{$type}:{$today}:shipsLost"); moveAndExpire($multi, $today, "tq:ranks:weekly:{$type}:{$today}:iskDestroyed"); moveAndExpire($multi, $today, "tq:ranks:weekly:{$type}:{$today}:iskLost"); moveAndExpire($multi, $today, "tq:ranks:weekly:{$type}:{$today}:pointsDestroyed"); moveAndExpire($multi, $today, "tq:ranks:weekly:{$type}:{$today}:pointsLost"); $multi->exec(); } function moveAndExpire(&$multi, $today, $key) { $newKey = str_replace(":{$today}", "", $key); $multi->rename($key, $newKey); $multi->expire($newKey, 3600); } $redis->setex($todaysKey, 9000, true); Util::out("Weekly rankings complete"); function zAdd(&$multi, $key, $value, $id) { $value = max(1, (int) $value); $multi->zAdd($key, $value, $id); $multi->expire($key, 9000); } function rankCheck($max, $rank) { return $rank === false ? $max : $rank + 1; } function getWeekly($type, $id, $isVictim, $ninetyDayKillID) { global $mdb; // build the query $query = [$type => $id, 'isVictim' => $isVictim];
} catch (MongoDuplicateKeyException $ex) { // ignore it *sigh* } } if (!$exists && $debug) { Util::out("Added {$killID} from API"); } } // helpful info for output if needed $info = $mdb->findDoc('information', ['type' => 'characterID', 'id' => $charID], [], ['name' => 1, 'corporationID' => 1]); $corpInfo = $mdb->findDoc('information', ['type' => 'corporationID', 'id' => @$info['corporationID']], [], ['name' => 1]); $apiVerifiedSet = new RedisTtlSortedSet('ttlss:apiVerified', 86400); $apiVerifiedSet->add(time(), $type == 'Corporation' ? @$info['corporationID'] : $charID); if ($newMaxKillID == 0) { $tqApiChars->setTime($row, time() + rand(72000, 86400)); } // If we got new kills tell the log about it if ($killsAdded > 0) { if ($type == 'Corporation') { $name = 'corp ' . @$corpInfo['name']; } else { $name = 'char ' . @$info['name']; } while (strlen("{$killsAdded}") < 3) { $killsAdded = ' ' . $killsAdded; } Util::out("{$killsAdded} kills added by {$name}"); } } sleep(1); }
function updateEntity($killID, $entity) { global $information, $mdb, $debug; $types = ['character', 'corporation', 'alliance', 'faction']; for ($index = 0; $index < 4; ++$index) { $type = $types[$index]; if (!isset($entity[$type]['id'])) { continue; } $id = $entity[$type]['id']; $name = $entity[$type]['name']; // Look for the current entry $query = ['type' => $type . 'ID', 'id' => $id, 'killID' => ['$gte' => $killID]]; if ($mdb->exists('information', $query)) { continue; } unset($query['killID']); $row = $mdb->findDoc('information', $query); $new = $row == null; if (!isset($row['killID'])) { $row['killID'] = 0; } if ($row != null && $killID <= $row['killID']) { echo "continuing..\n"; } if ($row != null && $killID <= $row['killID']) { continue; } $updates = []; $updates['name'] = $name; $updates['killID'] = $killID; for ($subIndex = $index + 1; $subIndex < 4; ++$subIndex) { $subType = $types[$subIndex]; $updates["{$subType}ID"] = (int) @$entity[$subType]['id']; } $mdb->insertUpdate('information', $query, $updates); if ($new && $debug) { Util::out("Added {$type}: {$name}"); } } }
} foreach ($types as $type => $value) { $multi = $redis->multi(); $multi->zUnion("tq:ranks:alltime:{$type}", ["tq:ranks:alltime:{$type}:{$today}"]); $multi->expire("tq:ranks:alltime:{$type}", 100000); $multi->expire("tq:ranks:alltime:{$type}:{$today}", 7 * 86400); moveAndExpire($multi, $today, "tq:ranks:alltime:{$type}:{$today}:shipsDestroyed"); moveAndExpire($multi, $today, "tq:ranks:alltime:{$type}:{$today}:shipsLost"); moveAndExpire($multi, $today, "tq:ranks:alltime:{$type}:{$today}:iskDestroyed"); moveAndExpire($multi, $today, "tq:ranks:alltime:{$type}:{$today}:iskLost"); moveAndExpire($multi, $today, "tq:ranks:alltime:{$type}:{$today}:pointsDestroyed"); moveAndExpire($multi, $today, "tq:ranks:alltime:{$type}:{$today}:pointsLost"); $multi->exec(); } $redis->setex($todaysKey, 87000, true); Util::out("Alltime rankings complete"); function zAdd(&$multi, $key, $value, $id) { $value = max(1, (int) $value); $multi->zAdd($key, $value, $id); $multi->expire($key, 100000); } function moveAndExpire(&$multi, $today, $key) { $newKey = str_replace(":{$today}", "", $key); $multi->rename($key, $newKey); $multi->expire($newKey, 100000); } function rankCheck($max, $rank) { return $rank === false ? $max : $rank + 1;
$mdb->getCollection('statistics')->update($key, ['$set' => $stats]); } function mergeAllTime(&$stats, $result, $isVictim) { if (sizeof($result) == 0) { return; } $row = $result[0]; $dl = $isVictim ? 'Lost' : 'Destroyed'; @($stats["recentShips{$dl}"] += $row['killIDCount']); @($stats["recentPoints{$dl}"] += $row['zkb_pointsSum']); @($stats["recentIsk{$dl}"] += (int) $row['zkb_totalValueSum']); } $categories = ['Ships', 'Isk', 'Points']; foreach ($types as $type) { Util::out("Starting recent ranking for {$type}"); $size = $mdb->count('statistics', ['type' => $type]); $rankingIDs = []; foreach ($categories as $category) { for ($i = 0; $i <= 1; ++$i) { $field = $category . ($i == 0 ? 'Destroyed' : 'Lost'); $currentValue = -1; $currentRank = 0; $allIDs = $mdb->find('statistics', ['type' => $type], ["recent{$field}" => -1], null, ['months' => 0, 'groups' => 0]); $currentRank = 0; foreach ($allIDs as $row) { if (!isset($row["recent{$field}"])) { continue; } ++$currentRank; $mdb->getCollection('statistics')->update($row, ['$set' => ["recent{$field}Rank" => $currentRank]]);
$pointsDestroyedRank = getValue($row, 'pointsDestroyedRank', $size); $pointsLost = getValue($row, 'pointsLost', $size); $pointsLostRank = getValue($row, 'pointsLostRank', $size); $pointsEff = $pointsDestroyed / ($pointsDestroyed + $pointsLost); $avg = ceil(($shipsDestroyedRank + $iskDestroyedRank + $pointsDestroyedRank) / 3); $adjuster = (1 + $shipsEff + $iskEff + $pointsEff) / 4; $score = ceil($avg / $adjuster); $mdb->getCollection('statistics')->update($row, ['$set' => ['overallScore' => (int) $score]]); } Util::out("{$type} Overall rank updating"); $currentRank = 0; $result = $mdb->find('statistics', ['type' => $type], ['overallScore' => 1], null, ['months' => 0, 'groups' => 0]); foreach ($result as $row) { if (@$row['overallScore'] == null) { $mdb->getCollection('statistics')->update($row, ['$unset' => ['overallRank' => 1]]); } else { ++$currentRank; $mdb->getCollection('statistics')->update($row, ['$set' => ['overallRank' => $currentRank]]); $mdb->insertUpdate('ranksProgress', ['type' => $type, 'id' => $row['id'], 'date' => $date], ['overallRank' => $currentRank]); } } Util::out("Completed all time ranks for {$type}"); } function getValue($array, $field, $default) { $value = @$array[$field]; if ((int) $value != 0) { return $value; } return $default; }
$iskLost = getValue($row, 'iskLost'); $iskLostRank = getValue($update, 'iskLostRank'); $iskEff = $iskDestroyed / ($iskDestroyed + $iskLost); $pointsDestroyed = getValue($row, 'pointsDestroyed'); $pointsDestroyedRank = getValue($row, 'pointsDestroyedRank'); $pointsLost = getValue($row, 'pointsLost'); $pointsLostRank = getValue($row, 'pointsLostRank'); $pointsEff = $pointsDestroyed / ($pointsDestroyed + $pointsLost); $avg = ceil(($shipsDestroyedRank + $iskDestroyedRank + $pointsDestroyedRank) / 3); $adjuster = (1 + $shipsEff + $iskEff + $pointsEff) / 4; $score = ceil($avg / $adjuster); $redis->zAdd("tq:ranks:{$type}:score", $score, $id); $mdb->set("statistics", $row, $update); } $count = 0; Util::out("All time ranks - third iteration"); $iter = $information->find(); while ($row = $iter->next()) { $type = $row['type']; $id = $row['id']; $cleanup["tq:ranks:{$type}:score"] = 1; $rank = 1 + $redis->zRank("tq:ranks:{$type}:score", $id); $mdb->set("statistics", $row, ['overallRank' => $rank]); $redis->hSet("tq:ranks:{$type}:alltime:{$today}", $id, $rank); $expires[$type] = true; } foreach ($expires as $type => $value) { $redis->expire("tq:ranks:{$type}:alltime:{$today}", 86400 * 15); } foreach ($cleanup as $key => $value) { $redis->del($key);
Util::out("(apiProducer) Error Validating {$keyID}: " . $ex->getCode() . ' ' . $ex->getMessage()); } $apis->update(['keyID' => $keyID, 'vCode' => $vCode], ['$set' => ['errorCode' => $errorCode]]); sleep(3); continue; } $key = @$apiKeyInfo->key; $accessMask = @$key->accessMask; $characterIDs = array(); if ($accessMask & 256) { foreach ($apiKeyInfo->key->characters as $character) { $characterID = (int) $character->characterID; if (!isset($row['characters'][$characterID])) { $row['characters']["{$characterID}"] = 0; } $lastKillID = $row['characters']["{$characterID}"]; // Make sure we have the names and id's in the information table $mdb->insertUpdate('information', ['type' => 'corporationID', 'id' => (int) $character->corporationID], ['name' => (string) $character->corporationName]); $mdb->insertUpdate('information', ['type' => 'characterID', 'id' => (int) $characterID], ['name' => (string) $character->characterName, 'corporationID' => (int) $character->corporationID]); $type = $apiKeyInfo->key->type; if ($debug) { Util::out("Adding {$keyID} {$characterID} {$type} {$vCode}"); } $char = ['keyID' => $keyID, 'vCode' => $vCode, 'characterID' => $characterID, 'type' => $type, 'userID' => $userID]; $tqApiChars->add($char); } } } } sleep(1); }
<?php require_once '../init.php'; $i = date('i'); if ($i != 15) { exit; } $information = $mdb->getCollection('information'); $types = $mdb->getCollection('information')->distinct('type'); foreach ($types as $type) { if ($type == 'warID') { continue; } $typeRows = $information->find(['type' => $type]); Util::out("Adding {$type} to redis"); foreach ($typeRows as $row) { $id = $row['id']; $key = "tq:{$type}:{$id}"; $redis->hMSet($key, $row); $redis->expire($key, 9600); } }
<?php require_once '../init.php'; $minute = (int) date('i'); if ($minute != 0) { exit; } $mdb = new Mdb(); $killsLastHour = new RedisTtlCounter('killsLastHour', 3600); $kills = $killsLastHour->count(); $count = $mdb->findField('storage', 'contents', ['locker' => 'totalKills']); if ($kills > 0) { Log::irc('|g|' . number_format($kills, 0) . '|n| kills processed.'); Util::out(number_format($kills, 0) . ' kills added, now at ' . number_format($count, 0) . ' kills.'); }
<?php require_once '../init.php'; if (date('i') % 5 != 0) { exit; } if ($redis->llen("queueProcess") >= 25) { exit; } $count = 0; $crest = $mdb->getCollection('crestmails')->find()->sort(['added' => -1]); foreach ($crest as $row) { $killID = $row['killID']; if (isset($row['npcOnly'])) { continue; } $killmail = $mdb->findDoc("killmails", ['killID' => $killID]); $count++; if ($killmail != null) { if ($count > 10000) { exit; } continue; } $count = 0; Util::out("Resetting {$killID}"); $mdb->set('crestmails', ['killID' => $killID], ['processed' => false]); sleep(1); }