function getFullSpot($msgId, $ourUserId, $nntp) {
$fullSpot = $this->_db->getFullSpot($msgId, $ourUserId);
if (empty($fullSpot)) {
# Vraag de volledige spot informatie op -- dit doet ook basic
# sanity en validatie checking
$fullSpot = $nntp->getFullSpot($msgId);
$this->_db->addFullSpot($fullSpot);
# we halen de fullspot opnieuw op zodat we de 'xover' informatie en de
# niet xover informatie in 1 hebben
$fullSpot = $this->_db->getFullSpot($msgId, $ourUserId);
} # if
$spotParser = new SpotParser();
$fullSpot = array_merge($spotParser->parseFull($fullSpot['fullxml']), $fullSpot);
/*
* Als je een fullspot ophaalt, maar er is nog gen 'spot' entry, dan blijf je een
* lege spot terugkrijgen omdat de join misgaat. Omdat dit verwarring op kan leveren
* gooien we dan een exception
*/
if (empty($fullSpot)) {
throw new Exception("Spot is not in our Spotweb database");
} # if
return $fullSpot;
} # getFullSpot
function getFullSpot($msgId, $ourUserId, $nntp)
{
SpotTiming::start('SpotsOverview::' . __FUNCTION__);
$fullSpot = $this->_db->getFullSpot($msgId, $ourUserId);
if (empty($fullSpot)) {
/*
* Retrieve a full loaded spot from the NNTP server
*/
$newFullSpot = $nntp->getFullSpot($msgId);
$this->_db->addFullSpots(array($newFullSpot));
/*
* If the current spotterid is empty, we probably now
* have a spotterid because we have the fullspot.
*/
if (empty($fullSpot['spotterid']) && $newFullSpot['verified']) {
$spotSigning = Services_Signing_Base::newServiceSigning();
$newFullSpot['spotterid'] = $spotSigning->calculateSpotterId($newFullSpot['user-key']['modulo']);
/*
* Update the spotterid in the spots table so it can be filtered later on
*/
$this->_db->updateSpotInfoFromFull($newFullSpot);
}
# if
/*
* We ask our DB to retrieve the fullspot again, this ensures
* us all information is present and in always the same format
*/
$fullSpot = $this->_db->getFullSpot($msgId, $ourUserId);
}
# if
/**
* We'll overwrite our spot info from the database with some information we parse from the
* XML. This is necessary because the XML contains better encoding.
*
* For example take the titel from spot bdZZdJ3gPxTAmSE%40spot.net.
*
* We cannot use all information from the XML because because some information just
* isn't present in the XML file
*/
$spotParser = new SpotParser();
$parsedXml = $spotParser->parseFull($fullSpot['fullxml']);
$fullSpot = array_merge($parsedXml, $fullSpot);
$fullSpot['title'] = $parsedXml['title'];
/*
* When we retrieve a fullspot entry but there is no spot entry the join in our DB query
* causes us to never get the spot, hence we throw this exception
*/
if (empty($fullSpot)) {
throw new Exception("Spot is not in our Spotweb database");
}
# if
SpotTiming::stop('SpotsOverview::' . __FUNCTION__, array($msgId, $ourUserId, $nntp, $fullSpot));
return $fullSpot;
}
function getFullSpot($msgId, $nntp)
{
$fullSpot = $this->_db->getFullSpot($msgId);
if (empty($fullSpot) || !isset($fullSpot['verified'])) {
# Vraag de volledige spot informatie op -- dit doet ook basic
# sanity en validatie checking
$fullSpot = $nntp->getFullSpot($msgId);
$this->_db->addFullSpot($fullSpot);
}
# if
$spotParser = new SpotParser();
$fullSpot = array_merge($spotParser->parseFull($fullSpot['xml']), $fullSpot);
return $fullSpot;
}
function getFullSpot($msgId, $nntp)
{
$fullSpot = $this->_db->getFullSpot($msgId);
if (empty($fullSpot)) {
# Vraag de volledige spot informatie op -- dit doet ook basic
# sanity en validatie checking
$fullSpot = $nntp->getFullSpot($msgId);
$this->_db->addFullSpot($fullSpot);
# we halen de fullspot opnieuw op zodat we de 'xover' informatie en de
# niet xover informatie in 1 hebben
$fullSpot = $this->_db->getFullSpot($msgId);
}
# if
$spotParser = new SpotParser();
$fullSpot = array_merge($spotParser->parseFull($fullSpot['fullxml']), $fullSpot);
return $fullSpot;
}
function getFullSpot($msgId, $ourUserId, $nntp)
{
$fullSpot = $this->_db->getFullSpot($msgId, $ourUserId);
if (empty($fullSpot)) {
# Vraag de volledige spot informatie op -- dit doet ook basic
# sanity en validatie checking
$fullSpot = $nntp->getFullSpot($msgId);
$this->_db->addFullSpots(array($fullSpot));
# we halen de fullspot opnieuw op zodat we de 'xover' informatie en de
# niet xover informatie in 1 hebben
$fullSpot = $this->_db->getFullSpot($msgId, $ourUserId);
}
# if
/**
* Overschrijf nu onze 'spot info' uit de database met sommige info welke we uit de
* de XML parseren, we doen dit omdat de XML o.a. betere encoding bevat, zie de titel van spot
* bdZZdJ3gPxTAmSE%40spot.net bijvoorbeeld.
*
* Alles uit de SpotsFull aannemen is niet interessant omdat niet elke XML versie alle
* informatie bevat
*/
$spotParser = new SpotParser();
$parsedXml = $spotParser->parseFull($fullSpot['fullxml']);
$fullSpot = array_merge($parsedXml, $fullSpot);
$fullSpot['title'] = $parsedXml['title'];
/*
* Als je een fullspot ophaalt, maar er is nog gen 'spot' entry, dan blijf je een
* lege spot terugkrijgen omdat de join misgaat. Omdat dit verwarring op kan leveren
* gooien we dan een exception
*/
if (empty($fullSpot)) {
throw new Exception("Spot is not in our Spotweb database");
}
# if
return $fullSpot;
}
function getFullSpot($msgId)
{
# initialize some variables
$spotParser = new SpotParser();
$spot = array('xml' => '', 'user-signature' => '', 'user-key' => '', 'verified' => false, 'messageid' => $msgId, 'userid' => '', 'xml-signature' => '');
# Vraag de volledige article header van de spot op
$header = $this->getHeader('<' . $msgId . '>');
# Parse de header
foreach ($header as $str) {
$keys = explode(':', $str);
switch ($keys[0]) {
case 'X-XML':
$spot['xml'] .= substr($str, 7);
break;
case 'X-User-Signature':
$spot['user-signature'] = base64_decode($spotParser->unspecialString(substr($str, 18)));
break;
case 'X-XML-Signature':
$spot['xml-signature'] = substr($str, 17);
break;
case 'X-User-Key':
$xml = simplexml_load_string(substr($str, 12));
if ($xml !== false) {
$spot['user-key']['exponent'] = (string) $xml->Exponent;
$spot['user-key']['modulo'] = (string) $xml->Modulus;
}
# if
break;
# x-user-key
}
# switch
}
# foreach
# Valideer de signature van de XML, deze is gesigned door de user zelf
if (!empty($spot['user-signature']) && !empty($spot['user-key'])) {
$spot['verified'] = $spotParser->checkRsaSignature('<' . $spot['messageid'] . '>', $spot['user-signature'], $spot['user-key']);
if (!$spot['verified']) {
$spot['verified'] = $spotParser->checkRsaSignature($spot['xml-signature'], $spot['user-signature'], $spot['user-key']);
}
# if
} else {
$spot['verified'] = false;
}
# else
# als de spot verified is, toon dan de userid van deze user
if ($spot['verified']) {
$spot['userid'] = $spotParser->calculateUserid($spot['user-key']['modulo']);
}
# if
# Parse nu de XML file, alles wat al gedefinieerd is eerder wordt niet overschreven
$spot = array_merge($spotParser->parseFull($spot['xml']), $spot);
return $spot;
}
$nzb_spotnntp->connect();
}
# else
$header = $hdr_spotnntp->getHeader('<' . $spot['messageid'] . '>');
$xml = '';
if ($header !== false) {
foreach ($header as $str) {
if (substr($str, 0, 7) == 'X-XML: ') {
$xml .= substr($str, 7);
}
# if
}
# foreach
}
# if
$spotParser = new SpotParser();
$xmlar = $spotParser->parseFull($xml);
/* Connect to the NZB group */
/* Get the NZB file */
$nzb = false;
if (is_array($xmlar['segment'])) {
foreach ($xmlar['segment'] as $seg) {
$nzb .= implode("", $nzb_spotnntp->getBody("<" . $seg . ">"));
}
# foreach
} else {
$nzb .= implode("", $nzb_spotnntp->getBody("<" . $xmlar['segment'] . ">"));
}
# if
if ($nzb !== false) {
if ($settings['nzb_download_local'] == true) {
public function postSpot($user, $spot, $imageFilename, $nzbFilename)
{
$errorList = array();
$hdr_newsgroup = $this->_settings->get('hdr_group');
$bin_newsgroup = $this->_settings->get('nzb_group');
/*
$hdr_newsgroup = 'alt.test';
$bin_newsgroup = 'alt.test';
*/
# If the hashcash doesn't match, we will never post it
if (substr(sha1('<' . $spot['newmessageid'] . '>'), 0, 4) != '0000') {
$errorList[] = array('postspot_invalidhashcash', array());
}
# if
# Read the contents of image so we can check it
$imageContents = file_get_contents($imageFilename);
# the image should be below 1MB
if (strlen($imageContents) > 1024 * 1024) {
$errorList[] = array('postspot_imagetoolarge', array());
}
# if
/*
* Get some image information, if it fails, this is an
* error as well
*/
$tmpGdImageSize = getimagesize($imageFilename);
if ($tmpGdImageSize === false) {
$errorList[] = array('postspot_imageinvalid', array());
} else {
$imageInfo = array('width' => $tmpGdImageSize[0], 'height' => $tmpGdImageSize[1]);
}
# if
# Body cannot be empty, very short or too long
$spot['body'] = trim($spot['body']);
if (strlen($spot['body']) < 30) {
$errorList[] = array('postspot_bodytooshort', array());
}
# if
if (strlen($spot['body']) > 9000) {
$errorList[] = array('postspot_bodytoolong', array());
}
# if
# Title cannot be empty or very short
$spot['title'] = trim($spot['title']);
if (strlen($spot['title']) < 5) {
$errorList[] = array('postspot_titletooshort', array());
}
# if
# Subcategory should be valid
if ($spot['category'] < 0 || $spot['category'] > count(SpotCategories::$_head_categories)) {
$errorList[] = array('postspot_invalidcategory', array($spot['category']));
}
# if
/*
* Load the NZB file as an XML file so we can make sure
* it's a valid XML and NZB file and we can determine the
* filesize
*/
$nzbFileContents = file_get_contents($nzbFilename);
$nzbXml = simplexml_load_string($nzbFileContents);
# Do some basic sanity checking for some required NZB elements
if (empty($nzbXml->file)) {
$errorList[] = array('postspot_invalidnzb', array());
}
# if
# and determine the total filesize
$spot['filesize'] = 0;
foreach ($nzbXml->file as $file) {
foreach ($file->segments->segment as $seg) {
$spot['filesize'] += (int) $seg['bytes'];
}
# foreach
}
# foreach
/*
* Make sure we didn't use this messageid recently or at all, this
* prevents people from not recalculating the hashcash in order to spam
* the system
*/
if (!$this->_db->isNewSpotMessageIdUnique($spot['newmessageid'])) {
$errorList[] = array('postspot_replayattack', array());
}
# if
# We require the keyid 7 because it is selfsigned
$spot['key'] = 7;
# Poster's username
$spot['poster'] = $user['username'];
# Fix up some overly long spot properties and other minor issues
$spot['tag'] = substr(trim($spot['tag'], " |;\r\n\t"), 0, 99);
$spot['http'] = substr(trim($spot['website']), 0, 449);
# Create one list of all subcategories
$spot['subcatlist'] = explode(',', $spot['subcatlist']);
/*
* Loop through all subcategories and check if they are valid in
* our list of subcategories
*/
$subCatSplitted = array('a' => array(), 'b' => array(), 'c' => array(), 'd' => array(), 'z' => array());
foreach ($spot['subcatlist'] as $subCat) {
$subcats = explode('_', $subCat);
$subCatLetter = substr($subcats[2], 0, 1);
$subCatSplitted[$subCatLetter][] = $subCat;
if (!isset(SpotCategories::$_categories[$spot['category']][$subCatLetter][substr($subcats[2], 1)])) {
$errorList[] = array('postspot_invalidsubcat', array($subCat . ' !! ' . $subCatLetter . ' !! ' . substr($subcats[2], 1)));
}
# if
}
# foreach
/*
* Make sure all subcategories are in the format we expect, for
* example we strip the 'cat' part and strip the z-subcat
*/
$subcatCount = count($spot['subcatlist']);
for ($i = 0; $i < $subcatCount; $i++) {
$subcats = explode('_', $spot['subcatlist'][$i]);
# If not in our format
if (count($subcats) != 3) {
$errorList[] = array('postspot_invalidsubcat', array($spot['subcatlist'][$i]));
} else {
$spot['subcatlist'][$i] = substr($subcats[2], 0, 1) . str_pad(substr($subcats[2], 1), 2, '0', STR_PAD_LEFT);
# Explicitly add the 'z'-category
$zcatStr = substr($subcats[1], 0, 1) . str_pad(substr($subcats[1], 1), 2, '0', STR_PAD_LEFT);
if (array_search($zcatStr, $spot['subcatlist']) === false) {
$spot['subcatlist'][] = $zcatStr;
}
# if
}
# else
}
# for
# Make sure the spot isn't being posted in many categories
if (count($subCatSplitted['a']) > 1) {
$errorList[] = array('postspot_canonlybeoneformat', count($spot['subcatlist']));
}
# if
# Make sure the spot has at least a format
if (count($subCatSplitted['a']) < 1) {
$errorList[] = array('postspot_musthaveformat', count($spot['subcatlist']));
}
# if
# Make sure the spot isn't being posted for too many categories
if (count($spot['subcatlist']) > 10) {
$errorList[] = array('postspot_toomanycategories', count($spot['subcatlist']));
}
# if
# Make sure the spot isn't being posted for too many categories
if (count($spot['subcatlist']) < 2) {
$errorList[] = array('postspot_toofewcategories', count($spot['subcatlist']));
}
# if
# en post daadwerkelijk de spot
if (empty($errorList)) {
/*
* Retrieve the image information and post the image to
* the appropriate newsgroup so we have the messageid list of
* images
*/
$imgSegmentList = $this->_nntp_post->postBinaryMessage($user, $bin_newsgroup, $imageContents, '');
$imageInfo['segments'] = $imgSegmentList;
# Post the NZB file to the appropriate newsgroups
$nzbSegmentList = $this->_nntp_post->postBinaryMessage($user, $bin_newsgroup, gzdeflate($nzbFileContents), '');
# Convert the current Spotnet info, to an XML structure
$spotParser = new SpotParser();
$spotXml = $spotParser->convertSpotToXml($spot, $imageInfo, $nzbSegmentList);
$spot['spotxml'] = $spotXml;
# And actually post to the newsgroups
$this->_nntp_post->postFullSpot($user, $this->_settings->get('privatekey'), $hdr_newsgroup, $spot);
$this->_db->addPostedSpot($user['userid'], $spot, $spotXml);
}
# if
return $errorList;
}
function process($hdrList, $curMsg, $endMsg)
{
$this->displayStatus("progress", $curMsg . " till " . $endMsg);
$this->_db->beginTransaction();
$signedCount = 0;
$hdrsRetrieved = 0;
$fullsRetrieved = 0;
$modCount = 0;
$skipCount = 0;
# pak onze lijst met messageid's, en kijk welke er al in de database zitten
$t = microtime(true);
$dbIdList = $this->_db->matchMessageIds($hdrList);
# en loop door elke header heen
foreach ($hdrList as $msgid => $msgheader) {
# Reset timelimit
set_time_limit(120);
# messageid to check
$msgId = substr($msgheader['Message-ID'], 1, -1);
# als we de spot overview nog niet in de database hebben, haal hem dan op
if (!in_array($msgId, $dbIdList['spot'])) {
$hdrsRetrieved++;
$spotParser = new SpotParser();
$spot = $spotParser->parseXover($msgheader['Subject'], $msgheader['From'], $msgheader['Message-ID'], $this->_rsakeys);
# als er een parse error was, negeren we de spot volledig, ook niet
# verified spots gooien we weg.
if ($spot === false || !$spot['Verified']) {
continue;
}
# if
if ($spot['KeyID'] == 2) {
$commandAr = explode(' ', strtolower($spot['Title']));
$validCommands = array('delete', 'dispose', 'remove');
# is dit een geldig commando?
if (array_search($commandAr[0], $validCommands) !== false) {
switch ($this->_settings['spot_moderation']) {
case 'disable':
break;
case 'markspot':
$this->_db->markSpotModerated($commandAr[1]);
break;
default:
$this->_db->deleteSpot($commandAr[1]);
break;
}
# switch
$modCount++;
}
# if
} else {
// Oudere spots niet toevoegen, hoeven we het later ook niet te verwijderen
if ($this->_settings['retention'] > 0 && $spot['Stamp'] < time() - $this->_settings['retention'] * 24 * 60 * 60) {
$skipCount++;
} else {
$this->_db->addSpot($spot);
$dbIdList['spot'][] = $msgId;
if ($spot['WasSigned']) {
$signedCount++;
}
# if
}
# if
}
# else
} else {
# anders halen we hem uit de database want we hebben die nodig
$spot = $this->_db->getFullSpot($msgId);
}
# else
# We willen enkel de volledige spot ophalen als de header in de database zit, omdat
# we dat hierboven eventueel doen, is het enkel daarop checken voldoende
if (in_array($msgId, $dbIdList['spot']) && !in_array($msgId, $dbIdList['fullspot'])) {
#
# We gebruiken altijd XOVER, dit is namelijk handig omdat eventueel ontbrekende
# artikel nummers (en soms zijn dat er duizenden) niet hoeven op te vragen, nu
# vragen we enkel de de headers op van de artikelen die er daadwerkelijk zijn
#
# KeyID 2 is een 'moderator' post en kan dus niet getrieved worden
#
if ($this->_retrieveFull && $spot['KeyID'] != 2) {
$fullSpot = array();
try {
$fullsRetrieved++;
$fullSpot = $this->_spotnntp->getFullSpot(substr($msgheader['Message-ID'], 1, -1));
# en voeg hem aan de database toe
$this->_db->addFullSpot($fullSpot);
} catch (ParseSpotXmlException $x) {
# swallow error
} catch (Exception $x) {
# messed up index aan de kant van de server ofzo? iig, dit gebeurt. soms, if so,
# swallow the error
if ($x->getMessage() == 'No such article found') {
} elseif ($x->getMessage() == 'String could not be parsed as XML') {
} else {
throw $x;
}
# else
}
# catch
}
# if retrievefull
}
# if fullspot is not in db yet
}
# foreach
if (count($hdrList) > 0) {
$this->displayStatus("hdrparsed", $hdrsRetrieved);
$this->displayStatus("fullretrieved", $fullsRetrieved);
$this->displayStatus("verified", $signedCount);
$this->displayStatus("modcount", $modCount);
$this->displayStatus("skipcount", $skipCount);
$this->displayStatus("loopcount", count($hdrList));
} else {
$this->displayStatus("hdrparsed", 0);
$this->displayStatus("fullretrieved", 0);
$this->displayStatus("verified", 0);
$this->displayStatus("modcount", 0);
$this->displayStatus("skipcount", 0);
$this->displayStatus("loopcount", 0);
}
# else
$this->_db->setMaxArticleid($this->_server['host'], $curMsg);
$this->_db->commitTransaction();
return count($hdrList);
}
function process($hdrList, $curMsg, $increment)
{
$this->displayStatus("progress", $curMsg . " till " . ($curMsg + $increment));
$this->_db->beginTransaction();
$signedCount = 0;
foreach ($hdrList as $msgid => $msgheader) {
# Reset timelimit
set_time_limit(120);
$spotParser = new SpotParser();
$spot = $spotParser->parseXover($msgheader['Subject'], $msgheader['From'], $msgheader['Message-ID'], $this->_rsakeys);
if ($spot != null && $spot['Verified']) {
$this->_db->addSpot($spot);
}
# if
if ($spot['Verified']) {
if ($spot['WasSigned']) {
$signedCount++;
}
# if
}
# if
}
# foreach
if (count($hdrList) > 0) {
$this->displayStatus("verified", $signedCount);
$this->displayStatus("loopcount", count($hdrList));
} else {
$this->displayStatus("verified", 0);
$this->displayStatus("loopcount", 0);
}
# else
$this->_db->setMaxArticleid($this->_server['host'], $curMsg);
$this->_db->commitTransaction();
return count($hdrList);
}
function getCleanRandomString($len)
{
$spotParser = new SpotParser();
$spotSigning = new SpotSigning();
return substr($spotParser->specialString(base64_encode($spotSigning->makeRandomStr($len))), 0, $len);
}
require_once "lib/SpotSecurity.php";
require_once "lib/SpotTiming.php";
require_once "settings.php";
require_once "lib/SpotDb.php";
define('USERID', 30);
/* -------------------------------------------------------------------- */
echo "Included PHP classes... " . PHP_EOL;
$db = new SpotDb($settings['db']);
$db->connect();
echo "Connected to the database.." . PHP_EOL;
$spotSettings = SpotSettings::singleton($db, $settings);
$spotSigning = new SpotSigning();
$spotPosting = new SpotPosting($db, $spotSettings);
$spotUserSystem = new SpotUserSystem($db, $spotSettings);
echo "Initialized classes.." . PHP_EOL;
$rsaKeys = $spotSettings->get('rsa_keys');
$retriever = new SpotRetriever_Spots($spotSettings->get('nntp_hdr'), $db, $spotSettings, $rsaKeys, '', $spotSettings->get('retrieve_full'));
$msgdata = $retriever->connect($spotSettings->get('hdr_group'));
var_dump($msgdata);
# Connct thru our own NNTP session to the server so we have an XOVER list
$headerServer = $spotSettings->get('nntp_hdr');
$spotnntp = new SpotNntp($spotSettings->get('nntp_hdr'));
$spotnntp->selectGroup($spotSettings->get('hdr_group'));
$hdrList = $spotnntp->getOverview(3244937, 3244938);
foreach ($hdrList as $msgid => $msgheader) {
$spotParser = new SpotParser();
$spot = $spotParser->parseXover($msgheader['Subject'], $msgheader['From'], $msgheader['Date'], $msgheader['Message-ID'], $rsaKeys);
var_dump($spot);
}
# and signal quit
$retriever->quit();
function process($hdrList, $curMsg, $endMsg, $timer)
{
$this->displayStatus("progress", $curMsg . " till " . $endMsg);
$spotParser = new SpotParser();
$signedCount = 0;
$hdrsRetrieved = 0;
$fullsRetrieved = 0;
$msgCounter = 0;
$modCount = 0;
$skipCount = 0;
$lastProcessedId = '';
$fullSpotDbList = array();
$spotDbList = array();
$moderationList = array();
$processingStartTime = time();
/*
* Determine the cutoff date (unixtimestamp) from whereon we do not want to
* load the spots
*/
if ($this->_settings->get('retention') > 0) {
$retentionStamp = time() - $this->_settings->get('retention') * 24 * 60 * 60;
} else {
$retentionStamp = 0;
}
# else
$this->debug('retentionStamp=' . $retentionStamp);
$this->debug('hdrList=' . serialize($hdrList));
/**
* We ask the database to match our messageid's we just retrieved with
* the list of id's we have just retrieved from the server
*/
$dbIdList = $this->_db->matchSpotMessageIds($hdrList);
$this->debug('dbIdList=' . serialize($dbIdList));
# if we need to fetch images or nzb files, we need an spotsoverview instance
if ($this->_retrieveFull && $this->_prefetch_image || $this->_prefetch_nzb) {
$spotsOverview = new SpotsOverview($this->_db, $this->_settings);
$spotsOverview->setActiveRetriever(true);
/*
* Only create a new NZB instance if the server differs from the
* header host, else re-use the connection
*/
$settings_nntp_nzb = $this->_settings->get('nntp_nzb');
if ($this->_server['host'] == $settings_nntp_nzb['host']) {
$nntp_nzb = $this->_spotnntp;
} else {
$nntp_nzb = new SpotNntp($settings_nntp_nzb);
$nntp_nzb->selectGroup($this->_settings->get('nzb_group'));
}
# else
}
# if
foreach ($hdrList as $msgheader) {
$msgCounter++;
$this->debug('foreach-loop, start. msgId= ' . $msgCounter);
/*
* Keep te usenet server alive when processing is slow.
*/
if ($processingStartTime - time() > 30) {
$this->_spotnntp->sendNoop();
if (isset($nntp_nzb) && $nntp_nzb != $this->_spotnntp) {
$nntp_nzb->sendNoop();
}
# if
$processingStartTime = time();
}
# if
/*
* We keep track whether we actually fetched this header and fullspot
* to add it to the database, because only then we can update the
* titel from the spots title or rely on our database to fetch
* the fullspot
*/
$didFetchHeader = false;
$didFetchFullSpot = false;
# Reset timelimit
set_time_limit(120);
# messageid to check
$msgId = substr($msgheader['Message-ID'], 1, -1);
/*
* We prepare some variables to we don't have to perform an array
* lookup for each check and the code is easier to read.
*/
$header_isInDb = isset($dbIdList['spot'][$msgId]);
$fullspot_isInDb = isset($dbIdList['fullspot'][$msgId]);
/*
* If the spotheader is not yet added to the database, parse the header
* information.
*
* If the header is present, but we don't have the fullspot yet or we are
* running in 'retro' mode, parse the header as well because some fields
* are only in the header and not in the full.
*
* We need some of those fields (for example KeyID)
*/
if (!$header_isInDb || (!$fullspot_isInDb || $this->_retro) && $this->_retrieveFull) {
$hdrsRetrieved++;
$this->debug('foreach-loop, parsingXover, start. msgId= ' . $msgCounter);
$spot = $spotParser->parseXover($msgheader['Subject'], $msgheader['From'], $msgheader['Date'], $msgheader['Message-ID'], $this->_rsakeys);
$this->debug('foreach-loop, parsingXover, done. msgId= ' . $msgCounter);
/*
* When a parse error occured, we ignore the spot, also unverified
* spots are ignored
*/
if ($spot === false || !$spot['verified']) {
$this->debug('foreach-loop, spot is either false or not verified');
continue;
}
# if
/*
* Special moderator commands always have keyid 2
*/
if ($spot['keyid'] == 2) {
$this->debug('foreach-loop, spot is a moderation spot');
$commandAr = explode(' ', strtolower($spot['title']));
$validCommands = array('delete', 'dispose', 'remove');
# is this one of the defined valid commands?
if (in_array($commandAr[0], $validCommands) !== false) {
$moderationList[$commandAr[1]] = 1;
$modCount++;
}
# if
} else {
/*
* Don't add spots older than specified for the retention stamp
*/
if ($retentionStamp > 0 && $spot['stamp'] < $retentionStamp && $this->_settings->get('retentiontype') == 'everything') {
$this->debug('foreach-loop, spot is expired: ' . $spot['stamp']);
continue;
} elseif ($spot['stamp'] < $this->_settings->get('retrieve_newer_than')) {
$this->debug('foreach-loop, spot is too old: ' . $spot['stamp']);
$skipCount++;
} else {
/*
* Do we have the header in the database? If not, lets add it
*/
if (!$header_isInDb) {
$spotDbList[] = $spot;
/*
* Some buggy NNTP servers give us the same messageid
* in one XOVER statement, hence we update the list of
* messageid's we already have retrieved and are ready
* to be added to the database
*/
$dbIdList['spot'][$msgId] = 1;
$header_isInDb = true;
$lastProcessedId = $msgId;
$didFetchHeader = true;
if ($spot['wassigned']) {
$signedCount++;
}
# if
}
# if
}
# if
}
# else
} else {
$lastProcessedId = $msgId;
}
# else
/*
* We don't want to retrieve the fullspot if we don't have the header
* in the database. Because we try to add headers in the above code we just have
* to check if the header is in the database.
*
* We cannot collapse this code with the header fetching code because we want to
* be able to add the fullspot to a system after all the headers are retrieved
*/
if ($header_isInDb && !$fullspot_isInDb) {
/*
* Don't add older fullspots than specified for the retention stamp
*/
if ($retentionStamp > 0 && strtotime($msgheader['Date']) < $retentionStamp) {
continue;
}
# if
if ($this->_retrieveFull) {
$fullSpot = array();
try {
$fullsRetrieved++;
$this->debug('foreach-loop, getFullSpot, start. msgId= ' . $msgId);
$fullSpot = $this->_spotnntp->getFullSpot($msgId);
$this->debug('foreach-loop, getFullSpot, done. msgId= ' . $msgId);
# add this spot to the database
$fullSpotDbList[] = $fullSpot;
$fullspot_isInDb = true;
$didFetchFullSpot = true;
/*
* Some buggy NNTP servers give us the same messageid
* in once XOVER statement, hence we update the list of
* messageid's we already have retrieved and are ready
* to be added to the database
*/
$dbIdList['fullspot'][$msgId] = 1;
/*
* Overwrite the spots' title because the fullspot contains the title in
* UTF-8 format.
* We also overwrite the spotterid from the spotsfull because the spotterid
* is only in the header in more recent spots.
*/
if ($didFetchHeader) {
$spotDbList[count($spotDbList) - 1]['title'] = $fullSpot['title'];
$spotDbList[count($spotDbList) - 1]['spotterid'] = $fullSpot['spotterid'];
}
# if
} catch (ParseSpotXmlException $x) {
# swallow error
} catch (Exception $x) {
/**
* Sometimes we get an 'No such article' error for a header we just retrieved,
* if we want to retrieve the full article. This is messed up, but let's just
* swallow the error
*/
if ($x->getCode() == 430) {
} elseif ($x->getMessage() == 'String could not be parsed as XML') {
} else {
throw $x;
}
# else
}
# catch
}
# if retrievefull
}
# if fullspot is not in db yet
if ($this->_retrieveFull && $header_isInDb && ($this->_prefetch_image || $this->_prefetch_nzb)) {
try {
/*
* If we are running in 'retro' mode, it is possible both the header and spot are in the
* database already, however -- we need the information from the fullspot so we retrieve it
* again
*/
if (!$didFetchFullSpot) {
$fullSpot = $this->_db->getFullSpot($msgId, SPOTWEB_ANONYMOUS_USERID);
$fullSpot = array_merge($spotParser->parseFull($fullSpot['fullxml']), $fullSpot);
}
# if
/*
* Prefetch (cache) the spots' image
*/
if ($this->_prefetch_image) {
/*
* If the spot is older than 30 days, and the image is on the web, we do not
* prefetch the image.
*/
if (is_array($fullSpot['image']) || $fullSpot['stamp'] > (int) time() - 30 * 24 * 60 * 60) {
$this->debug('foreach-loop, getImage(), start. msgId= ' . $msgId);
$spotsOverview->getImage($fullSpot, $nntp_nzb);
$this->debug('foreach-loop, getImage(), done. msgId= ' . $msgId);
}
# if
}
# if
/*
* Prefetch (cache) the spots' NZB file
*/
if ($this->_prefetch_nzb) {
/*
* Only do so if we can expect an NZB file
*/
if (!empty($fullSpot['nzb']) && $fullSpot['stamp'] > 1290578400) {
$this->debug('foreach-loop, getNzb(), start. msgId= ' . $msgId);
$spotsOverview->getNzb($fullSpot, $nntp_nzb);
$this->debug('foreach-loop, getNzb(), done. msgId= ' . $msgId);
}
# if
}
# if
} catch (ParseSpotXmlException $x) {
# swallow error
} catch (Exception $x) {
/**
* Sometimes we get an 'No such article' error for a header we just retrieved,
* if we want to retrieve the full article. This is messed up, but let's just
* swallow the error
*/
if ($x->getCode() == 430) {
} elseif ($x->getMessage() == 'String could not be parsed as XML') {
} else {
throw $x;
}
# else
}
# catch
}
# if prefetch image and/or nzb
$this->debug('foreach-loop, done. msgId= ' . $msgCounter);
}
# foreach
if (count($hdrList) > 0) {
$this->displayStatus("hdrparsed", $hdrsRetrieved);
$this->displayStatus("fullretrieved", $fullsRetrieved);
$this->displayStatus("verified", $signedCount);
$this->displayStatus("modcount", $modCount);
$this->displayStatus("skipcount", $skipCount);
$this->displayStatus("loopcount", count($hdrList));
} else {
$this->displayStatus("hdrparsed", 0);
$this->displayStatus("fullretrieved", 0);
$this->displayStatus("verified", 0);
$this->displayStatus("modcount", 0);
$this->displayStatus("skipcount", 0);
$this->displayStatus("loopcount", 0);
}
# else
/*
* Add the spots to the database and update the last article
* number found
*/
$this->_db->addSpots($spotDbList, $fullSpotDbList);
$this->debug('added Spots, spotDbList=' . serialize($spotDbList));
$this->debug('added Spots, fullSpotDbList=' . serialize($fullSpotDbList));
/*
* Actually act on the moderation settings. We cannot process this inline
* because a spot can be added and moderated within the same iteration
*/
switch ($this->_settings->get('spot_moderation')) {
case 'disable':
break;
case 'markspot':
$this->_db->markCommentsModerated($moderationList);
$this->_db->markSpotsModerated($moderationList);
break;
# case 'markspot'
# case 'markspot'
default:
$this->_db->removeSpots($moderationList);
$this->_db->removeComments($moderationList);
break;
# default
}
# switch
# update the maximum article id
if ($this->_retro) {
$this->_db->setMaxArticleid('spots_retro', $endMsg);
} else {
$this->_db->setMaxArticleid($this->_server['host'], $endMsg);
}
# if
$this->debug('loop finished, setMaxArticleId=' . serialize($endMsg));
$this->displayStatus("timer", round(microtime(true) - $timer, 2));
return array('count' => count($hdrList), 'headercount' => $hdrsRetrieved, 'lastmsgid' => $lastProcessedId);
}
function process($hdrList, $curMsg, $endMsg)
{
$this->displayStatus("progress", $curMsg . " till " . $endMsg);
$this->_db->beginTransaction();
$signedCount = 0;
$hdrsRetrieved = 0;
$fullsRetrieved = 0;
$modCount = 0;
$skipCount = 0;
$lastProcessedId = '';
# pak onze lijst met messageid's, en kijk welke er al in de database zitten
$dbIdList = $this->_db->matchSpotMessageIds($hdrList);
#var_dump($hdrList);
# en loop door elke header heen
$spotParser = new SpotParser();
foreach ($hdrList as $msgid => $msgheader) {
# Reset timelimit
set_time_limit(120);
# messageid to check
$msgId = substr($msgheader['Message-ID'], 1, -1);
# definieer een paar booleans zodat we niet steeds een array lookup moeten doen
# en de code wat duidelijker is
$header_isInDb = isset($dbIdList['spot'][$msgId]);
$fullspot_isInDb = isset($dbIdList['fullspot'][$msgId]);
# als we de spot overview nog niet in de database hebben, haal hem dan op,
# ook als de fullspot er nog niet is, moeten we dit doen want een aantal velden
# die wel in de header zitten, zitten niet in de database (denk aan 'keyid')
if (!$header_isInDb || !$fullspot_isInDb) {
$hdrsRetrieved++;
$spot = $spotParser->parseXover($msgheader['Subject'], $msgheader['From'], $msgheader['Date'], $msgheader['Message-ID'], $this->_rsakeys);
# als er een parse error was, negeren we de spot volledig, ook niet-
# verified spots gooien we weg.
if ($spot === false || !$spot['verified']) {
continue;
}
# if
if ($spot['keyid'] == 2) {
$commandAr = explode(' ', strtolower($spot['title']));
$validCommands = array('delete', 'dispose', 'remove');
# FIXME: Message-ID kan ook van een comment zijn,
# onderstaande code gaat uit van een spot.
# is dit een geldig commando?
if (in_array($commandAr[0], $validCommands) !== false) {
switch ($this->_settings->get('spot_moderation')) {
case 'disable':
break;
case 'markspot':
$this->_db->markSpotModerated($commandAr[1]);
break;
default:
$this->_db->deleteSpot($commandAr[1]);
break;
}
# switch
$modCount++;
}
# if
} else {
# Oudere spots niet toevoegen, hoeven we het later ook niet te verwijderen
if ($this->_settings->get('retention') > 0 && $spot['stamp'] < time() - $this->_settings->get('retention') * 24 * 60 * 60) {
$skipCount++;
} elseif ($spot['stamp'] < $this->_settings->get('retrieve_newer_than')) {
$skipCount++;
} else {
# Hier kijken we alleen of de spotheader niet bestaat
if (!$header_isInDb) {
$this->_db->addSpot($spot);
# definieer de header als al ontvangen, we moeten ook de
# msgid lijst updaten omdat soms een messageid meerdere
# keren per xover mee komt ...
$dbIdList['spot'][$msgId] = 1;
$header_isInDb = true;
$lastProcessedId = $msgId;
if ($spot['wassigned']) {
$signedCount++;
}
# if
}
# if
}
# if
}
# else
} else {
$lastProcessedId = $msgId;
}
# else
# We willen enkel de volledige spot ophalen als de header in de database zit, omdat
# we dat hierboven eventueel doen, is het enkel daarop checken voldoende
if ($header_isInDb && !$fullspot_isInDb) {
#
# We gebruiken altijd XOVER, dit is namelijk handig omdat eventueel ontbrekende
# artikel nummers (en soms zijn dat er duizenden) niet hoeven op te vragen, nu
# vragen we enkel de de headers op van de artikelen die er daadwerkelijk zijn
#
# KeyID 2 is een 'moderator' post en kan dus niet getrieved worden
#
if ($this->_retrieveFull && $spot['keyid'] != 2) {
$fullSpot = array();
try {
$fullsRetrieved++;
$fullSpot = $this->_spotnntp->getFullSpot($msgId);
# en voeg hem aan de database toe
$this->_db->addFullSpot($fullSpot);
$fullspot_isInDb = true;
# we moeten ook de msgid lijst updaten omdat soms een messageid meerdere
# keren per xover mee komt ...
$dbIdList['fullspot'][$msgId] = 1;
} catch (ParseSpotXmlException $x) {
# swallow error
} catch (Exception $x) {
# messed up index aan de kant van de server ofzo? iig, dit gebeurt. soms, if so,
# swallow the error
if ($x->getMessage() == 'No such article found') {
} elseif ($x->getMessage() == 'String could not be parsed as XML') {
} else {
throw $x;
}
# else
}
# catch
}
# if retrievefull
}
# if fullspot is not in db yet
}
# foreach
if (count($hdrList) > 0) {
$this->displayStatus("hdrparsed", $hdrsRetrieved);
$this->displayStatus("fullretrieved", $fullsRetrieved);
$this->displayStatus("verified", $signedCount);
$this->displayStatus("modcount", $modCount);
$this->displayStatus("skipcount", $skipCount);
$this->displayStatus("loopcount", count($hdrList));
} else {
$this->displayStatus("hdrparsed", 0);
$this->displayStatus("fullretrieved", 0);
$this->displayStatus("verified", 0);
$this->displayStatus("modcount", 0);
$this->displayStatus("skipcount", 0);
$this->displayStatus("loopcount", 0);
}
# else
$this->_db->setMaxArticleid($this->_server['host'], $curMsg);
$this->_db->commitTransaction();
return array('count' => count($hdrList), 'headercount' => $hdrsRetrieved, 'lastmsgid' => $lastProcessedId);
}
