function getFullSpot($msgId, $ourUserId, $nntp) { $fullSpot = $this->_db->getFullSpot($msgId, $ourUserId); if (empty($fullSpot)) { # Vraag de volledige spot informatie op -- dit doet ook basic # sanity en validatie checking $fullSpot = $nntp->getFullSpot($msgId); $this->_db->addFullSpot($fullSpot); # we halen de fullspot opnieuw op zodat we de 'xover' informatie en de # niet xover informatie in 1 hebben $fullSpot = $this->_db->getFullSpot($msgId, $ourUserId); } # if $spotParser = new SpotParser(); $fullSpot = array_merge($spotParser->parseFull($fullSpot['fullxml']), $fullSpot); /* * Als je een fullspot ophaalt, maar er is nog gen 'spot' entry, dan blijf je een * lege spot terugkrijgen omdat de join misgaat. Omdat dit verwarring op kan leveren * gooien we dan een exception */ if (empty($fullSpot)) { throw new Exception("Spot is not in our Spotweb database"); } # if return $fullSpot; } # getFullSpot
function getFullSpot($msgId, $ourUserId, $nntp) { SpotTiming::start('SpotsOverview::' . __FUNCTION__); $fullSpot = $this->_db->getFullSpot($msgId, $ourUserId); if (empty($fullSpot)) { /* * Retrieve a full loaded spot from the NNTP server */ $newFullSpot = $nntp->getFullSpot($msgId); $this->_db->addFullSpots(array($newFullSpot)); /* * If the current spotterid is empty, we probably now * have a spotterid because we have the fullspot. */ if (empty($fullSpot['spotterid']) && $newFullSpot['verified']) { $spotSigning = Services_Signing_Base::newServiceSigning(); $newFullSpot['spotterid'] = $spotSigning->calculateSpotterId($newFullSpot['user-key']['modulo']); /* * Update the spotterid in the spots table so it can be filtered later on */ $this->_db->updateSpotInfoFromFull($newFullSpot); } # if /* * We ask our DB to retrieve the fullspot again, this ensures * us all information is present and in always the same format */ $fullSpot = $this->_db->getFullSpot($msgId, $ourUserId); } # if /** * We'll overwrite our spot info from the database with some information we parse from the * XML. This is necessary because the XML contains better encoding. * * For example take the titel from spot bdZZdJ3gPxTAmSE%40spot.net. * * We cannot use all information from the XML because because some information just * isn't present in the XML file */ $spotParser = new SpotParser(); $parsedXml = $spotParser->parseFull($fullSpot['fullxml']); $fullSpot = array_merge($parsedXml, $fullSpot); $fullSpot['title'] = $parsedXml['title']; /* * When we retrieve a fullspot entry but there is no spot entry the join in our DB query * causes us to never get the spot, hence we throw this exception */ if (empty($fullSpot)) { throw new Exception("Spot is not in our Spotweb database"); } # if SpotTiming::stop('SpotsOverview::' . __FUNCTION__, array($msgId, $ourUserId, $nntp, $fullSpot)); return $fullSpot; }
function getFullSpot($msgId, $nntp) { $fullSpot = $this->_db->getFullSpot($msgId); if (empty($fullSpot) || !isset($fullSpot['verified'])) { # Vraag de volledige spot informatie op -- dit doet ook basic # sanity en validatie checking $fullSpot = $nntp->getFullSpot($msgId); $this->_db->addFullSpot($fullSpot); } # if $spotParser = new SpotParser(); $fullSpot = array_merge($spotParser->parseFull($fullSpot['xml']), $fullSpot); return $fullSpot; }
function getFullSpot($msgId, $nntp) { $fullSpot = $this->_db->getFullSpot($msgId); if (empty($fullSpot)) { # Vraag de volledige spot informatie op -- dit doet ook basic # sanity en validatie checking $fullSpot = $nntp->getFullSpot($msgId); $this->_db->addFullSpot($fullSpot); # we halen de fullspot opnieuw op zodat we de 'xover' informatie en de # niet xover informatie in 1 hebben $fullSpot = $this->_db->getFullSpot($msgId); } # if $spotParser = new SpotParser(); $fullSpot = array_merge($spotParser->parseFull($fullSpot['fullxml']), $fullSpot); return $fullSpot; }
function getFullSpot($msgId, $ourUserId, $nntp) { $fullSpot = $this->_db->getFullSpot($msgId, $ourUserId); if (empty($fullSpot)) { # Vraag de volledige spot informatie op -- dit doet ook basic # sanity en validatie checking $fullSpot = $nntp->getFullSpot($msgId); $this->_db->addFullSpots(array($fullSpot)); # we halen de fullspot opnieuw op zodat we de 'xover' informatie en de # niet xover informatie in 1 hebben $fullSpot = $this->_db->getFullSpot($msgId, $ourUserId); } # if /** * Overschrijf nu onze 'spot info' uit de database met sommige info welke we uit de * de XML parseren, we doen dit omdat de XML o.a. betere encoding bevat, zie de titel van spot * bdZZdJ3gPxTAmSE%40spot.net bijvoorbeeld. * * Alles uit de SpotsFull aannemen is niet interessant omdat niet elke XML versie alle * informatie bevat */ $spotParser = new SpotParser(); $parsedXml = $spotParser->parseFull($fullSpot['fullxml']); $fullSpot = array_merge($parsedXml, $fullSpot); $fullSpot['title'] = $parsedXml['title']; /* * Als je een fullspot ophaalt, maar er is nog gen 'spot' entry, dan blijf je een * lege spot terugkrijgen omdat de join misgaat. Omdat dit verwarring op kan leveren * gooien we dan een exception */ if (empty($fullSpot)) { throw new Exception("Spot is not in our Spotweb database"); } # if return $fullSpot; }
function getFullSpot($msgId) { # initialize some variables $spotParser = new SpotParser(); $spot = array('xml' => '', 'user-signature' => '', 'user-key' => '', 'verified' => false, 'messageid' => $msgId, 'userid' => '', 'xml-signature' => ''); # Vraag de volledige article header van de spot op $header = $this->getHeader('<' . $msgId . '>'); # Parse de header foreach ($header as $str) { $keys = explode(':', $str); switch ($keys[0]) { case 'X-XML': $spot['xml'] .= substr($str, 7); break; case 'X-User-Signature': $spot['user-signature'] = base64_decode($spotParser->unspecialString(substr($str, 18))); break; case 'X-XML-Signature': $spot['xml-signature'] = substr($str, 17); break; case 'X-User-Key': $xml = simplexml_load_string(substr($str, 12)); if ($xml !== false) { $spot['user-key']['exponent'] = (string) $xml->Exponent; $spot['user-key']['modulo'] = (string) $xml->Modulus; } # if break; # x-user-key } # switch } # foreach # Valideer de signature van de XML, deze is gesigned door de user zelf if (!empty($spot['user-signature']) && !empty($spot['user-key'])) { $spot['verified'] = $spotParser->checkRsaSignature('<' . $spot['messageid'] . '>', $spot['user-signature'], $spot['user-key']); if (!$spot['verified']) { $spot['verified'] = $spotParser->checkRsaSignature($spot['xml-signature'], $spot['user-signature'], $spot['user-key']); } # if } else { $spot['verified'] = false; } # else # als de spot verified is, toon dan de userid van deze user if ($spot['verified']) { $spot['userid'] = $spotParser->calculateUserid($spot['user-key']['modulo']); } # if # Parse nu de XML file, alles wat al gedefinieerd is eerder wordt niet overschreven $spot = array_merge($spotParser->parseFull($spot['xml']), $spot); return $spot; }
} # else $header = $hdr_spotnntp->getHeader('<' . $spot['messageid'] . '>'); $xml = ''; if ($header !== false) { foreach ($header as $str) { if (substr($str, 0, 7) == 'X-XML: ') { $xml .= substr($str, 7); } # if } # foreach } # if $spotParser = new SpotParser(); $xmlar = $spotParser->parseFull($xml); /* Connect to the NZB group */ /* Get the NZB file */ $nzb = false; if (is_array($xmlar['segment'])) { foreach ($xmlar['segment'] as $seg) { $nzb .= implode("", $nzb_spotnntp->getBody("<" . $seg . ">")); } # foreach } else { $nzb .= implode("", $nzb_spotnntp->getBody("<" . $xmlar['segment'] . ">")); } # if if ($nzb !== false) { if ($settings['nzb_download_local'] == true) { $myFile = $settings['nzb_local_queue_dir'] . $xmlar['title'] . ".nzb";
function process($hdrList, $curMsg, $endMsg, $timer) { $this->displayStatus("progress", $curMsg . " till " . $endMsg); $spotParser = new SpotParser(); $signedCount = 0; $hdrsRetrieved = 0; $fullsRetrieved = 0; $msgCounter = 0; $modCount = 0; $skipCount = 0; $lastProcessedId = ''; $fullSpotDbList = array(); $spotDbList = array(); $moderationList = array(); $processingStartTime = time(); /* * Determine the cutoff date (unixtimestamp) from whereon we do not want to * load the spots */ if ($this->_settings->get('retention') > 0) { $retentionStamp = time() - $this->_settings->get('retention') * 24 * 60 * 60; } else { $retentionStamp = 0; } # else $this->debug('retentionStamp=' . $retentionStamp); $this->debug('hdrList=' . serialize($hdrList)); /** * We ask the database to match our messageid's we just retrieved with * the list of id's we have just retrieved from the server */ $dbIdList = $this->_db->matchSpotMessageIds($hdrList); $this->debug('dbIdList=' . serialize($dbIdList)); # if we need to fetch images or nzb files, we need an spotsoverview instance if ($this->_retrieveFull && $this->_prefetch_image || $this->_prefetch_nzb) { $spotsOverview = new SpotsOverview($this->_db, $this->_settings); $spotsOverview->setActiveRetriever(true); /* * Only create a new NZB instance if the server differs from the * header host, else re-use the connection */ $settings_nntp_nzb = $this->_settings->get('nntp_nzb'); if ($this->_server['host'] == $settings_nntp_nzb['host']) { $nntp_nzb = $this->_spotnntp; } else { $nntp_nzb = new SpotNntp($settings_nntp_nzb); $nntp_nzb->selectGroup($this->_settings->get('nzb_group')); } # else } # if foreach ($hdrList as $msgheader) { $msgCounter++; $this->debug('foreach-loop, start. msgId= ' . $msgCounter); /* * Keep te usenet server alive when processing is slow. */ if ($processingStartTime - time() > 30) { $this->_spotnntp->sendNoop(); if (isset($nntp_nzb) && $nntp_nzb != $this->_spotnntp) { $nntp_nzb->sendNoop(); } # if $processingStartTime = time(); } # if /* * We keep track whether we actually fetched this header and fullspot * to add it to the database, because only then we can update the * titel from the spots title or rely on our database to fetch * the fullspot */ $didFetchHeader = false; $didFetchFullSpot = false; # Reset timelimit set_time_limit(120); # messageid to check $msgId = substr($msgheader['Message-ID'], 1, -1); /* * We prepare some variables to we don't have to perform an array * lookup for each check and the code is easier to read. */ $header_isInDb = isset($dbIdList['spot'][$msgId]); $fullspot_isInDb = isset($dbIdList['fullspot'][$msgId]); /* * If the spotheader is not yet added to the database, parse the header * information. * * If the header is present, but we don't have the fullspot yet or we are * running in 'retro' mode, parse the header as well because some fields * are only in the header and not in the full. * * We need some of those fields (for example KeyID) */ if (!$header_isInDb || (!$fullspot_isInDb || $this->_retro) && $this->_retrieveFull) { $hdrsRetrieved++; $this->debug('foreach-loop, parsingXover, start. msgId= ' . $msgCounter); $spot = $spotParser->parseXover($msgheader['Subject'], $msgheader['From'], $msgheader['Date'], $msgheader['Message-ID'], $this->_rsakeys); $this->debug('foreach-loop, parsingXover, done. msgId= ' . $msgCounter); /* * When a parse error occured, we ignore the spot, also unverified * spots are ignored */ if ($spot === false || !$spot['verified']) { $this->debug('foreach-loop, spot is either false or not verified'); continue; } # if /* * Special moderator commands always have keyid 2 */ if ($spot['keyid'] == 2) { $this->debug('foreach-loop, spot is a moderation spot'); $commandAr = explode(' ', strtolower($spot['title'])); $validCommands = array('delete', 'dispose', 'remove'); # is this one of the defined valid commands? if (in_array($commandAr[0], $validCommands) !== false) { $moderationList[$commandAr[1]] = 1; $modCount++; } # if } else { /* * Don't add spots older than specified for the retention stamp */ if ($retentionStamp > 0 && $spot['stamp'] < $retentionStamp && $this->_settings->get('retentiontype') == 'everything') { $this->debug('foreach-loop, spot is expired: ' . $spot['stamp']); continue; } elseif ($spot['stamp'] < $this->_settings->get('retrieve_newer_than')) { $this->debug('foreach-loop, spot is too old: ' . $spot['stamp']); $skipCount++; } else { /* * Do we have the header in the database? If not, lets add it */ if (!$header_isInDb) { $spotDbList[] = $spot; /* * Some buggy NNTP servers give us the same messageid * in one XOVER statement, hence we update the list of * messageid's we already have retrieved and are ready * to be added to the database */ $dbIdList['spot'][$msgId] = 1; $header_isInDb = true; $lastProcessedId = $msgId; $didFetchHeader = true; if ($spot['wassigned']) { $signedCount++; } # if } # if } # if } # else } else { $lastProcessedId = $msgId; } # else /* * We don't want to retrieve the fullspot if we don't have the header * in the database. Because we try to add headers in the above code we just have * to check if the header is in the database. * * We cannot collapse this code with the header fetching code because we want to * be able to add the fullspot to a system after all the headers are retrieved */ if ($header_isInDb && !$fullspot_isInDb) { /* * Don't add older fullspots than specified for the retention stamp */ if ($retentionStamp > 0 && strtotime($msgheader['Date']) < $retentionStamp) { continue; } # if if ($this->_retrieveFull) { $fullSpot = array(); try { $fullsRetrieved++; $this->debug('foreach-loop, getFullSpot, start. msgId= ' . $msgId); $fullSpot = $this->_spotnntp->getFullSpot($msgId); $this->debug('foreach-loop, getFullSpot, done. msgId= ' . $msgId); # add this spot to the database $fullSpotDbList[] = $fullSpot; $fullspot_isInDb = true; $didFetchFullSpot = true; /* * Some buggy NNTP servers give us the same messageid * in once XOVER statement, hence we update the list of * messageid's we already have retrieved and are ready * to be added to the database */ $dbIdList['fullspot'][$msgId] = 1; /* * Overwrite the spots' title because the fullspot contains the title in * UTF-8 format. * We also overwrite the spotterid from the spotsfull because the spotterid * is only in the header in more recent spots. */ if ($didFetchHeader) { $spotDbList[count($spotDbList) - 1]['title'] = $fullSpot['title']; $spotDbList[count($spotDbList) - 1]['spotterid'] = $fullSpot['spotterid']; } # if } catch (ParseSpotXmlException $x) { # swallow error } catch (Exception $x) { /** * Sometimes we get an 'No such article' error for a header we just retrieved, * if we want to retrieve the full article. This is messed up, but let's just * swallow the error */ if ($x->getCode() == 430) { } elseif ($x->getMessage() == 'String could not be parsed as XML') { } else { throw $x; } # else } # catch } # if retrievefull } # if fullspot is not in db yet if ($this->_retrieveFull && $header_isInDb && ($this->_prefetch_image || $this->_prefetch_nzb)) { try { /* * If we are running in 'retro' mode, it is possible both the header and spot are in the * database already, however -- we need the information from the fullspot so we retrieve it * again */ if (!$didFetchFullSpot) { $fullSpot = $this->_db->getFullSpot($msgId, SPOTWEB_ANONYMOUS_USERID); $fullSpot = array_merge($spotParser->parseFull($fullSpot['fullxml']), $fullSpot); } # if /* * Prefetch (cache) the spots' image */ if ($this->_prefetch_image) { /* * If the spot is older than 30 days, and the image is on the web, we do not * prefetch the image. */ if (is_array($fullSpot['image']) || $fullSpot['stamp'] > (int) time() - 30 * 24 * 60 * 60) { $this->debug('foreach-loop, getImage(), start. msgId= ' . $msgId); $spotsOverview->getImage($fullSpot, $nntp_nzb); $this->debug('foreach-loop, getImage(), done. msgId= ' . $msgId); } # if } # if /* * Prefetch (cache) the spots' NZB file */ if ($this->_prefetch_nzb) { /* * Only do so if we can expect an NZB file */ if (!empty($fullSpot['nzb']) && $fullSpot['stamp'] > 1290578400) { $this->debug('foreach-loop, getNzb(), start. msgId= ' . $msgId); $spotsOverview->getNzb($fullSpot, $nntp_nzb); $this->debug('foreach-loop, getNzb(), done. msgId= ' . $msgId); } # if } # if } catch (ParseSpotXmlException $x) { # swallow error } catch (Exception $x) { /** * Sometimes we get an 'No such article' error for a header we just retrieved, * if we want to retrieve the full article. This is messed up, but let's just * swallow the error */ if ($x->getCode() == 430) { } elseif ($x->getMessage() == 'String could not be parsed as XML') { } else { throw $x; } # else } # catch } # if prefetch image and/or nzb $this->debug('foreach-loop, done. msgId= ' . $msgCounter); } # foreach if (count($hdrList) > 0) { $this->displayStatus("hdrparsed", $hdrsRetrieved); $this->displayStatus("fullretrieved", $fullsRetrieved); $this->displayStatus("verified", $signedCount); $this->displayStatus("modcount", $modCount); $this->displayStatus("skipcount", $skipCount); $this->displayStatus("loopcount", count($hdrList)); } else { $this->displayStatus("hdrparsed", 0); $this->displayStatus("fullretrieved", 0); $this->displayStatus("verified", 0); $this->displayStatus("modcount", 0); $this->displayStatus("skipcount", 0); $this->displayStatus("loopcount", 0); } # else /* * Add the spots to the database and update the last article * number found */ $this->_db->addSpots($spotDbList, $fullSpotDbList); $this->debug('added Spots, spotDbList=' . serialize($spotDbList)); $this->debug('added Spots, fullSpotDbList=' . serialize($fullSpotDbList)); /* * Actually act on the moderation settings. We cannot process this inline * because a spot can be added and moderated within the same iteration */ switch ($this->_settings->get('spot_moderation')) { case 'disable': break; case 'markspot': $this->_db->markCommentsModerated($moderationList); $this->_db->markSpotsModerated($moderationList); break; # case 'markspot' # case 'markspot' default: $this->_db->removeSpots($moderationList); $this->_db->removeComments($moderationList); break; # default } # switch # update the maximum article id if ($this->_retro) { $this->_db->setMaxArticleid('spots_retro', $endMsg); } else { $this->_db->setMaxArticleid($this->_server['host'], $endMsg); } # if $this->debug('loop finished, setMaxArticleId=' . serialize($endMsg)); $this->displayStatus("timer", round(microtime(true) - $timer, 2)); return array('count' => count($hdrList), 'headercount' => $hdrsRetrieved, 'lastmsgid' => $lastProcessedId); }