/**
  * Returns an Sevice_Nntp_Engine but tries to minimize
  * the amount of different objects and hence connections 
  * which are created by issueing existing NNTP engines
  * when possible
  *
  * @returns Services_Nntp_Engine Instance of Services_NNTP_Engine
  */
 public static function pool(Services_Settings_Container $settings, $type)
 {
     SpotDebug::msg(SpotDebug::DEBUG, __CLASS__ . '::pool:(' . $type . ') called');
     if (isset(self::$_instances[$type])) {
         return self::$_instances[$type];
     }
     # if
     /*
      * Make sure we have a valid NNTP configuration
      */
     $settings_nntp_hdr = $settings->get('nntp_hdr');
     if (empty($settings_nntp_hdr)) {
         throw new MissingNntpConfigurationException();
     }
     # if
     /*
      * Retrieve the NNTP header settings we can validate those
      */
     switch ($type) {
         case 'hdr':
             self::$_instances[$type] = new Services_Nntp_Engine($settings_nntp_hdr);
             break;
         case 'bin':
             $settings_nntp_bin = $settings->get('nntp_nzb');
             if (empty($settings_nntp_bin['host'])) {
                 self::$_instances[$type] = self::pool($settings, 'hdr');
             } else {
                 self::$_instances[$type] = new Services_Nntp_Engine($settings_nntp_bin);
             }
             # else
             break;
             # nzb
         # nzb
         case 'post':
             $settings_nntp_post = $settings->get('nntp_post');
             if (empty($settings_nntp_post['host'])) {
                 self::$_instances[$type] = self::pool($settings, 'hdr');
             } else {
                 self::$_instances[$type] = new Services_Nntp_Engine($settings_nntp_post);
             }
             # else
             break;
             # post
         # post
         default:
             throw new Exception("Unknown NNTP type engine (" . $type . ") for pool creation");
     }
     # switch
     return self::$_instances[$type];
 }
 /**
  * Retrieves an uncached GET from the web
  *
  * @param $url string to retrieve
  * @param $lastModTime int Last modification time, can be null
  * @param int $redirTries Amount of tries already passed to follow a redirect
  * @return mixed array with first element the HTTP code, and second with the data (if any)
  */
 public function perform($url, $lastModTime = null, $redirTries = 0)
 {
     SpotTiming::start(__CLASS__ . '::' . __FUNCTION__);
     /*
      * Default our effectiveUrl to be the current URL,
      * so this way we can always return the effectiveUrl
      */
     $effectiveUrl = $url;
     $ch = curl_init();
     curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows NT 6.2; WOW64; rv:26.0) Gecko/20100101 Firefox/26.0');
     curl_setopt($ch, CURLOPT_URL, $url);
     curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
     curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 5);
     curl_setopt($ch, CURLOPT_TIMEOUT, 15);
     curl_setopt($ch, CURLOPT_ENCODING, '');
     // Don't use fail on error, because sometimes we do want to se
     // the output of the content
     //      curl_setopt ($ch, CURLOPT_FAILONERROR, 1);
     // eg, if a site returns an 400 we might want to know why.
     curl_setopt($ch, CURLOPT_HEADER, 1);
     curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
     curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
     curl_setopt($ch, CURLINFO_HEADER_OUT, true);
     curl_setopt($ch, CURLOPT_VERBOSE, true);
     // send a cookie with the request if defined
     if ($this->getCookie() !== null) {
         curl_setopt($ch, CURLOPT_COOKIE, $this->getCookie());
     }
     # if
     // Only use these curl options if no open base dir is set and php mode is off.
     $manualRedirect = false;
     if (ini_get('open_basedir') != '' || ini_get('safe_mode')) {
         $manualRedirect = true;
         curl_setopt($ch, CURLOPT_FOLLOWLOCATION, false);
         curl_setopt($ch, CURLOPT_MAXREDIRS, 1);
     } else {
         curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
     }
     # else
     /*
      * If specified, pass authorization for this request
      */
     $username = $this->getUsername();
     if (!empty($username)) {
         curl_setopt($ch, CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
         curl_setopt($ch, CURLOPT_USERPWD, $this->getUsername() . ':' . $this->getPassword());
     }
     // # if
     /*
      * OAuth 2.0 uses 'Bearer' authentication, we support this by manually sending the
      * HTTP header field
      */
     $bearerAuth = $this->getBearerAuth();
     if (!empty($bearerAuth)) {
         $this->addHttpHeaders(array('Authorization: Bearer ' . $this->getBearerAuth()));
     }
     # if
     /*
      * Should we be posting?
      */
     if ($this->getMethod() == 'POST') {
         curl_setopt($ch, CURLOPT_POST, true);
     }
     # if
     /*
      * If we are passed fields to post to the server, actuall post them
      */
     if (($this->getPostContent() != null || $this->getUploadFiles() != null || $this->getRawPostData() != null) && $this->getMethod() == 'POST') {
         $this->addPostFieldsToCurl($ch, $this->getPostContent(), $this->getUploadFiles(), $this->getRawPostData());
     }
     # if
     /*
      * If we already have content stored in our cache, just ask
      * the server if the content is modified since our last
      * time this was stored in the cache
      */
     if ($lastModTime != null && $lastModTime > 0) {
         curl_setopt($ch, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
         curl_setopt($ch, CURLOPT_TIMEVALUE, $lastModTime);
     }
     # if
     /*
      * Send our custom HTTP headers
      */
     $httpHeaders = $this->getHttpHeaders();
     if (!empty($httpHeaders)) {
         curl_setopt($ch, CURLOPT_HTTPHEADER, $this->getHttpHeaders());
     }
     # if
     $response = curl_exec($ch);
     $errorStr = curl_error($ch);
     /*
      * Curl returns false on some unspecified errors (eg: a timeout)
      */
     if ($response !== false) {
         $curl_info = curl_getinfo($ch);
         $http_code = curl_getinfo($ch, CURLINFO_HTTP_CODE);
         /*
          * Server responded with 304 (Resource not modified)
          */
         if ($http_code != 304) {
             $data = substr($response, $curl_info['header_size']);
         } else {
             $data = '';
         }
         # else
         /*
          * We also follow redirects, but PHP's safemode doesn't allow
          * for redirects, so fix those as well.
          */
         $effectiveUrl = curl_getinfo($ch, CURLINFO_EFFECTIVE_URL);
         if (($effectiveUrl != $url || $http_code == 301 || $http_code == 302) && $manualRedirect) {
             if (preg_match('/Location:(.*?)\\n/', $response, $matches)) {
                 $redirUrl = trim(array_pop($matches));
                 $redirTries++;
                 if ($redirTries < 20) {
                     return $this->perform($redirUrl, $lastModTime, $redirTries);
                 }
                 # if
             }
             # if
         }
         # if
         // Get the url.
         if (preg_match('/meta.+?http-equiv\\W+?refresh/i', $response)) {
             preg_match('/content.+?url\\W+?(.+?)\\"/i', $response, $matches);
             if (isset($matches[1])) {
                 SpotDebug::msg(SpotDebug::DEBUG, __CLASS__ . '-perform(), matches[1]= ' . $matches[1]);
                 /*
                  * We can get either an relative redirect, or an fully
                  * qualified redirect. Hideref, for example, uses an
                  * relative direct. Look for those.
                  *
                  * parse_url() doesn't support relative url's, so we have
                  * to do a guess ourselves.
                  */
                 $redirUrl = $matches[1];
                 if (stripos($redirUrl, 'http://') !== 0 && stripos($redirUrl, 'https://') !== 0 && stripos($redirUrl, '//') !== 0) {
                     SpotDebug::msg(SpotDebug::DEBUG, __CLASS__ . '->perform(), we have gotten an correct url');
                     $urlParts = parse_url($url);
                     SpotDebug::msg(SpotDebug::DEBUG, __CLASS__ . '->perform(), parse_url: ' . json_encode($urlParts));
                     if ($redirUrl[0] == '/') {
                         $redirUrl = $urlParts['scheme'] . '://' . $urlParts['host'] . $redirUrl;
                     } else {
                         if (!isset($urlParts['path'])) {
                             $urlParts['path'] = '';
                         }
                         // if
                         $redirUrl = $urlParts['scheme'] . '://' . $urlParts['host'] . $urlParts['path'] . $redirUrl;
                     }
                     # if
                 }
                 # if
                 SpotDebug::msg(SpotDebug::DEBUG, __CLASS__ . '->perform(), after metafresh, url = : ' . $url);
                 $redirTries++;
                 if ($redirTries < 20) {
                     return $this->perform($redirUrl, $lastModTime, $redirTries);
                 }
                 # if
             }
             # if
         }
         # if
     } else {
         $http_code = 700;
         # Curl returned an error
         $curl_info = curl_getinfo($ch);
         $data = '';
     }
     # else
     curl_close($ch);
     /*
      * Sometimes we get an HTTP error of 0 back, which
      * probably means a timeout or something, so fix up
      * the error string manually.
      */
     if ($errorStr == '' && $http_code == 0) {
         $errorStr = 'unable to connect to URL: ' . $url;
     }
     # if
     SpotTiming::stop(__CLASS__ . '::' . __FUNCTION__, array($url));
     return array('http_code' => $http_code, 'data' => $data, 'finalurl' => $effectiveUrl, 'successful' => $http_code == 200 || $http_code == 304, 'errorstr' => 'http returncode: ' . $http_code . ' / ' . $errorStr, 'curl_info' => $curl_info);
 }
 function process($hdrList, $curArtNr, $increment, $timer)
 {
     $this->displayStatus("progress", $curArtNr . " till " . $increment);
     $lastProcessedId = '';
     $lastProcessedArtNr = 0;
     $commentDbList = array();
     $fullCommentDbList = array();
     /*
      * Determine the cutoff date (unixtimestamp) from whereon we do not want to 
      * load the spots
      */
     if ($this->_settings->get('retention') > 0) {
         $retentionStamp = time() - $this->_settings->get('retention') * 24 * 60 * 60;
     } else {
         $retentionStamp = 0;
     }
     # else
     /**
      * We ask the database to match our messageid's we just retrieved with
      * the list of id's we have just retrieved from the server
      */
     SpotTiming::start(__CLASS__ . '::' . __FUNCTION__ . ':matchCommentMessageIds');
     $dbIdList = $this->_commentDao->matchCommentMessageIds($hdrList);
     SpotTiming::stop(__CLASS__ . '::' . __FUNCTION__ . ':matchCommentMessageIds');
     /*
      * We keep a seperate list of messageid's for updating the amount of
      * comments each spot.
      */
     $spotMsgIdList = array();
     /*
      * and a different list for comments with a rating, this way we wont
      * calculcate the rating for a spot when a comments has no rating
      */
     $spotMsgIdRatingList = array();
     # Process each header
     SpotTiming::start(__CLASS__ . '::' . __FUNCTION__ . ':forEach-hdrList');
     foreach ($hdrList as $msgheader) {
         SpotDebug::msg(SpotDebug::DEBUG, 'foreach-loop: iter-start');
         # Reset timelimit
         set_time_limit(120);
         # strip the <>'s from the reference
         $commentId = $msgheader['Message-ID'];
         $artNr = $msgheader['Number'];
         SpotDebug::msg(SpotDebug::DEBUG, 'foreach-loop: processing: ' . $commentId . ', artNr=' . $artNr);
         /*
          * We prepare some variables to we don't have to perform an array
          * lookup for each check and the code is easier to read.
          */
         $header_isInDb = isset($dbIdList['comment'][$commentId]);
         $fullcomment_isInDb = isset($dbIdList['fullcomment'][$commentId]);
         SpotDebug::msg(SpotDebug::DEBUG, 'foreach-loop: headerIsInDb: ' . (int) $header_isInDb . ', fullComment=' . (int) $fullcomment_isInDb . ', retrieveFull= ' . (int) $this->_retrieveFull);
         /*
          * Do we have the comment in the database already? If not, lets process it
          */
         if (!$header_isInDb || !$fullcomment_isInDb && $this->_retrieveFull) {
             SpotTiming::start(__CLASS__ . '::' . __FUNCTION__ . ':forEach-hdrList->retrieveFull');
             /*
              * Because not all usenet servers pass the reference field properly,
              * we manually create this reference field by using the messageid of
              * the comment
              */
             $msgIdParts = explode(".", $commentId);
             $msgheader['References'] = $msgIdParts[0] . substr($commentId, strpos($commentId, '@'));
             $msgheader['stamp'] = strtotime($msgheader['Date']);
             SpotDebug::msg(SpotDebug::TRACE, 'foreach-loop: msgHeader=' . serialize($msgheader));
             /*
              * Don't add older comments than specified for the retention stamp
              */
             if ($retentionStamp > 0 && $msgheader['stamp'] < $retentionStamp && $this->_settings->get('retentiontype') == 'everything') {
                 continue;
             }
             # if
             if ($msgheader['stamp'] < $this->_settings->get('retrieve_newer_than')) {
                 continue;
             }
             # if
             /*
              * Newer kind of comments contain a rating, if we think this comment
              * is such a comment, extract the rating
              */
             if (count($msgIdParts) == 5) {
                 $msgheader['rating'] = (int) $msgIdParts[1];
                 /*
                  * Some older comments contain an non-numeric string
                  * on this position. Make sure this is an number else
                  * reset to zero (no rating given)
                  */
                 if (!is_numeric($msgIdParts[1])) {
                     $msgheader['rating'] = 0;
                 }
                 # if
             } else {
                 $msgheader['rating'] = 0;
             }
             # if
             /*
              * Determine whether we need to add the header to the database
              * and extract the required fields 
              */
             if (!$header_isInDb) {
                 $commentDbList[] = array('messageid' => $commentId, 'nntpref' => $msgheader['References'], 'stamp' => $msgheader['stamp'], 'rating' => $msgheader['rating']);
                 /*
                  * Some buggy NNTP servers give us the same messageid
                  * in one XOVER statement, hence we update the list of
                  * messageid's we already have retrieved and are ready
                  * to be added to the database
                  */
                 $dbIdList['comment'][$commentId] = 1;
                 $spotMsgIdList[$msgheader['References']] = 1;
                 /*
                  * If this comment contains a rating, mark the spot to
                  * have it's rating be recalculated
                  */
                 if ($msgheader['rating'] >= 1 && $msgheader['rating'] <= 10) {
                     $spotMsgIdRatingList[$msgheader['References']] = 1;
                 }
                 # if
                 $header_isInDb = true;
                 $lastProcessedId = $commentId;
                 $lastProcessedArtNr = $artNr;
                 $didFetchHeader = true;
             }
             # if
             SpotTiming::stop(__CLASS__ . '::' . __FUNCTION__ . ':forEach-hdrList->retrieveFull');
         } else {
             $lastProcessedId = $commentId;
             $lastProcessedArtNr = $artNr;
         }
         # else
         /*
          * We don't want to retrieve the full comment body if we don't have the header
          * in the database. Because we try to add headers in the above code we just have
          * to check if the header is in the database.
          *
          * We cannot collapse this code with the header fetching code because we want to
          * be able to add the full body to a system after all the headers are retrieved
          */
         if ($header_isInDb && !$fullcomment_isInDb) {
             /*
              * Don't add older fullcomments than specified for the retention stamp
              */
             if ($retentionStamp > 0 && strtotime($msgheader['Date']) < $retentionStamp) {
                 continue;
             }
             # if
             if ($this->_retrieveFull) {
                 try {
                     SpotDebug::msg(SpotDebug::DEBUG, 'foreach-loop: readFullComment start:' . $commentId);
                     $fullComment = $this->_svcNntpTextReading->readComments(array(array('messageid' => $commentId)));
                     SpotDebug::msg(SpotDebug::DEBUG, 'foreach-loop: readFullComment finished:' . $commentId);
                     # Add this comment to the datbase and mark it as such
                     $fullCommentDbList[] = $fullComment;
                     $fullcomment_isInDb = true;
                     /*
                      * Some buggy NNTP servers give us the same messageid
                      * in one XOVER statement, hence we update the list of
                      * messageid's we already have retrieved and are ready
                      * to be added to the database
                      */
                     $dbIdList['fullcomment'][$commentId] = 1;
                 } catch (ParseSpotXmlException $x) {
                     # swallow error
                 } catch (Exception $x) {
                     /**
                      * Sometimes we get an 'No such article' error for a header we just retrieved,
                      * if we want to retrieve the full article. This is messed up, but let's just
                      * swallow the error
                      */
                     if ($x->getCode() == 430) {
                         /*
                          * Reset error count, so other errors are actually re-tried
                          */
                         $this->_svcNntpText->resetErrorCount();
                         $this->_svcNntpBin->resetErrorCount();
                     } elseif ($x->getMessage() == 'String could not be parsed as XML') {
                     } else {
                         throw $x;
                     }
                     # else
                 }
                 # catch
             }
             # if retrievefull
         }
         # if fullcomment is not in db yet
         SpotDebug::msg(SpotDebug::DEBUG, 'foreach-loop: iter-stop');
     }
     # foreach
     SpotTiming::stop(__CLASS__ . '::' . __FUNCTION__ . ':forEach-hdrList');
     SpotDebug::msg(SpotDebug::DEBUG, 'foreach-loop: done');
     if (count($hdrList) > 0) {
         $this->displayStatus("loopcount", count($hdrList));
     } else {
         $this->displayStatus("loopcount", 0);
     }
     # else
     $this->displayStatus("timer", round(microtime(true) - $timer, 2));
     /* 
      * Add the comments to the database and update the last article
      * number found
      */
     $fullComments = array();
     while ($fullComment = array_shift($fullCommentDbList)) {
         $fullComments = array_merge($fullComments, $fullComment);
     }
     # while
     SpotTiming::start(__CLASS__ . '::' . __FUNCTION__ . ':addComments() call');
     $this->_commentDao->addComments($commentDbList, $fullComments);
     SpotTiming::stop(__CLASS__ . '::' . __FUNCTION__ . ':addComments() call');
     # update the maximum article id
     if (!empty($lastProcessedId) && $lastProcessedArtNr > 0) {
         $this->_usenetStateDao->setMaxArticleId(Dao_UsenetState::State_Comments, $lastProcessedArtNr, $lastProcessedId);
     }
     # if
     /*
      * Recalculate the average spotrating and update the amount
      * of unverified comments
      */
     SpotTiming::start(__CLASS__ . '::' . __FUNCTION__ . ':updateSpotRating()');
     $this->_spotDao->updateSpotRating($spotMsgIdRatingList);
     SpotTiming::stop(__CLASS__ . '::' . __FUNCTION__ . ':updateSpotRating()');
     SpotTiming::start(__CLASS__ . '::' . __FUNCTION__ . ':updateSpotCommentCount()');
     $this->_spotDao->updateSpotCommentCount($spotMsgIdList);
     SpotTiming::stop(__CLASS__ . '::' . __FUNCTION__ . ':updateSpotCommentCount()');
     return array('count' => count($hdrList), 'headercount' => count($hdrList), 'lastmsgid' => $lastProcessedId);
 }
 /**
  *
  * Execute a POST to the given url and return the body.
  * @param String $url
  * @param array $postdata
  * @return bool|mixed
  */
 protected function postAndDownloadNzb($url, array $postdata)
 {
     // Initialize download retrieval class
     $svcHttp = new Services_Providers_Http($this->_cacheDao);
     $svcHttp->setPostContent($postdata);
     $svcHttp->setMethod('POST');
     $result = $svcHttp->perform($url);
     // Check if any error occured
     if (!$result['successful']) {
         SpotDebug::msg(SpotDebug::DEBUG, __CLASS__ . '->postAndDownloadNzb(), not succesful=' . $result['errorstr']);
         return false;
     }
     # if
     // Load the body into simplexml.
     // If the xml is well formed this will result in true thus returning the xml.
     // Suppress errors if the string is not well formed, where testing here.
     if (@simplexml_load_string($result['data'])) {
         return $result['data'];
     } else {
         return false;
     }
     # else
 }
 function process($hdrList, $curArtNr, $increment, $timer)
 {
     $this->displayStatus("progress", $curArtNr . " till " . $increment);
     $signedCount = 0;
     $hdrsParsed = 0;
     $fullsRetrieved = 0;
     $invalidCount = 0;
     $msgCounter = 0;
     $modCount = 0;
     $headerInDbCount = 0;
     $skipCount = 0;
     $lastProcessedId = '';
     $lastProcessedArtNr = 0;
     $fullSpotDbList = array();
     $spotDbList = array();
     $moderationList = array();
     $processingStartTime = time();
     /*
      * Determine the cutoff date (unixtimestamp) from whereon we do not want to 
      * load the spots
      */
     if ($this->_settings->get('retention') > 0) {
         $retentionStamp = time() - $this->_settings->get('retention') * 24 * 60 * 60;
     } else {
         $retentionStamp = 0;
     }
     # else
     SpotDebug::msg(SpotDebug::DEBUG, 'retentionStamp=' . $retentionStamp);
     SpotDebug::msg(SpotDebug::TRACE, 'hdrList=' . serialize($hdrList));
     /**
      * We ask the database to match our messageid's we just retrieved with
      * the list of id's we have just retrieved from the server
      */
     SpotTiming::start(__CLASS__ . '::' . __FUNCTION__ . ':matchSpotMessageIds');
     $dbIdList = $this->_spotDao->matchSpotMessageIds($hdrList);
     SpotTiming::stop(__CLASS__ . '::' . __FUNCTION__ . ':matchSpotMessageIds');
     SpotTiming::start(__CLASS__ . '::' . __FUNCTION__ . ':getMassCacheRecords');
     $cachedIdList = $this->_cacheDao->getMassCacheRecords($hdrList);
     SpotTiming::stop(__CLASS__ . '::' . __FUNCTION__ . ':getMassCacheRecords');
     SpotDebug::msg(SpotDebug::TRACE, 'dbIdList=' . serialize($dbIdList));
     /*
      * We get a list of spots which have been blacklisted before,
      * we do this because when the 'buggy' flag is set, we else keep
      * retrieving the same spots, nzb's and images over and over again
      */
     $preModdedList = $this->_modListDao->matchAgainst($hdrList);
     SpotTiming::start(__CLASS__ . '::' . __FUNCTION__ . ':forEach');
     foreach ($hdrList as $msgheader) {
         SpotTiming::start(__CLASS__ . '::' . __FUNCTION__ . ':forEach-to-ParseHeader');
         $msgCounter++;
         SpotDebug::msg(SpotDebug::DEBUG, 'foreach-loop, start. msgId= ' . $msgCounter);
         /* 
          * Keep te usenet server alive when processing is slow.
          */
         if ($processingStartTime - time() > 30) {
             $this->_svcNntpText->sendNoop();
             $this->_svcNntpBin->sendNoop();
             $processingStartTime = time();
         }
         # if
         /*
          * We keep track whether we actually fetched this header and fullspot
          * to add it to the database, because only then we can update the
          * title from the spots title or rely on our database to fetch
          * the fullspot
          */
         $didFetchHeader = false;
         $didFetchFullSpot = false;
         # Reset timelimit
         set_time_limit(120);
         # messageid to check
         $msgId = $msgheader['Message-ID'];
         $artNr = $msgheader['Number'];
         /*
          * If this message was already deleted in a previous run,
          * les not even consider it
          */
         if (isset($preModdedList[$msgId])) {
             SpotTiming::stop(__CLASS__ . '::' . __FUNCTION__ . ':forEach-to-ParseHeader');
             $skipCount++;
             continue;
         }
         # if
         /*
          * We prepare some variables to we don't have to perform an array
          * lookup for each check and the code is easier to read.
          */
         $header_isInDb = isset($dbIdList['spot'][$msgId]);
         $fullspot_isInDb = isset($dbIdList['fullspot'][$msgId]);
         /*
          * If the spotheader is not yet added to the database, parse the header
          * information.
          *
          * If the header is present, but we don't have the fullspot yet or we are
          * running in 'retro' mode, parse the header as well because some fields
          * are only in the header and not in the full.
          * 
          * We need some of those fields (for example KeyID)
          */
         if (!$header_isInDb || (!$fullspot_isInDb || $this->_retro) && $this->_retrieveFull) {
             $hdrsParsed++;
             SpotDebug::msg(SpotDebug::TRACE, 'foreach-loop, parsingXover, start. msgId= ' . $msgCounter);
             SpotTiming::start(__CLASS__ . '::' . __FUNCTION__ . ':parseHeader');
             $spot = $this->_svcSpotParser->parseHeader($msgheader['Subject'], $msgheader['From'], $msgheader['Date'], $msgheader['Message-ID'], $this->_rsakeys);
             SpotTiming::stop(__CLASS__ . '::' . __FUNCTION__ . ':parseHeader');
             SpotDebug::msg(SpotDebug::TRACE, 'foreach-loop, parsingXover, done. msgId= ' . $msgCounter);
             /*
              * When a parse error occurred, we ignore the spot, also unverified
              * spots are ignored
              */
             if ($spot === false || !$spot['verified']) {
                 SpotTiming::stop(__CLASS__ . '::' . __FUNCTION__ . ':forEach-to-ParseHeader');
                 $invalidCount++;
                 continue;
             }
             # if
             /*
              * Special moderator commands always have keyid 2
              */
             if ($spot['keyid'] == 2) {
                 $commandAr = explode(' ', $spot['title']);
                 $validCommands = array('delete', 'dispose', 'remove');
                 # is this one of the defined valid commands?
                 if (in_array(strtolower($commandAr[0]), $validCommands) !== false) {
                     $moderationList[$commandAr[1]] = 1;
                     $modCount++;
                 }
                 # if
             } else {
                 /*
                  * Don't add spots older than specified for the retention stamp
                  */
                 if ($retentionStamp > 0 && $spot['stamp'] < $retentionStamp && $this->_settings->get('retentiontype') == 'everything') {
                     SpotTiming::stop(__CLASS__ . '::' . __FUNCTION__ . ':forEach-to-ParseHeader');
                     $skipCount++;
                     continue;
                 } elseif ($spot['stamp'] < $this->_settings->get('retrieve_newer_than')) {
                     $skipCount++;
                 } else {
                     /*
                      * Do we have the header in the database? If not, lets add it
                      */
                     if (!$header_isInDb) {
                         $spotDbList[] = $spot;
                         /*
                          * Some buggy NNTP servers give us the same messageid
                          * in one XOVER statement, hence we update the list of
                          * messageid's we already have retrieved and are ready
                          * to be added to the database
                          */
                         $dbIdList['spot'][$msgId] = 1;
                         $header_isInDb = true;
                         $lastProcessedId = $msgId;
                         $lastProcessedArtNr = $artNr;
                         $didFetchHeader = true;
                         if ($spot['wassigned']) {
                             $signedCount++;
                         }
                         # if
                     }
                     # if
                 }
                 # if
             }
             # else
         } else {
             $lastProcessedId = $msgId;
             $lastProcessedArtNr = $artNr;
             $headerInDbCount++;
         }
         # else
         SpotTiming::start(__CLASS__ . '::' . __FUNCTION__ . ':forEach-getFullSpot');
         /*
          * We don't want to retrieve the fullspot if we don't have the header
          * in the database. Because we try to add headers in the above code we just have
          * to check if the header is in the database.
          *
          * We cannot collapse this code with the header fetching code because we want to
          * be able to add the fullspot to a system after all the headers are retrieved
          */
         if ($header_isInDb && !$fullspot_isInDb) {
             /*
              * Don't add older fullspots than specified for the retention stamp
              */
             if ($retentionStamp > 0 && strtotime($msgheader['Date']) < $retentionStamp) {
                 SpotTiming::stop(__CLASS__ . '::' . __FUNCTION__ . ':forEach-to-ParseHeader');
                 SpotTiming::stop(__CLASS__ . '::' . __FUNCTION__ . ':forEach-getFullSpot');
                 continue;
             }
             # if
             if ($this->_retrieveFull) {
                 $fullSpot = array();
                 try {
                     $fullsRetrieved++;
                     SpotDebug::msg(SpotDebug::TRACE, 'foreach-loop, getFullSpot, start. msgId= ' . $msgId);
                     $fullSpot = $this->_svcNntpTextReading->readFullSpot($msgId);
                     SpotDebug::msg(SpotDebug::TRACE, 'foreach-loop, getFullSpot, done. msgId= ' . $msgId);
                     # did we fail to parse the spot? if so, skip this one
                     if (empty($fullSpot)) {
                         $invalidCount++;
                         continue;
                     }
                     // if
                     # add this spot to the database
                     $fullSpotDbList[] = $fullSpot;
                     $fullspot_isInDb = true;
                     $didFetchFullSpot = true;
                     /*
                      * Some buggy NNTP servers give us the same messageid
                      * in the same XOVER statement, hence we update the list of
                      * messageid's we already have retrieved and are ready
                      * to be added to the database
                      */
                     $dbIdList['fullspot'][$msgId] = 1;
                     /*
                      * Overwrite the spots' title because the fullspot contains the title in
                      * UTF-8 format.
                      * We also overwrite the spotterid from the spotsfull because the spotterid
                      * is only in the header in more recent spots.
                      */
                     if ($didFetchHeader) {
                         $spotDbList[count($spotDbList) - 1]['title'] = $fullSpot['title'];
                         $spotDbList[count($spotDbList) - 1]['spotterid'] = $fullSpot['spotterid'];
                     }
                     # if
                 } catch (ParseSpotXmlException $x) {
                     # swallow error
                 } catch (Exception $x) {
                     /**
                      * Sometimes we get an 'No such article' error for a header we just retrieved,
                      * if we want to retrieve the full article. This is messed up, but let's just
                      * swallow the error
                      */
                     if ($x->getCode() == 430) {
                         /*
                          * Reset error count, so other errors are actually re-tried
                          */
                         $this->_svcNntpText->resetErrorCount();
                         $this->_svcNntpBin->resetErrorCount();
                     } elseif ($x->getMessage() == 'String could not be parsed as XML') {
                     } else {
                         throw $x;
                     }
                     # else
                 }
                 # catch
             }
             # if retrievefull
         }
         # if fullspot is not in db yet
         SpotTiming::stop(__CLASS__ . '::' . __FUNCTION__ . ':forEach-getFullSpot');
         SpotTiming::start(__CLASS__ . '::' . __FUNCTION__ . ':forEach-getNzbOrImage');
         /*
          * If both the image and the NZB file are already in the cache,
          * or we are set to not prefetch them, don't bother to retrieve
          * the full spot either from the database
          */
         $needPrefetch = $this->_prefetch_image || $this->_prefetch_nzb;
         if (!$this->_retrieveFull || !$header_isInDb) {
             $needPrefetch = false;
         }
         # if
         if ($needPrefetch) {
             $needPrefetch = !isset($cachedIdList[Dao_Cache::SpotImage][$msgId]) || !isset($cachedIdList[Dao_Cache::SpotNzb][$msgId]);
         }
         # if
         if ($needPrefetch) {
             try {
                 /*
                  * If we are running in 'retro' mode, it is possible both the header and spot are in the
                  * database already, however -- we need the information from the fullspot so we retrieve it
                  * again
                  */
                 if (!$didFetchFullSpot) {
                     SpotTiming::start(__CLASS__ . '::' . __FUNCTION__ . ':daoGetFullSpot');
                     $fullSpot = $this->_spotDao->getFullSpot($msgId, SPOTWEB_ANONYMOUS_USERID);
                     SpotTiming::start(__CLASS__ . '::' . __FUNCTION__ . ':retrieveParseFullSpot');
                     $fullSpot = array_merge($this->_svcSpotParser->parseFull($fullSpot['fullxml']), $fullSpot);
                     SpotTiming::stop(__CLASS__ . '::' . __FUNCTION__ . ':retrieveParseFullSpot', array());
                     SpotTiming::stop(__CLASS__ . '::' . __FUNCTION__ . ':daoGetFullSpot');
                 }
                 # if
                 /*
                  * Prefetch (cache) the spots' image
                  */
                 SpotTiming::start(__CLASS__ . '::' . __FUNCTION__ . ':forEach-getImage');
                 if ($this->_prefetch_image) {
                     SpotDebug::msg(SpotDebug::TRACE, 'foreach-loop, getImage(), start. msgId= ' . $msgId);
                     if (!isset($cachedIdList[Dao_Cache::SpotImage][$fullSpot['messageid']])) {
                         $this->_svcProvImage->fetchSpotImage($fullSpot);
                     }
                     # if
                     SpotDebug::msg(SpotDebug::TRACE, 'foreach-loop, getImage(), done. msgId= ' . $msgId);
                 }
                 # if
                 SpotTiming::stop(__CLASS__ . '::' . __FUNCTION__ . ':forEach-getImage');
                 SpotTiming::start(__CLASS__ . '::' . __FUNCTION__ . ':forEach-getNzb');
                 /*
                  * Prefetch (cache) the spots' NZB file
                  */
                 if ($this->_prefetch_nzb) {
                     /*
                      * Only do so if we can expect an NZB file
                      */
                     if (!empty($fullSpot['nzb']) && $fullSpot['stamp'] > 1290578400) {
                         SpotDebug::msg(SpotDebug::TRACE, 'foreach-loop, getNzb(), start. msgId= ' . $msgId);
                         if (!isset($cachedIdList[Dao_Cache::SpotNzb][$fullSpot['messageid']])) {
                             $this->_svcProvNzb->fetchNzb($fullSpot);
                         }
                         # if
                         SpotDebug::msg(SpotDebug::TRACE, 'foreach-loop, getNzb(), done. msgId= ' . $msgId);
                     }
                     # if
                 }
                 # if
                 SpotTiming::stop(__CLASS__ . '::' . __FUNCTION__ . ':forEach-getNzb');
             } catch (ParseSpotXmlException $x) {
                 # swallow error
             } catch (Exception $x) {
                 SpotTiming::stop(__CLASS__ . '::' . __FUNCTION__ . ':retrieveParseFullSpot', array());
                 SpotTiming::stop(__CLASS__ . '::' . __FUNCTION__ . ':daoGetFullSpot');
                 /**
                  * Sometimes we get an 'No such article' error for a header we just retrieved,
                  * if we want to retrieve the full article. This is messed up, but let's just
                  * swallow the error
                  */
                 if ($x->getCode() == 430) {
                     /*
                      * Reset error count, so other errors are actually re-tried
                      */
                     $this->_svcNntpText->resetErrorCount();
                     $this->_svcNntpBin->resetErrorCount();
                 } elseif ($x->getMessage() == 'String could not be parsed as XML') {
                 } else {
                     throw $x;
                 }
                 # else
             }
             # catch
         }
         # if prefetch image and/or nzb
         SpotTiming::stop(__CLASS__ . '::' . __FUNCTION__ . ':forEach-getNzbOrImage');
         SpotTiming::stop(__CLASS__ . '::' . __FUNCTION__ . ':forEach-to-ParseHeader');
         /*
          * If we are under memory pressure, flush the cache to disk in advance so we
          * can free up memory. This is slower, but might avoid ballooning memory.
          */
         if ($this->hasMemoryPressure()) {
             SpotDebug::msg(SpotDebug::DEBUG, 'we are under memory pressure, flushing to disk');
             echo "We are under memory pressure... ";
             $this->_spotDao->addSpots($spotDbList, $fullSpotDbList);
             $spotDbList = array();
             $fullSpotDbList = array();
         }
         // if
         SpotDebug::msg(SpotDebug::DEBUG, 'foreach-loop, done. msgId= ' . $msgCounter);
     }
     # foreach
     SpotTiming::stop(__CLASS__ . '::' . __FUNCTION__ . ':forEach');
     if (count($hdrList) > 0) {
         $this->displayStatus("hdrparsed", $hdrsParsed);
         $this->displayStatus("hdrindbcount", $headerInDbCount);
         $this->displayStatus("verified", $signedCount);
         $this->displayStatus("invalidcount", $invalidCount);
         $this->displayStatus("skipcount", $skipCount);
         $this->displayStatus("modcount", $modCount);
         $this->displayStatus("fullretrieved", $fullsRetrieved);
         $this->displayStatus("loopcount", count($hdrList));
     } else {
         $this->displayStatus("hdrparsed", 0);
         $this->displayStatus("hdrindbcount", 0);
         $this->displayStatus("verified", 0);
         $this->displayStatus("invalidcount", 0);
         $this->displayStatus("skipcount", 0);
         $this->displayStatus("modcount", 0);
         $this->displayStatus("fullretrieved", 0);
         $this->displayStatus("loopcount", 0);
     }
     # else
     /* 
      * Add the spots to the database and update the last article
      * number found
      */
     $this->_spotDao->addSpots($spotDbList, $fullSpotDbList);
     SpotDebug::msg(SpotDebug::TRACE, 'added Spots, spotDbList=' . serialize($spotDbList));
     SpotDebug::msg(SpotDebug::TRACE, 'added Spots, fullSpotDbList=' . serialize($fullSpotDbList));
     /*
      * Actually act on the moderation settings. We cannot process this inline
      * because a spot can be added and moderated within the same iteration
      */
     switch ($this->_settings->get('spot_moderation')) {
         case 'disable':
             break;
         case 'markspot':
             $this->_commentDao->markCommentsModerated($moderationList);
             $this->_spotDao->markSpotsModerated($moderationList);
             break;
             # case 'markspot'
         # case 'markspot'
         default:
             $this->_spotDao->removeSpots($moderationList);
             $this->_commentDao->removeComments($moderationList);
             /*
              * If the spots actually get removed, we want to make
              * sure we write the deleted spots down. This prevents
              * us from retrieving and deleting them over and over again
              */
             $this->_modListDao->addToRingBuffer($moderationList);
             break;
             # default
     }
     # switch
     # update the maximum article id
     if (!empty($lastProcessedId) && $lastProcessedArtNr > 0) {
         $this->_usenetStateDao->setMaxArticleId(Dao_UsenetState::State_Spots, $lastProcessedArtNr, $lastProcessedId);
     }
     # if
     SpotDebug::msg(SpotDebug::DEBUG, 'loop finished, setMaxArticleId=' . serialize($increment));
     /*
      * And remove old list of moderated spots
      */
     $this->_modListDao->deleteOldest();
     $this->displayStatus("timer", round(microtime(true) - $timer, 2));
     return array('count' => count($hdrList), 'headercount' => $hdrsParsed, 'lastmsgid' => $lastProcessedId);
 }
Example #6
0
 public function validateServer()
 {
     SpotDebug::msg(SpotDebug::TRACE, __CLASS__ . "->validateServer()");
     /*
      * We need to select a group, because authentication
      * is not always entered but sometimes required
      */
     $this->selectGroup('free.pt');
     $this->quit();
 }
 function loopTillEnd($curArticleNr, $increment = 1000)
 {
     $processed = 0;
     $headersProcessed = 0;
     $highestMessageId = '';
     # make sure we handle articlenumber wrap arounds
     if ($curArticleNr < $this->_msgdata['first']) {
         $curArticleNr = $this->_msgdata['first'];
     }
     # if
     $this->displayStatus("groupmessagecount", $this->_msgdata['last'] - $this->_msgdata['first']);
     $this->displayStatus("firstmsg", $this->_msgdata['first']);
     $this->displayStatus("lastmsg", $this->_msgdata['last']);
     $this->displayStatus("curartnr", $curArticleNr);
     $this->displayStatus("", "");
     SpotTiming::start(__CLASS__ . '::' . __FUNCTION__ . ':whileLoop');
     while ($curArticleNr < $this->_msgdata['last']) {
         $timer = microtime(true);
         # get the list of headers (XOVER)
         SpotTiming::start(__CLASS__ . '::' . __FUNCTION__ . ':getOverview');
         $hdrList = $this->_svcNntpText->getOverview($curArticleNr, $curArticleNr + $increment);
         SpotTiming::stop(__CLASS__ . '::' . __FUNCTION__ . ':getOverview');
         $saveCurArtNr = $curArticleNr;
         # If no spots were found, just manually increase the
         # messagenumber with the increment to make sure we advance
         if (count($hdrList) < 1 || $hdrList[count($hdrList) - 1]['Number'] < $curArticleNr) {
             $curArticleNr += $increment;
         } else {
             $curArticleNr = $hdrList[count($hdrList) - 1]['Number'] + 1;
         }
         # else
         # run the processing method
         SpotTiming::start(__CLASS__ . '::' . __FUNCTION__ . ':callProcess');
         $processOutput = $this->process($hdrList, $saveCurArtNr, $curArticleNr, $timer);
         SpotTiming::stop(__CLASS__ . '::' . __FUNCTION__ . ':callProcess');
         $processed += $processOutput['count'];
         $headersProcessed += $processOutput['headercount'];
         $highestMessageId = $processOutput['lastmsgid'];
         # reset the start time to prevent a another retriever from starting
         # during the intial retrieve which can take many hours
         $this->_usenetStateDao->setRetrieverRunning(true);
         /*
          * Make sure if we run with timing on, we do not fetch too many
          * spots as that would make us run out of memory
          */
         if ($processed > 3000 && SpotTiming::isEnabled()) {
             break;
         }
         # if
     }
     # while
     SpotTiming::stop(__CLASS__ . '::' . __FUNCTION__ . ':whileLoop');
     # we are done updating, make sure that if the newsserver deleted
     # earlier retrieved messages, we remove them from our database
     if ($highestMessageId != '') {
         SpotDebug::msg(SpotDebug::DEBUG, 'loopTillEnd() finished, highestMessageId = ' . $highestMessageId);
         $this->removeTooNewRecords($highestMessageId);
     }
     # if
     $this->displayStatus("totalprocessed", $processed);
     return $headersProcessed;
 }