コード例 #1
0
ファイル: SpotRetriever_Spots.php プロジェクト: niel/spotweb
 function process($hdrList, $curMsg, $endMsg, $timer)
 {
     $this->displayStatus("progress", $curMsg . " till " . $endMsg);
     $spotParser = new SpotParser();
     $signedCount = 0;
     $hdrsRetrieved = 0;
     $fullsRetrieved = 0;
     $msgCounter = 0;
     $modCount = 0;
     $skipCount = 0;
     $lastProcessedId = '';
     $fullSpotDbList = array();
     $spotDbList = array();
     $moderationList = array();
     $processingStartTime = time();
     /*
      * Determine the cutoff date (unixtimestamp) from whereon we do not want to 
      * load the spots
      */
     if ($this->_settings->get('retention') > 0) {
         $retentionStamp = time() - $this->_settings->get('retention') * 24 * 60 * 60;
     } else {
         $retentionStamp = 0;
     }
     # else
     $this->debug('retentionStamp=' . $retentionStamp);
     $this->debug('hdrList=' . serialize($hdrList));
     /**
      * We ask the database to match our messageid's we just retrieved with
      * the list of id's we have just retrieved from the server
      */
     $dbIdList = $this->_db->matchSpotMessageIds($hdrList);
     $this->debug('dbIdList=' . serialize($dbIdList));
     # if we need to fetch images or nzb files, we need an spotsoverview instance
     if ($this->_retrieveFull && $this->_prefetch_image || $this->_prefetch_nzb) {
         $spotsOverview = new SpotsOverview($this->_db, $this->_settings);
         $spotsOverview->setActiveRetriever(true);
         /*
          * Only create a new NZB instance if the server differs from the
          * header host, else re-use the connection
          */
         $settings_nntp_nzb = $this->_settings->get('nntp_nzb');
         if ($this->_server['host'] == $settings_nntp_nzb['host']) {
             $nntp_nzb = $this->_spotnntp;
         } else {
             $nntp_nzb = new SpotNntp($settings_nntp_nzb);
             $nntp_nzb->selectGroup($this->_settings->get('nzb_group'));
         }
         # else
     }
     # if
     foreach ($hdrList as $msgheader) {
         $msgCounter++;
         $this->debug('foreach-loop, start. msgId= ' . $msgCounter);
         /* 
          * Keep te usenet server alive when processing is slow.
          */
         if ($processingStartTime - time() > 30) {
             $this->_spotnntp->sendNoop();
             if (isset($nntp_nzb) && $nntp_nzb != $this->_spotnntp) {
                 $nntp_nzb->sendNoop();
             }
             # if
             $processingStartTime = time();
         }
         # if
         /*
          * We keep track whether we actually fetched this header and fullspot
          * to add it to the database, because only then we can update the
          * titel from the spots title or rely on our database to fetch
          * the fullspot
          */
         $didFetchHeader = false;
         $didFetchFullSpot = false;
         # Reset timelimit
         set_time_limit(120);
         # messageid to check
         $msgId = substr($msgheader['Message-ID'], 1, -1);
         /*
          * We prepare some variables to we don't have to perform an array
          * lookup for each check and the code is easier to read.
          */
         $header_isInDb = isset($dbIdList['spot'][$msgId]);
         $fullspot_isInDb = isset($dbIdList['fullspot'][$msgId]);
         /*
          * If the spotheader is not yet added to the database, parse the header
          * information.
          *
          * If the header is present, but we don't have the fullspot yet or we are
          * running in 'retro' mode, parse the header as well because some fields
          * are only in the header and not in the full.
          * 
          * We need some of those fields (for example KeyID)
          */
         if (!$header_isInDb || (!$fullspot_isInDb || $this->_retro) && $this->_retrieveFull) {
             $hdrsRetrieved++;
             $this->debug('foreach-loop, parsingXover, start. msgId= ' . $msgCounter);
             $spot = $spotParser->parseXover($msgheader['Subject'], $msgheader['From'], $msgheader['Date'], $msgheader['Message-ID'], $this->_rsakeys);
             $this->debug('foreach-loop, parsingXover, done. msgId= ' . $msgCounter);
             /*
              * When a parse error occured, we ignore the spot, also unverified
              * spots are ignored
              */
             if ($spot === false || !$spot['verified']) {
                 $this->debug('foreach-loop, spot is either false or not verified');
                 continue;
             }
             # if
             /*
              * Special moderator commands always have keyid 2
              */
             if ($spot['keyid'] == 2) {
                 $this->debug('foreach-loop, spot is a moderation spot');
                 $commandAr = explode(' ', strtolower($spot['title']));
                 $validCommands = array('delete', 'dispose', 'remove');
                 # is this one of the defined valid commands?
                 if (in_array($commandAr[0], $validCommands) !== false) {
                     $moderationList[$commandAr[1]] = 1;
                     $modCount++;
                 }
                 # if
             } else {
                 /*
                  * Don't add spots older than specified for the retention stamp
                  */
                 if ($retentionStamp > 0 && $spot['stamp'] < $retentionStamp && $this->_settings->get('retentiontype') == 'everything') {
                     $this->debug('foreach-loop, spot is expired: ' . $spot['stamp']);
                     continue;
                 } elseif ($spot['stamp'] < $this->_settings->get('retrieve_newer_than')) {
                     $this->debug('foreach-loop, spot is too old: ' . $spot['stamp']);
                     $skipCount++;
                 } else {
                     /*
                      * Do we have the header in the database? If not, lets add it
                      */
                     if (!$header_isInDb) {
                         $spotDbList[] = $spot;
                         /*
                          * Some buggy NNTP servers give us the same messageid
                          * in one XOVER statement, hence we update the list of
                          * messageid's we already have retrieved and are ready
                          * to be added to the database
                          */
                         $dbIdList['spot'][$msgId] = 1;
                         $header_isInDb = true;
                         $lastProcessedId = $msgId;
                         $didFetchHeader = true;
                         if ($spot['wassigned']) {
                             $signedCount++;
                         }
                         # if
                     }
                     # if
                 }
                 # if
             }
             # else
         } else {
             $lastProcessedId = $msgId;
         }
         # else
         /*
          * We don't want to retrieve the fullspot if we don't have the header
          * in the database. Because we try to add headers in the above code we just have
          * to check if the header is in the database.
          *
          * We cannot collapse this code with the header fetching code because we want to
          * be able to add the fullspot to a system after all the headers are retrieved
          */
         if ($header_isInDb && !$fullspot_isInDb) {
             /*
              * Don't add older fullspots than specified for the retention stamp
              */
             if ($retentionStamp > 0 && strtotime($msgheader['Date']) < $retentionStamp) {
                 continue;
             }
             # if
             if ($this->_retrieveFull) {
                 $fullSpot = array();
                 try {
                     $fullsRetrieved++;
                     $this->debug('foreach-loop, getFullSpot, start. msgId= ' . $msgId);
                     $fullSpot = $this->_spotnntp->getFullSpot($msgId);
                     $this->debug('foreach-loop, getFullSpot, done. msgId= ' . $msgId);
                     # add this spot to the database
                     $fullSpotDbList[] = $fullSpot;
                     $fullspot_isInDb = true;
                     $didFetchFullSpot = true;
                     /*
                      * Some buggy NNTP servers give us the same messageid
                      * in once XOVER statement, hence we update the list of
                      * messageid's we already have retrieved and are ready
                      * to be added to the database
                      */
                     $dbIdList['fullspot'][$msgId] = 1;
                     /* 
                      * Overwrite the spots' title because the fullspot contains the title in
                      * UTF-8 format.
                      * We also overwrite the spotterid from the spotsfull because the spotterid
                      * is only in the header in more recent spots.
                      */
                     if ($didFetchHeader) {
                         $spotDbList[count($spotDbList) - 1]['title'] = $fullSpot['title'];
                         $spotDbList[count($spotDbList) - 1]['spotterid'] = $fullSpot['spotterid'];
                     }
                     # if
                 } catch (ParseSpotXmlException $x) {
                     # swallow error
                 } catch (Exception $x) {
                     /**
                      * Sometimes we get an 'No such article' error for a header we just retrieved,
                      * if we want to retrieve the full article. This is messed up, but let's just
                      * swallow the error
                      */
                     if ($x->getCode() == 430) {
                     } elseif ($x->getMessage() == 'String could not be parsed as XML') {
                     } else {
                         throw $x;
                     }
                     # else
                 }
                 # catch
             }
             # if retrievefull
         }
         # if fullspot is not in db yet
         if ($this->_retrieveFull && $header_isInDb && ($this->_prefetch_image || $this->_prefetch_nzb)) {
             try {
                 /*
                  * If we are running in 'retro' mode, it is possible both the header and spot are in the
                  * database already, however -- we need the information from the fullspot so we retrieve it
                  * again
                  */
                 if (!$didFetchFullSpot) {
                     $fullSpot = $this->_db->getFullSpot($msgId, SPOTWEB_ANONYMOUS_USERID);
                     $fullSpot = array_merge($spotParser->parseFull($fullSpot['fullxml']), $fullSpot);
                 }
                 # if
                 /*
                  * Prefetch (cache) the spots' image
                  */
                 if ($this->_prefetch_image) {
                     /*
                      * If the spot is older than 30 days, and the image is on the web, we do not 
                      * prefetch the image.
                      */
                     if (is_array($fullSpot['image']) || $fullSpot['stamp'] > (int) time() - 30 * 24 * 60 * 60) {
                         $this->debug('foreach-loop, getImage(), start. msgId= ' . $msgId);
                         $spotsOverview->getImage($fullSpot, $nntp_nzb);
                         $this->debug('foreach-loop, getImage(), done. msgId= ' . $msgId);
                     }
                     # if
                 }
                 # if
                 /*
                  * Prefetch (cache) the spots' NZB file
                  */
                 if ($this->_prefetch_nzb) {
                     /*
                      * Only do so if we can expect an NZB file
                      */
                     if (!empty($fullSpot['nzb']) && $fullSpot['stamp'] > 1290578400) {
                         $this->debug('foreach-loop, getNzb(), start. msgId= ' . $msgId);
                         $spotsOverview->getNzb($fullSpot, $nntp_nzb);
                         $this->debug('foreach-loop, getNzb(), done. msgId= ' . $msgId);
                     }
                     # if
                 }
                 # if
             } catch (ParseSpotXmlException $x) {
                 # swallow error
             } catch (Exception $x) {
                 /**
                  * Sometimes we get an 'No such article' error for a header we just retrieved,
                  * if we want to retrieve the full article. This is messed up, but let's just
                  * swallow the error
                  */
                 if ($x->getCode() == 430) {
                 } elseif ($x->getMessage() == 'String could not be parsed as XML') {
                 } else {
                     throw $x;
                 }
                 # else
             }
             # catch
         }
         # if prefetch image and/or nzb
         $this->debug('foreach-loop, done. msgId= ' . $msgCounter);
     }
     # foreach
     if (count($hdrList) > 0) {
         $this->displayStatus("hdrparsed", $hdrsRetrieved);
         $this->displayStatus("fullretrieved", $fullsRetrieved);
         $this->displayStatus("verified", $signedCount);
         $this->displayStatus("modcount", $modCount);
         $this->displayStatus("skipcount", $skipCount);
         $this->displayStatus("loopcount", count($hdrList));
     } else {
         $this->displayStatus("hdrparsed", 0);
         $this->displayStatus("fullretrieved", 0);
         $this->displayStatus("verified", 0);
         $this->displayStatus("modcount", 0);
         $this->displayStatus("skipcount", 0);
         $this->displayStatus("loopcount", 0);
     }
     # else
     /* 
      * Add the spots to the database and update the last article
      * number found
      */
     $this->_db->addSpots($spotDbList, $fullSpotDbList);
     $this->debug('added Spots, spotDbList=' . serialize($spotDbList));
     $this->debug('added Spots, fullSpotDbList=' . serialize($fullSpotDbList));
     /*
      * Actually act on the moderation settings. We cannot process this inline
      * because a spot can be added and moderated within the same iteration
      */
     switch ($this->_settings->get('spot_moderation')) {
         case 'disable':
             break;
         case 'markspot':
             $this->_db->markCommentsModerated($moderationList);
             $this->_db->markSpotsModerated($moderationList);
             break;
             # case 'markspot'
         # case 'markspot'
         default:
             $this->_db->removeSpots($moderationList);
             $this->_db->removeComments($moderationList);
             break;
             # default
     }
     # switch
     # update the maximum article id
     if ($this->_retro) {
         $this->_db->setMaxArticleid('spots_retro', $endMsg);
     } else {
         $this->_db->setMaxArticleid($this->_server['host'], $endMsg);
     }
     # if
     $this->debug('loop finished, setMaxArticleId=' . serialize($endMsg));
     $this->displayStatus("timer", round(microtime(true) - $timer, 2));
     return array('count' => count($hdrList), 'headercount' => $hdrsRetrieved, 'lastmsgid' => $lastProcessedId);
 }
コード例 #2
0
ファイル: SpotPage_getimage.php プロジェクト: niel/spotweb
 function render()
 {
     $settings_nntp_hdr = $this->_settings->get('nntp_hdr');
     $settings_nntp_nzb = $this->_settings->get('nntp_nzb');
     # Check users' permissions
     $this->_spotSec->fatalPermCheck(SpotSecurity::spotsec_view_spotimage, '');
     # Haal de image op
     if (isset($this->_image['type']) && $this->_image['type'] == 'speeddial') {
         /*
          * Because the speeddial image shows stuff like last update and amount of new spots,
          * we want to make sure this is not a totally closed system
          */
         $this->_spotSec->fatalPermCheck(SpotSecurity::spotsec_view_spots_index, '');
         # init
         $spotImage = new SpotImage($this->_db);
         $totalSpots = $this->_db->getSpotCount('');
         $newSpots = $this->_tplHelper->getNewCountForFilter('');
         $lastUpdate = $this->_tplHelper->formatDate($this->_db->getLastUpdate($settings_nntp_hdr['host']), 'lastupdate');
         $data = $spotImage->createSpeedDial($totalSpots, $newSpots, $lastUpdate);
     } elseif (isset($this->_image['type']) && $this->_image['type'] == 'statistics') {
         $this->_spotSec->fatalPermCheck(SpotSecurity::spotsec_view_statistics, '');
         # init
         $spotsOverview = new SpotsOverview($this->_db, $this->_settings);
         $graph = isset($this->_image['graph']) ? $this->_image['graph'] : false;
         $limit = isset($this->_image['limit']) ? $this->_image['limit'] : false;
         $data = $spotsOverview->getStatisticsImage($graph, $limit, $settings_nntp_hdr, $this->_currentSession['user']['prefs']['user_language']);
     } elseif (isset($this->_image['type']) && $this->_image['type'] == 'avatar') {
         # Check users' permissions
         $this->_spotSec->fatalPermCheck(SpotSecurity::spotsec_view_spotimage, 'avatar');
         # init
         $spotsOverview = new SpotsOverview($this->_db, $this->_settings);
         $imgDefaults = array('md5' => false, 'size' => 80, 'default' => 'identicon', 'rating' => 'g');
         $imgSettings = array_merge($imgDefaults, $this->_image);
         if ($imgSettings['size'] < 1 || $imgSettings['size'] > 512) {
             $imgSettings['size'] = $imgDefaults['size'];
         }
         # if
         if (!in_array($imgSettings['default'], array('identicon', 'mm', 'monsterid', 'retro', 'wavatar'))) {
             $imgSettings['default'] = $imgDefaults['default'];
         }
         # if
         if (!in_array($imgSettings['rating'], array('g', 'pg', 'r', 'x'))) {
             $imgSettings['rating'] = $imgDefaults['rating'];
         }
         # if
         $data = $spotsOverview->getAvatarImage($imgSettings['md5'], $imgSettings['size'], $imgSettings['default'], $imgSettings['rating']);
     } else {
         # init
         $spotsOverview = new SpotsOverview($this->_db, $this->_settings);
         $hdr_spotnntp = new SpotNntp($settings_nntp_hdr);
         /* Als de HDR en de NZB host hetzelfde zijn, zet geen tweede verbinding op */
         if ($settings_nntp_hdr['host'] == $settings_nntp_nzb['host']) {
             $nzb_spotnntp = $hdr_spotnntp;
         } else {
             $nzb_spotnntp = new SpotNntp($this->_settings->get('nntp_nzb'));
         }
         # else
         # Haal de volledige spotinhoud op
         $fullSpot = $this->_tplHelper->getFullSpot($this->_messageid, false);
         $data = $spotsOverview->getImage($fullSpot, $nzb_spotnntp);
     }
     # else
     # Images mogen gecached worden op de client, behalve als is opgegeven dat het niet mag
     if (isset($data['expire'])) {
         $this->sendExpireHeaders(true);
     } else {
         $this->sendExpireHeaders(false);
     }
     # else
     header("Content-Type: " . image_type_to_mime_type($data['metadata']['imagetype']));
     header("Content-Length: " . strlen($data['content']));
     echo $data['content'];
 }