/** * Look for an .nfo file in the NZB, return the NFO message id. * Gets the NZB completion. * Looks for PAR2 files in the NZB. * * @param string $guid * @param string $relID * @param int $groupID * @param string $groupName * * @return bool * * @access public */ public function getNfoFromNZB($guid, $relID, $groupID, $groupName) { $fetchedBinary = false; $messageID = $this->parseNZB($guid, $relID, $groupID, true); if ($messageID !== false) { $fetchedBinary = $this->nntp->getMessages($groupName, $messageID['id'], $this->alternateNNTP); if ($this->nntp->isError($fetchedBinary)) { // NFO download failed, increment attempts. $this->pdo->queryExec(sprintf('UPDATE releases SET nfostatus = nfostatus - 1 WHERE id = %d', $relID)); if ($this->echooutput) { echo 'f'; } return false; } if ($this->nfo->isNFO($fetchedBinary, $guid) === true) { if ($this->echooutput) { echo $messageID['hidden'] === false ? '+' : '*'; } } else { if ($this->echooutput) { echo '-'; } $this->pdo->queryExec(sprintf('UPDATE releases SET nfostatus = %d WHERE id = %d', Nfo::NFO_NONFO, $relID)); $fetchedBinary = false; } } else { if ($this->echooutput) { echo '-'; } $this->pdo->queryExec(sprintf('UPDATE releases SET nfostatus = %d WHERE id = %d', Nfo::NFO_NONFO, $relID)); } return $fetchedBinary; }
/** * Fetch a comment and insert it. * * @param string $messageID Message-ID for the article. * @param string $siteID id of the site. * * @return bool */ protected function insertNewComment(&$messageID, &$siteID) { // Get the article body. $body = $this->nntp->getMessages(self::group, $messageID); // Check if there's an error. if ($this->nntp->isError($body)) { return false; } // Decompress the body. $body = @gzinflate($body); if ($body === false) { return false; } // JSON Decode the body. $body = json_decode($body, true); if ($body === false) { return false; } // Just in case. if (!isset($body['USER']) || !isset($body['SID']) || !isset($body['RID']) || !isset($body['TIME']) | !isset($body['BODY'])) { return false; } $cid = md5($body['SID'] . $body['USER'] . $body['TIME'] . $siteID); // Insert the comment. if ($this->pdo->queryExec(sprintf(' INSERT IGNORE INTO releasecomment (text, createddate, issynced, shareid, cid, gid, nzb_guid, siteid, username, userid, releaseid, shared, host, sourceID) VALUES (%s, %s, 1, %s, %s, %s, %s, %s, %s, 0, 0, 2, "", 999)', $this->pdo->escapeString($body['BODY']), $this->pdo->from_unixtime($body['TIME'] > time() ? time() : $body['TIME']), $this->pdo->escapeString($body['SID']), $this->pdo->escapeString($cid), $this->pdo->escapeString($body['RID']), $this->pdo->escapeString($body['RID']), $this->pdo->escapeString($siteID), $this->pdo->escapeString(substr($body['USER'], 0, 3) === 'sn-' ? 'SH_ANON' : 'SH_' . $body['USER'])))) { return true; } return false; }
/** * Attempt to get a better name from a par2 file and categorize the release. * * @note Called from NZBContents.php * * @param string $messageID MessageID from NZB file. * @param int $relID id of the release. * @param int $groupID Group id of the release. * @param \NNTP $nntp Class NNTP * @param int $show Only show result or apply iy. * * @return bool */ public function parsePAR2($messageID, $relID, $groupID, &$nntp, $show) { if ($messageID === '') { return false; } $query = $this->pdo->queryOneRow(sprintf(' SELECT id, groupid, categoryid, name, searchname, UNIX_TIMESTAMP(postdate) AS post_date, id AS releaseid FROM releases WHERE isrenamed = 0 AND id = %d', $relID)); if ($query === false) { return false; } // Only get a new name if the category is OTHER. $foundName = true; if (!in_array((int) $query['categoryid'], array(\Category::CAT_BOOK_OTHER, \Category::CAT_GAME_OTHER, \Category::CAT_MOVIE_OTHER, \Category::CAT_MUSIC_OTHER, \Category::CAT_PC_MOBILEOTHER, \Category::CAT_TV_OTHER, \Category::CAT_MISC_HASHED, \Category::CAT_XXX_OTHER, \Category::CAT_MISC_OTHER))) { $foundName = false; } // Get the PAR2 file. $par2 = $nntp->getMessages($this->groups->getByNameByID($groupID), $messageID, $this->alternateNNTP); if ($nntp->isError($par2)) { return false; } // Put the PAR2 into Par2Info, check if there's an error. $this->_par2Info->setData($par2); if ($this->_par2Info->error) { return false; } // Get the file list from Par2Info. $files = $this->_par2Info->getFileList(); if ($files !== false && count($files) > 0) { $filesAdded = 0; // Loop through the files. foreach ($files as $file) { if (!isset($file['name'])) { continue; } // If we found a name and added 10 files, stop. if ($foundName === true && $filesAdded > 10) { break; } if ($this->addpar2) { // Add to release files. if ($filesAdded < 11 && $this->pdo->queryOneRow(sprintf(' SELECT id FROM releasefiles WHERE releaseid = %d AND name = %s', $relID, $this->pdo->escapeString($file['name']))) === false) { // Try to add the files to the DB. if ($this->releaseFiles->add($relID, $file['name'], $file['size'], $query['post_date'], 0)) { $filesAdded++; } } } else { $filesAdded++; } // Try to get a new name. if ($foundName === false) { $query['textstring'] = $file['name']; if ($this->nameFixer->checkName($query, 1, 'PAR2, ', 1, $show) === true) { $foundName = true; } } } // If we found some files. if ($filesAdded > 0) { $this->debugging->log(get_class(), __FUNCTION__, 'Added ' . $filesAdded . ' releasefiles from PAR2 for ' . $query['searchname'], \Logger::LOG_INFO); // Update the file count with the new file count + old file count. $this->pdo->queryExec(sprintf(' UPDATE releases SET rarinnerfilecount = rarinnerfilecount + %d WHERE id = %d', $filesAdded, $relID)); } if ($foundName === true) { return true; } } return false; }
/** * Returns unix time for an article number. * * @param int $post The article number to get the time from. * @param array $groupData Usenet group info from NNTP selectGroup method. * * @return bool|int */ public function postdate($post, array $groupData) { // Set table names $groupID = $this->_groups->getIDByName($groupData['group']); $group = []; if ($groupID !== '') { $group = $this->_groups->getCBPTableNames($this->_tablePerGroup, $groupID); } $currentPost = $post; $attempts = $date = 0; do { // Try to get the article date locally first. if ($groupID !== '') { // Try to get locally. $local = $this->_pdo->queryOneRow(sprintf(' SELECT b.date AS date FROM %s b, %s p WHERE b.id = p.binaryid AND b.groupid = %s AND p.number = %s LIMIT 1', $group['bname'], $group['pname'], $groupID, $currentPost)); if ($local !== false) { $date = $local['date']; break; } } // If we could not find it locally, try usenet. $header = $this->_nntp->getXOVER($currentPost); if (!$this->_nntp->isError($header)) { // Check if the date is set. if (isset($header[0]['Date']) && strlen($header[0]['Date']) > 0) { $date = $header[0]['Date']; break; } } // Try to get a different article number. if (abs($currentPost - $groupData['first']) > abs($groupData['last'] - $currentPost)) { $tempPost = round($currentPost / (mt_rand(1005, 1012) / 1000), 0, PHP_ROUND_HALF_UP); if ($tempPost < $groupData['first']) { $tempPost = $groupData['first']; } } else { $tempPost = round(mt_rand(1005, 1012) / 1000 * $currentPost, 0, PHP_ROUND_HALF_UP); if ($tempPost > $groupData['last']) { $tempPost = $groupData['last']; } } // If we got the same article number as last time, give up. if ($tempPost === $currentPost) { break; } $currentPost = $tempPost; if ($this->_debug) { $this->_colorCLI->doEcho($this->_colorCLI->debug('Postdate retried ' . $attempts . " time(s).")); } } while ($attempts++ <= 20); // If we didn't get a date, set it to now. if (!$date) { $date = time(); } else { $date = strtotime($date); } if ($this->_debug) { $this->_debugging->log('Binaries', "postdate", 'Article (' . $post . "'s) date is (" . $date . ') (' . $this->daysOld($date) . " days old)", \Logger::LOG_INFO); } return $date; }
<?php require_once dirname(__FILE__) . '/../../../config.php'; use nzedb\db\Settings; $start = TIME(); $pdo = new Settings(); $consoleTools = new ConsoleTools(['ColorCLI' => $pdo->log]); // Create the connection here and pass $nntp = new NNTP(['Settings' => $pdo]); if ($nntp->doConnect() !== true) { exit($pdo->log->error("Unable to connect to usenet.")); } echo $pdo->log->header("Getting first/last for all your active groups."); $data = $nntp->getGroups(); if ($nntp->isError($data)) { exit($pdo->log->error("Failed to getGroups() from nntp server.")); } echo $pdo->log->header("Inserting new values into shortgroups table."); $pdo->queryExec('TRUNCATE TABLE shortgroups'); // Put into an array all active groups $res = $pdo->query('SELECT name FROM groups WHERE active = 1 OR backfill = 1'); foreach ($data as $newgroup) { if (myInArray($res, $newgroup['group'], 'name')) { $pdo->queryInsert(sprintf('INSERT INTO shortgroups (name, first_record, last_record, updated) VALUES (%s, %s, %s, NOW())', $pdo->escapeString($newgroup['group']), $pdo->escapeString($newgroup['first']), $pdo->escapeString($newgroup['last']))); echo $pdo->log->primary('Updated ' . $newgroup['group']); } } echo $pdo->log->header('Running time: ' . $consoleTools->convertTimer(TIME() - $start)); function myInArray($array, $value, $key) { //loop through the array
private function _postArticle($article, $retries = 3) { // Extract message id if (!preg_match('/Message-ID: <(?P<id>[^>]+)>/', $article[0], $matches)) { // we couldn't extract the message id return false; } $msg_id = $matches['id']; set_error_handler('snHandleError'); // Connect to server if (($this->_pdo->getSetting('alternate_nntp') == 1 ? $this->_nntp->doConnect(true, true) : $this->_nntp->doConnect()) !== true) { exit($this->_pdo->log->error("Unable to connect to usenet." . PHP_EOL)); } while ($retries > 0) { try { $summary = $this->_nntp->selectGroup($this->_post_group); if (NNTP::isError($summary)) { $summary = $this->_nntpReset($this->_post_group); $retries--; continue; } // Check if server will receive an article $_err = $this->_nntp->cmdPost(); if (NNTP::isError($_err)) { $summary = $this->_nntpReset($this->_post_group); $retries--; continue; } // Actually send the article $_err = $this->_nntp->cmdPost2($article); } catch (Exception $e) { // Ensure We're not connected try { $this->_nntp->doQuit(); } catch (Exception $e) { /* do nothing */ } // Post failed $retries--; // try again continue; } // Now we verify the post worked okay // The below code was commented out but not removed as it // is good reference on how to quickly scan for an article. // The problem with the below code is some providers were // taking up to 20 min for the post to show... so checking // right after posting was failing for this group. // $summary = $this->_nntp->selectGroup($this->_post_group); // if(NNTP::isError($summary)) { // $summary = $this->_nntpReset($this->_post_group); // $retries--; // continue; // } // $last = intval($summary['last']); // $batch = $last - SpotNab::VERIFY_FETCH_HEADER_COUNT; // $headers = $this->_get_headers($this->_post_group, // "$batch-$last", $retries); // // Ensure We're not connected // try{$this->_nntp->doQuit();} // catch(Exception $e) // {/* do nothing */} // if($headers === false){ // // We failed // return false; // } // // Okay we have headers to scan (we'll work back from the end to // // the front to speed the process up) // for($i=count($headers)-1; $i >0; $i--){ // if(!array_key_exists('Message-ID', $headers[$i])) // continue; // if ($headers[$i]['Message-ID'] == "<$msg_id>") // // Great! we found the post we just posted // return true; // } // // Otherwise we didn't find the Message-ID so we fail // return false; // Restore handler restore_error_handler(); // We're done return true; } // Restore handler restore_error_handler(); return false; }
function daytopost($nntp, $group, $days, $debug = true, $bfcheck = true) { global $pdo; $st = false; if ($debug && $bfcheck) { echo $pdo->log->primary('Finding start and end articles for ' . $group . '.'); } if (!isset($nntp)) { $nntp = new \NNTP(['Settings' => $pdo]); if ($nntp->doConnect(false) !== true) { return; } $st = true; } $binaries = new \Binaries(['NNTP' => $nntp, 'Settings' => $pdo]); $data = $nntp->selectGroup($group); if ($nntp->isError($data)) { $data = $nntp->dataError($nntp, $group, false); if ($data === false) { return; } } // Goal timestamp. $goaldate = date('U') - 86400 * $days; $totalnumberofarticles = $data['last'] - $data['first']; $upperbound = $data['last']; $lowerbound = $data['first']; if ($debug && $bfcheck) { echo $pdo->log->header('Total Articles: ' . number_format($totalnumberofarticles) . ' Newest: ' . number_format($upperbound) . ' Oldest: ' . number_format($lowerbound)); } if ($data['last'] == PHP_INT_MAX) { exit($pdo->log->error("Group data is coming back as php's max value. You should not see this since we use a patched Net_NNTP that fixes this bug.")); } $firstDate = $binaries->postdate($data['first'], $data); $lastDate = $binaries->postdate($data['last'], $data); if ($goaldate < $firstDate && $bfcheck) { if ($st === true) { $nntp->doQuit(); } echo $pdo->log->warning("The oldest post indexed from {$days} day(s) ago is older than the first article stored on your news server.\nSetting to First available article of (date('r', {$firstDate}) or daysOld({$firstDate}) days)."); return $data['first']; } else { if ($goaldate > $lastDate && $bfcheck) { if ($st === true) { $nntp->doQuit(); } echo $pdo->log->error("ERROR: The oldest post indexed from {$days} day(s) ago is newer than the last article stored on your news server.\nTo backfill this group you need to set Backfill Days to at least ceil(daysOld({$lastDate})+1) days (date('r', {$lastDate}-86400)."); return ''; } } if ($debug && $bfcheck) { echo $pdo->log->primary("Searching for postdates.\nGroup's Firstdate: " . $firstDate . ' (' . (is_int($firstDate) ? date('r', $firstDate) : 'n/a') . ").\nGroup's Lastdate: " . $lastDate . ' (' . date('r', $lastDate) . ")."); } $interval = floor(($upperbound - $lowerbound) * 0.5); $templowered = ''; $dateofnextone = $lastDate; // Match on days not timestamp to speed things up. while (daysOld($dateofnextone) < $days) { while (($tmpDate = $binaries->postdate($upperbound - $interval, $data)) > $goaldate) { $upperbound = $upperbound - $interval; } if (!$templowered) { $interval = ceil($interval / 2); } $dateofnextone = $binaries->postdate($upperbound - 1, $data); while (!$dateofnextone) { $dateofnextone = $binaries->postdate($upperbound - 1, $data); } } if ($st === true) { $nntp->doQuit(); } if ($bfcheck) { echo $pdo->log->header("\nBackfill article determined to be " . $upperbound . " " . $pdo->log->setColor('Yellow') . "(" . date('r', $dateofnextone) . ")" . $pdo->log->rsetcolor()); } else { echo $pdo->log->header('Current article determined to be ' . $upperbound . " " . $pdo->log->setColor('Yellow') . "(" . date('r', $dateofnextone) . ")" . $pdo->log->rsetcolor()); } // which is '.daysOld($dateofnextone)." days old.\n"; return $upperbound; }
/** * Download and process binaries for JPG pictures. * * @void * @access protected */ protected function _processJPGMessageIDs() { // Download JPG file. if ($this->_foundJPGSample === false && !empty($this->_JPGMessageIDs)) { // Try to download it. $jpgBinary = $this->_nntp->getMessages($this->_releaseGroupName, $this->_JPGMessageIDs, $this->_alternateNNTP); if ($this->_nntp->isError($jpgBinary)) { $jpgBinary = false; } if ($jpgBinary !== false) { if ($this->_echoCLI) { $this->_echo('(jB)', 'primaryOver', false); } // Try to create a file with it. @file_put_contents($this->tmpPath . 'samplepicture.jpg', $jpgBinary); // Try to resize and move it. $this->_foundJPGSample = $this->_releaseImage->saveImage($this->_release['guid'] . '_thumb', $this->tmpPath . 'samplepicture.jpg', $this->_releaseImage->jpgSavePath, 650, 650) === 1 ? true : false; if ($this->_foundJPGSample !== false) { // Update the DB to say we got it. $this->pdo->queryExec(sprintf(' UPDATE releases SET jpgstatus = %d WHERE id = %d', 1, $this->_release['id'])); if ($this->_echoCLI) { $this->_echo('j', 'primaryOver', false); } } @unlink($this->tmpPath . 'samplepicture.jpg'); } else { if ($this->_echoCLI) { $this->_echo('f', 'warningOver', false); } } } }
/** * Update the list of newsgroups and return an array of messages. * * @param string $groupList * @param int $active * @param int $backfill * * @return array */ public function addBulk($groupList, $active = 1, $backfill = 1) { if (preg_match('/^\\s*$/m', $groupList)) { $ret = "No group list provided."; } else { $nntp = new NNTP(['Echo' => false]); if ($nntp->doConnect() !== true) { return 'Problem connecting to usenet.'; } $groups = $nntp->getGroups(); $nntp->doQuit(); if ($nntp->isError($groups)) { return 'Problem fetching groups from usenet.'; } $regFilter = '/' . $groupList . '/i'; $ret = []; foreach ($groups as $group) { if (preg_match($regFilter, $group['group']) > 0) { $res = $this->pdo->queryOneRow(sprintf('SELECT id FROM groups WHERE name = %s', $this->pdo->escapeString($group['group']))); if ($res === false) { $this->pdo->queryInsert(sprintf('INSERT INTO groups (name, active, backfill) VALUES (%s, %d, %d)', $this->pdo->escapeString($group['group']), $active, $backfill)); $ret[] = ['group' => $group['group'], 'msg' => 'Created']; } } } if (count($ret) === 0) { $ret = 'No groups found with your regex, try again!'; } } return $ret; }
/** * Get nzpre data from usenet and parse. */ public function nzpreUpdate() { if (empty($this->pdo->getSetting('nzpregroup')) || empty($this->pdo->getSetting('nzpresubject')) || empty($this->pdo->getSetting('nzpreposter')) || empty($this->pdo->getSetting('nzprefield')) || empty($this->pdo->getSetting('nzprekey'))) { return false; } if ($this->echooutput) { echo "Predb : Checking for new pre data "; } $nntp = new NNTP(); if (!$nntp->doConnect()) { echo "Failed to get NNTP connection\n"; return false; } $ret = $groupData = $nntp->selectGroup($this->pdo->getSetting('nzpregroup')); if ($nntp->isError($ret)) { echo "Predb : Error " . $ret->getMessage() . "\n"; return false; } $ret = $groupMsgs = $nntp->getOverview($groupData['last'] - (!empty($this->pdo->getSetting('nzprearticles')) ? $this->pdo->getSetting('nzprearticles') : 500) . '-' . $groupData['last']); if ($nntp->isError($ret)) { echo "Predb : Error " . $ret->getMessage() . "\n"; return false; } $added_updated = 0; $nzprekey = $this->pdo->getSetting('nzprekey'); while (strlen($nzprekey) < 1024) { $nzprekey = $nzprekey . $nzprekey; } $cnt = !empty($this->pdo->getSetting('nzprearticles')) ? $this->pdo->getSetting('nzprearticles') : 500; foreach ($groupMsgs as $groupMsg) { if ($cnt % 50 == 0 && $cnt != 0 && $this->echooutput) { echo $cnt . ".."; } $cnt--; if (preg_match('/^' . $this->pdo->getSetting('nzpresubject') . '$/', $groupMsg['Subject']) && preg_match('/^' . $this->pdo->getSetting('nzpreposter') . '/', $groupMsg['From'])) { $ret = $msgHeader = $nntp->getHeader($groupMsg['Message-ID']); if ($nntp->isError($ret)) { continue; } for ($i = 0; $i < count($msgHeader); $i++) { if (preg_match('/^' . $this->pdo->getSetting('nzprefield') . ': /', $msgHeader[$i])) { if ($nzpreParse = $this->nzpreParse(str_replace($this->pdo->getSetting('nzprefield') . ': ', '', $msgHeader[$i]), $nzprekey)) { if ($this->updatePreDB($this->pdo, $nzpreParse)) { $added_updated++; } } break; } } } } $nntp->disconnect(); if ($this->echooutput) { echo "\nPredb : Added/Updated " . $added_updated . " records\n"; } }
require_once dirname(__FILE__) . '/../../../www/config.php'; if (!isset($argv[1])) { exit; } $group = $argv[1]; //$cleaner = new CollectionsCleaning(); $nntp = new NNTP(); $cli = new ColorCLI(); if ($nntp->doConnect() !== true) { exit($cli->error("Unable to connect to usenet.")); } $number = 1000000; //exec("tmux kill-session -t NNTPProxy"); $groupArr = $nntp->selectGroup($group); if ($nntp->isError($groupArr) || !isset($groupArr['first']) || !isset($groupArr['last'])) { exit; } print_r($groupArr); if (isset($argv[2]) && is_numeric($argv[2])) { $first = $argv[2]; } else { if ($groupArr['last'] - $number > $groupArr['first']) { $first = $groupArr['last'] - $number; } else { $first = $groupArr['first']; } } $last = $groupArr['last']; @unlink(nZEDb_RES . 'logs' . DS . 'not_yenc' . DS . $group . '.txt'); @unlink(nZEDb_RES . 'logs' . DS . 'not_yenc' . DS . $group . '.failed.regex.txt');
/** * Backfill single group. * * @param array $groupArr * @param int $left * @param int|string $articles * * @return void */ public function backfillGroup($groupArr, $left, $articles = '') { // Start time for this group. $startGroup = microtime(true); $groupName = str_replace('alt.binaries', 'a.b', $groupArr['name']); // If our local oldest article 0, it means we never ran update_binaries on the group. if ($groupArr['first_record'] <= 0) { $dMessage = "You need to run update_binaries on " . $groupName . ". Otherwise the group is dead, you must disable it."; if ($this->_debug) { $this->_debugging->log('Backfill', "backfillGroup", $dMessage, Logger::LOG_ERROR); } if ($this->_echoCLI) { $this->pdo->log->doEcho($this->pdo->log->error($dMessage)); } return; } // Select group, here, only once $data = $this->_nntp->selectGroup($groupArr['name']); if ($this->_nntp->isError($data)) { $data = $this->_nntp->dataError($this->_nntp, $groupArr['name']); if ($this->_nntp->isError($data)) { return; } } if ($this->_echoCLI) { $this->pdo->log->doEcho($this->pdo->log->primary('Processing ' . $groupName), true); } // Check if this is days or post backfill. $postCheck = $articles === '' ? false : true; // Get target post based on date or user specified number. $targetpost = (string) ($postCheck ? round($groupArr['first_record'] - $articles) : $this->_binaries->daytopost($groupArr['backfill_target'], $data)); // Check if target post is smaller than server's oldest, set it to oldest if so. if ($targetpost < $data['first']) { $targetpost = $data['first']; } // Check if our target post is newer than our oldest post or if our local oldest article is older than the servers oldest. if ($targetpost >= $groupArr['first_record'] || $groupArr['first_record'] <= $data['first']) { $dMessage = "We have hit the maximum we can backfill for " . $groupName . ", skipping it, consider disabling backfill on it."; if ($this->_debug) { $this->_debugging->log('Backfill', "backfillGroup", $dMessage, Logger::LOG_NOTICE); } if ($this->_echoCLI) { $this->pdo->log->doEcho($this->pdo->log->notice($dMessage), true); } return; } if ($this->_echoCLI) { $this->pdo->log->doEcho($this->pdo->log->primary('Group ' . $groupName . "'s oldest article is " . number_format($data['first']) . ', newest is ' . number_format($data['last']) . ".\nOur target article is " . number_format($targetpost) . '. Our oldest article is article ' . number_format($groupArr['first_record']) . '.')); } // Set first and last, moving the window by max messages. $last = (string) ($groupArr['first_record'] - 1); // Set the initial "chunk". $first = (string) ($last - $this->_binaries->messageBuffer + 1); // Just in case this is the last chunk we needed. if ($targetpost > $first) { $first = $targetpost; } $done = false; while ($done === false) { if ($this->_echoCLI) { $this->pdo->log->doEcho($this->pdo->log->set256('Yellow') . "\nGetting " . number_format($last - $first + 1) . " articles from " . $groupName . ", " . $left . " group(s) left. (" . number_format($first - $targetpost) . " articles in queue)." . $this->pdo->log->rsetColor(), true); } flush(); $lastMsg = $this->_binaries->scan($groupArr, $first, $last, $this->_safePartRepair); // Get the oldest date. if (isset($lastMsg['firstArticleDate'])) { // Try to get it from the oldest pulled article. $newdate = strtotime($lastMsg['firstArticleDate']); } else { // If above failed, try to get it with postdate method. $newdate = $this->_binaries->postdate($first, $data); } $this->pdo->queryExec(sprintf(' UPDATE groups SET first_record_postdate = %s, first_record = %s, last_updated = NOW() WHERE id = %d', $this->pdo->from_unixtime($newdate), $this->pdo->escapeString($first), $groupArr['id'])); if ($first == $targetpost) { $done = true; } else { // Keep going: set new last, new first, check for last chunk. $last = (string) ($first - 1); $first = (string) ($last - $this->_binaries->messageBuffer + 1); if ($targetpost > $first) { $first = $targetpost; } } } if ($this->_echoCLI) { $this->pdo->log->doEcho($this->pdo->log->primary(PHP_EOL . 'Group ' . $groupName . ' processed in ' . number_format(microtime(true) - $startGroup, 2) . " seconds."), true); } }
} else { $groups = $pdo->query(sprintf('SELECT * FROM groups WHERE name = %s', $pdo->escapeString($argv[1]))); } if (count($groups) === 0) { if ($argv[1] === 'all') { exit('ERROR! No groups were found with backfill enabled!' . PHP_EOL); } else { exit('ERROR! Group (' . $argv[1] . ') not found!' . PHP_EOL); } } $nntp = new NNTP(['Settings' => $pdo]); $nntp->doConnect() or exit('Could not connect to Usenet!' . PHP_EOL); $binaries = new Binaries(['NNTP' => $nntp, 'Settings' => $pdo]); foreach ($groups as $group) { $groupNNTP = $nntp->selectGroup($group['name']); if ($nntp->isError($groupNNTP)) { echo 'ERROR! Could not fetch information from NNTP for group (' . $group['name'] . ')' . PHP_EOL; continue; } $postDate = $pdo->queryOneRow(sprintf('SELECT UNIX_TIMESTAMP(postdate) AS postdate FROM releases WHERE group_id = %d ORDER BY postdate ASC LIMIT 1', $group['id'])); if ($postDate === false) { echo 'ERROR! Could not find any existing releases for group (' . $group['name'] . ')' . PHP_EOL; continue; } $articleNumber = (int) $binaries->daytopost(round((time() - $postDate['postdate']) / 86400), $groupNNTP); if ($group['last_record'] != 0 && $articleNumber >= $group['last_record']) { echo 'ERROR! Could not determine the article number for this date: (' . $postDate['postdate'] . ') on group (' . $group['name'] . ')' . PHP_EOL; continue; } $articleDate = $binaries->postdate($articleNumber, $groupNNTP); $pdo->queryExec(sprintf('
public function fetchTestBinaries($groupname, $numarticles, $clearexistingbins) { $nntp = new NNTP(); $binaries = new Binaries(); $groups = new Groups(); $ret = []; if ($clearexistingbins == true) { $this->pdo->queryExec('truncate releaseregextesting'); } $nntp->doConnect(); $groupsToFetch = []; if (preg_match('/^[a-z]{2,3}(\\.[a-z0-9\\-]+)+$/', $groupname)) { $groupsToFetch[] = array('name' => $groupname); } elseif ($groupname === 0) { $groupsToFetch = $groups->getAll(); } else { $newsgroups = $nntp->getGroups(); foreach ($newsgroups as $ngroup) { if (preg_match('/' . $groupname . '/', $ngroup['group'])) { $groupsToFetch[] = array('name' => $ngroup['group']); } } } foreach ($groupsToFetch as $groupArr) { $group = $groupArr['name']; $data = $nntp->selectGroup($group); if (NNTP::isError($data)) { $ret[] = "Could not select group (doesnt exist on USP): {$group}"; continue; } else { $rangeStart = $data['last'] - $numarticles; $rangeEnd = $groupEnd = $data['last']; $rangeTotal = $rangeEnd - $rangeStart; $done = false; while ($done === false) { if ($rangeTotal > $binaries->messageBuffer) { if ($rangeStart + $binaries->messageBuffer > $groupEnd) { $rangeEnd = $groupEnd; } else { $rangeEnd = $rangeStart + $binaries->messageBuffer; } } $msgs = $nntp->getXOver($rangeStart . "-" . $rangeEnd, true, false); if (NNTP::isError($msgs)) { $ret[] = "Error {$msgs->code}: {$msgs->message} on " . $group; continue 2; } $headers = []; if (is_array($msgs)) { //loop headers, figure out parts foreach ($msgs as $msg) { if (!isset($msg['Number'])) { continue; } $msgPart = $msgTotalParts = 0; $pattern = '|\\((\\d+)[\\/](\\d+)\\)|i'; preg_match_all($pattern, $msg['Subject'], $matches, PREG_PATTERN_ORDER); $matchcnt = sizeof($matches[0]); for ($i = 0; $i < $matchcnt; $i++) { //not (int)'d here because of the preg_replace later on $msgPart = $matches[1][$i]; $msgTotalParts = $matches[2][$i]; } if (!isset($msg['Subject']) || $matchcnt == 0) { // not a binary post most likely.. continue continue; } if ((int) $msgPart > 0 && (int) $msgTotalParts > 0) { $subject = utf8_encode(trim(preg_replace('|\\(' . $msgPart . '[\\/]' . $msgTotalParts . '\\)|i', '', $msg['Subject']))); if (!isset($headers[$subject])) { $headers[$subject]['Subject'] = $subject; $headers[$subject]['From'] = $msg['From']; $headers[$subject]['Date'] = strtotime($msg['Date']); $headers[$subject]['Message-ID'] = $msg['Message-ID']; $headers[$subject]['Size'] = $msg['Bytes']; } else { $headers[$subject]['Size'] += $msg['Bytes']; } } } unset($msgs); if (isset($headers) && count($headers)) { $groupRegexes = $this->getForGroup($group); $binSetData = []; foreach ($headers as $subject => $data) { $binData = array('name' => $subject, 'fromname' => $data['From'], 'date' => $data['Date'], 'binaryhash' => md5($subject . $data['From'] . $group), 'groupname' => $group, 'regexid' => "null", 'categoryid' => "null", 'reqid' => "null", 'blacklistid' => 0, 'size' => $data['Size'], 'relname' => "null", 'relpart' => "null", 'reltotalpart' => "null"); //Filter binaries based on black/white list if ($binaries->isBlackListed($data, $group)) { //binary is blacklisted $binData['blacklistid'] = 1; } //Apply Regexes $regexMatches = []; foreach ($groupRegexes as $groupRegex) { $regexCheck = $this->performMatch($groupRegex, $subject, $data['From']); if ($regexCheck !== false) { $regexMatches = $regexCheck; $binData['regexid'] = $regexCheck['regexid']; $binData['categoryid'] = $regexCheck['regcatid']; $binData['reqid'] = empty($regexCheck['reqid']) ? "null" : $regexCheck['reqid']; $binData['relname'] = $regexCheck['name']; break; } } $binSetData[] = $binData; } //insert 500 bins at a time $binChunks = array_chunk($binSetData, 500); foreach ($binChunks as $binChunk) { foreach ($binChunk as $chunk) { $binParams[] = sprintf("(%s, %s, FROM_UNIXTIME(%s), %s, %s, %s, %s, %s, %d, %d, now())", $this->pdo->escapeString($chunk['name']), $this->pdo->escapeString($chunk['fromname']), $this->pdo->escapeString($chunk['date']), $this->pdo->escapeString($chunk['binaryhash']), $this->pdo->escapeString($chunk['groupname']), $chunk['regexid'], $chunk['categoryid'], $chunk['reqid'], $chunk['blacklistid'], $chunk['size']); } $binSql = "INSERT IGNORE INTO releaseregextesting (name, fromname, date, binaryhash, groupname, regexid, categoryid, reqid, blacklistid, size, dateadded) VALUES " . implode(', ', $binParams); //echo $binSql; $this->pdo->queryExec($binSql); } $ret[] = "Fetched " . number_format($numarticles) . " articles from " . $group; } else { $ret[] = "No headers found on " . $group; continue; } } else { $ret[] = "Can't get parts from server (msgs not array) on " . $group; continue; } if ($rangeEnd == $groupEnd) { $done = true; } $rangeStart = $rangeEnd + 1; } } } $nntp->doQuit(); return $ret; }