/**
* Look for an .nfo file in the NZB, return the NFO message id.
* Gets the NZB completion.
* Looks for PAR2 files in the NZB.
*
* @param string $guid
* @param string $relID
* @param int $groupID
* @param string $groupName
*
* @return bool
*
* @access public
*/
public function getNfoFromNZB($guid, $relID, $groupID, $groupName)
{
$fetchedBinary = false;
$messageID = $this->parseNZB($guid, $relID, $groupID, true);
if ($messageID !== false) {
$fetchedBinary = $this->nntp->getMessages($groupName, $messageID['id'], $this->alternateNNTP);
if ($this->nntp->isError($fetchedBinary)) {
// NFO download failed, increment attempts.
$this->pdo->queryExec(sprintf('UPDATE releases SET nfostatus = nfostatus - 1 WHERE id = %d', $relID));
if ($this->echooutput) {
echo 'f';
}
return false;
}
if ($this->nfo->isNFO($fetchedBinary, $guid) === true) {
if ($this->echooutput) {
echo $messageID['hidden'] === false ? '+' : '*';
}
} else {
if ($this->echooutput) {
echo '-';
}
$this->pdo->queryExec(sprintf('UPDATE releases SET nfostatus = %d WHERE id = %d', Nfo::NFO_NONFO, $relID));
$fetchedBinary = false;
}
} else {
if ($this->echooutput) {
echo '-';
}
$this->pdo->queryExec(sprintf('UPDATE releases SET nfostatus = %d WHERE id = %d', Nfo::NFO_NONFO, $relID));
}
return $fetchedBinary;
}
/**
* Fetch a comment and insert it.
*
* @param string $messageID Message-ID for the article.
* @param string $siteID id of the site.
*
* @return bool
*/
protected function insertNewComment(&$messageID, &$siteID)
{
// Get the article body.
$body = $this->nntp->getMessages(self::group, $messageID);
// Check if there's an error.
if ($this->nntp->isError($body)) {
return false;
}
// Decompress the body.
$body = @gzinflate($body);
if ($body === false) {
return false;
}
// JSON Decode the body.
$body = json_decode($body, true);
if ($body === false) {
return false;
}
// Just in case.
if (!isset($body['USER']) || !isset($body['SID']) || !isset($body['RID']) || !isset($body['TIME']) | !isset($body['BODY'])) {
return false;
}
$cid = md5($body['SID'] . $body['USER'] . $body['TIME'] . $siteID);
// Insert the comment.
if ($this->pdo->queryExec(sprintf('
INSERT IGNORE INTO releasecomment
(text, createddate, issynced, shareid, cid, gid, nzb_guid, siteid, username, userid, releaseid, shared, host, sourceID)
VALUES (%s, %s, 1, %s, %s, %s, %s, %s, %s, 0, 0, 2, "", 999)', $this->pdo->escapeString($body['BODY']), $this->pdo->from_unixtime($body['TIME'] > time() ? time() : $body['TIME']), $this->pdo->escapeString($body['SID']), $this->pdo->escapeString($cid), $this->pdo->escapeString($body['RID']), $this->pdo->escapeString($body['RID']), $this->pdo->escapeString($siteID), $this->pdo->escapeString(substr($body['USER'], 0, 3) === 'sn-' ? 'SH_ANON' : 'SH_' . $body['USER'])))) {
return true;
}
return false;
}
/**
* Attempt to get a better name from a par2 file and categorize the release.
*
* @note Called from NZBContents.php
*
* @param string $messageID MessageID from NZB file.
* @param int $relID id of the release.
* @param int $groupID Group id of the release.
* @param \NNTP $nntp Class NNTP
* @param int $show Only show result or apply iy.
*
* @return bool
*/
public function parsePAR2($messageID, $relID, $groupID, &$nntp, $show)
{
if ($messageID === '') {
return false;
}
$query = $this->pdo->queryOneRow(sprintf('
SELECT id, groupid, categoryid, name, searchname, UNIX_TIMESTAMP(postdate) AS post_date, id AS releaseid
FROM releases
WHERE isrenamed = 0
AND id = %d', $relID));
if ($query === false) {
return false;
}
// Only get a new name if the category is OTHER.
$foundName = true;
if (!in_array((int) $query['categoryid'], array(\Category::CAT_BOOK_OTHER, \Category::CAT_GAME_OTHER, \Category::CAT_MOVIE_OTHER, \Category::CAT_MUSIC_OTHER, \Category::CAT_PC_MOBILEOTHER, \Category::CAT_TV_OTHER, \Category::CAT_MISC_HASHED, \Category::CAT_XXX_OTHER, \Category::CAT_MISC_OTHER))) {
$foundName = false;
}
// Get the PAR2 file.
$par2 = $nntp->getMessages($this->groups->getByNameByID($groupID), $messageID, $this->alternateNNTP);
if ($nntp->isError($par2)) {
return false;
}
// Put the PAR2 into Par2Info, check if there's an error.
$this->_par2Info->setData($par2);
if ($this->_par2Info->error) {
return false;
}
// Get the file list from Par2Info.
$files = $this->_par2Info->getFileList();
if ($files !== false && count($files) > 0) {
$filesAdded = 0;
// Loop through the files.
foreach ($files as $file) {
if (!isset($file['name'])) {
continue;
}
// If we found a name and added 10 files, stop.
if ($foundName === true && $filesAdded > 10) {
break;
}
if ($this->addpar2) {
// Add to release files.
if ($filesAdded < 11 && $this->pdo->queryOneRow(sprintf('
SELECT id
FROM releasefiles
WHERE releaseid = %d
AND name = %s', $relID, $this->pdo->escapeString($file['name']))) === false) {
// Try to add the files to the DB.
if ($this->releaseFiles->add($relID, $file['name'], $file['size'], $query['post_date'], 0)) {
$filesAdded++;
}
}
} else {
$filesAdded++;
}
// Try to get a new name.
if ($foundName === false) {
$query['textstring'] = $file['name'];
if ($this->nameFixer->checkName($query, 1, 'PAR2, ', 1, $show) === true) {
$foundName = true;
}
}
}
// If we found some files.
if ($filesAdded > 0) {
$this->debugging->log(get_class(), __FUNCTION__, 'Added ' . $filesAdded . ' releasefiles from PAR2 for ' . $query['searchname'], \Logger::LOG_INFO);
// Update the file count with the new file count + old file count.
$this->pdo->queryExec(sprintf('
UPDATE releases
SET rarinnerfilecount = rarinnerfilecount + %d
WHERE id = %d', $filesAdded, $relID));
}
if ($foundName === true) {
return true;
}
}
return false;
}
/**
* Returns unix time for an article number.
*
* @param int $post The article number to get the time from.
* @param array $groupData Usenet group info from NNTP selectGroup method.
*
* @return bool|int
*/
public function postdate($post, array $groupData)
{
// Set table names
$groupID = $this->_groups->getIDByName($groupData['group']);
$group = [];
if ($groupID !== '') {
$group = $this->_groups->getCBPTableNames($this->_tablePerGroup, $groupID);
}
$currentPost = $post;
$attempts = $date = 0;
do {
// Try to get the article date locally first.
if ($groupID !== '') {
// Try to get locally.
$local = $this->_pdo->queryOneRow(sprintf('
SELECT b.date AS date
FROM %s b, %s p
WHERE b.id = p.binaryid
AND b.groupid = %s
AND p.number = %s LIMIT 1', $group['bname'], $group['pname'], $groupID, $currentPost));
if ($local !== false) {
$date = $local['date'];
break;
}
}
// If we could not find it locally, try usenet.
$header = $this->_nntp->getXOVER($currentPost);
if (!$this->_nntp->isError($header)) {
// Check if the date is set.
if (isset($header[0]['Date']) && strlen($header[0]['Date']) > 0) {
$date = $header[0]['Date'];
break;
}
}
// Try to get a different article number.
if (abs($currentPost - $groupData['first']) > abs($groupData['last'] - $currentPost)) {
$tempPost = round($currentPost / (mt_rand(1005, 1012) / 1000), 0, PHP_ROUND_HALF_UP);
if ($tempPost < $groupData['first']) {
$tempPost = $groupData['first'];
}
} else {
$tempPost = round(mt_rand(1005, 1012) / 1000 * $currentPost, 0, PHP_ROUND_HALF_UP);
if ($tempPost > $groupData['last']) {
$tempPost = $groupData['last'];
}
}
// If we got the same article number as last time, give up.
if ($tempPost === $currentPost) {
break;
}
$currentPost = $tempPost;
if ($this->_debug) {
$this->_colorCLI->doEcho($this->_colorCLI->debug('Postdate retried ' . $attempts . " time(s)."));
}
} while ($attempts++ <= 20);
// If we didn't get a date, set it to now.
if (!$date) {
$date = time();
} else {
$date = strtotime($date);
}
if ($this->_debug) {
$this->_debugging->log('Binaries', "postdate", 'Article (' . $post . "'s) date is (" . $date . ') (' . $this->daysOld($date) . " days old)", \Logger::LOG_INFO);
}
return $date;
}
<?php
require_once dirname(__FILE__) . '/../../../config.php';
use nzedb\db\Settings;
$start = TIME();
$pdo = new Settings();
$consoleTools = new ConsoleTools(['ColorCLI' => $pdo->log]);
// Create the connection here and pass
$nntp = new NNTP(['Settings' => $pdo]);
if ($nntp->doConnect() !== true) {
exit($pdo->log->error("Unable to connect to usenet."));
}
echo $pdo->log->header("Getting first/last for all your active groups.");
$data = $nntp->getGroups();
if ($nntp->isError($data)) {
exit($pdo->log->error("Failed to getGroups() from nntp server."));
}
echo $pdo->log->header("Inserting new values into shortgroups table.");
$pdo->queryExec('TRUNCATE TABLE shortgroups');
// Put into an array all active groups
$res = $pdo->query('SELECT name FROM groups WHERE active = 1 OR backfill = 1');
foreach ($data as $newgroup) {
if (myInArray($res, $newgroup['group'], 'name')) {
$pdo->queryInsert(sprintf('INSERT INTO shortgroups (name, first_record, last_record, updated) VALUES (%s, %s, %s, NOW())', $pdo->escapeString($newgroup['group']), $pdo->escapeString($newgroup['first']), $pdo->escapeString($newgroup['last'])));
echo $pdo->log->primary('Updated ' . $newgroup['group']);
}
}
echo $pdo->log->header('Running time: ' . $consoleTools->convertTimer(TIME() - $start));
function myInArray($array, $value, $key)
{
//loop through the array
private function _postArticle($article, $retries = 3)
{
// Extract message id
if (!preg_match('/Message-ID: <(?P<id>[^>]+)>/', $article[0], $matches)) {
// we couldn't extract the message id
return false;
}
$msg_id = $matches['id'];
set_error_handler('snHandleError');
// Connect to server
if (($this->_pdo->getSetting('alternate_nntp') == 1 ? $this->_nntp->doConnect(true, true) : $this->_nntp->doConnect()) !== true) {
exit($this->_pdo->log->error("Unable to connect to usenet." . PHP_EOL));
}
while ($retries > 0) {
try {
$summary = $this->_nntp->selectGroup($this->_post_group);
if (NNTP::isError($summary)) {
$summary = $this->_nntpReset($this->_post_group);
$retries--;
continue;
}
// Check if server will receive an article
$_err = $this->_nntp->cmdPost();
if (NNTP::isError($_err)) {
$summary = $this->_nntpReset($this->_post_group);
$retries--;
continue;
}
// Actually send the article
$_err = $this->_nntp->cmdPost2($article);
} catch (Exception $e) {
// Ensure We're not connected
try {
$this->_nntp->doQuit();
} catch (Exception $e) {
/* do nothing */
}
// Post failed
$retries--;
// try again
continue;
}
// Now we verify the post worked okay
// The below code was commented out but not removed as it
// is good reference on how to quickly scan for an article.
// The problem with the below code is some providers were
// taking up to 20 min for the post to show... so checking
// right after posting was failing for this group.
// $summary = $this->_nntp->selectGroup($this->_post_group);
// if(NNTP::isError($summary)) {
// $summary = $this->_nntpReset($this->_post_group);
// $retries--;
// continue;
// }
// $last = intval($summary['last']);
// $batch = $last - SpotNab::VERIFY_FETCH_HEADER_COUNT;
// $headers = $this->_get_headers($this->_post_group,
// "$batch-$last", $retries);
// // Ensure We're not connected
// try{$this->_nntp->doQuit();}
// catch(Exception $e)
// {/* do nothing */}
// if($headers === false){
// // We failed
// return false;
// }
// // Okay we have headers to scan (we'll work back from the end to
// // the front to speed the process up)
// for($i=count($headers)-1; $i >0; $i--){
// if(!array_key_exists('Message-ID', $headers[$i]))
// continue;
// if ($headers[$i]['Message-ID'] == "<$msg_id>")
// // Great! we found the post we just posted
// return true;
// }
// // Otherwise we didn't find the Message-ID so we fail
// return false;
// Restore handler
restore_error_handler();
// We're done
return true;
}
// Restore handler
restore_error_handler();
return false;
}
function daytopost($nntp, $group, $days, $debug = true, $bfcheck = true)
{
global $pdo;
$st = false;
if ($debug && $bfcheck) {
echo $pdo->log->primary('Finding start and end articles for ' . $group . '.');
}
if (!isset($nntp)) {
$nntp = new \NNTP(['Settings' => $pdo]);
if ($nntp->doConnect(false) !== true) {
return;
}
$st = true;
}
$binaries = new \Binaries(['NNTP' => $nntp, 'Settings' => $pdo]);
$data = $nntp->selectGroup($group);
if ($nntp->isError($data)) {
$data = $nntp->dataError($nntp, $group, false);
if ($data === false) {
return;
}
}
// Goal timestamp.
$goaldate = date('U') - 86400 * $days;
$totalnumberofarticles = $data['last'] - $data['first'];
$upperbound = $data['last'];
$lowerbound = $data['first'];
if ($debug && $bfcheck) {
echo $pdo->log->header('Total Articles: ' . number_format($totalnumberofarticles) . ' Newest: ' . number_format($upperbound) . ' Oldest: ' . number_format($lowerbound));
}
if ($data['last'] == PHP_INT_MAX) {
exit($pdo->log->error("Group data is coming back as php's max value. You should not see this since we use a patched Net_NNTP that fixes this bug."));
}
$firstDate = $binaries->postdate($data['first'], $data);
$lastDate = $binaries->postdate($data['last'], $data);
if ($goaldate < $firstDate && $bfcheck) {
if ($st === true) {
$nntp->doQuit();
}
echo $pdo->log->warning("The oldest post indexed from {$days} day(s) ago is older than the first article stored on your news server.\nSetting to First available article of (date('r', {$firstDate}) or daysOld({$firstDate}) days).");
return $data['first'];
} else {
if ($goaldate > $lastDate && $bfcheck) {
if ($st === true) {
$nntp->doQuit();
}
echo $pdo->log->error("ERROR: The oldest post indexed from {$days} day(s) ago is newer than the last article stored on your news server.\nTo backfill this group you need to set Backfill Days to at least ceil(daysOld({$lastDate})+1) days (date('r', {$lastDate}-86400).");
return '';
}
}
if ($debug && $bfcheck) {
echo $pdo->log->primary("Searching for postdates.\nGroup's Firstdate: " . $firstDate . ' (' . (is_int($firstDate) ? date('r', $firstDate) : 'n/a') . ").\nGroup's Lastdate: " . $lastDate . ' (' . date('r', $lastDate) . ").");
}
$interval = floor(($upperbound - $lowerbound) * 0.5);
$templowered = '';
$dateofnextone = $lastDate;
// Match on days not timestamp to speed things up.
while (daysOld($dateofnextone) < $days) {
while (($tmpDate = $binaries->postdate($upperbound - $interval, $data)) > $goaldate) {
$upperbound = $upperbound - $interval;
}
if (!$templowered) {
$interval = ceil($interval / 2);
}
$dateofnextone = $binaries->postdate($upperbound - 1, $data);
while (!$dateofnextone) {
$dateofnextone = $binaries->postdate($upperbound - 1, $data);
}
}
if ($st === true) {
$nntp->doQuit();
}
if ($bfcheck) {
echo $pdo->log->header("\nBackfill article determined to be " . $upperbound . " " . $pdo->log->setColor('Yellow') . "(" . date('r', $dateofnextone) . ")" . $pdo->log->rsetcolor());
} else {
echo $pdo->log->header('Current article determined to be ' . $upperbound . " " . $pdo->log->setColor('Yellow') . "(" . date('r', $dateofnextone) . ")" . $pdo->log->rsetcolor());
}
// which is '.daysOld($dateofnextone)." days old.\n";
return $upperbound;
}
/**
* Download and process binaries for JPG pictures.
*
* @void
* @access protected
*/
protected function _processJPGMessageIDs()
{
// Download JPG file.
if ($this->_foundJPGSample === false && !empty($this->_JPGMessageIDs)) {
// Try to download it.
$jpgBinary = $this->_nntp->getMessages($this->_releaseGroupName, $this->_JPGMessageIDs, $this->_alternateNNTP);
if ($this->_nntp->isError($jpgBinary)) {
$jpgBinary = false;
}
if ($jpgBinary !== false) {
if ($this->_echoCLI) {
$this->_echo('(jB)', 'primaryOver', false);
}
// Try to create a file with it.
@file_put_contents($this->tmpPath . 'samplepicture.jpg', $jpgBinary);
// Try to resize and move it.
$this->_foundJPGSample = $this->_releaseImage->saveImage($this->_release['guid'] . '_thumb', $this->tmpPath . 'samplepicture.jpg', $this->_releaseImage->jpgSavePath, 650, 650) === 1 ? true : false;
if ($this->_foundJPGSample !== false) {
// Update the DB to say we got it.
$this->pdo->queryExec(sprintf('
UPDATE releases
SET jpgstatus = %d
WHERE id = %d', 1, $this->_release['id']));
if ($this->_echoCLI) {
$this->_echo('j', 'primaryOver', false);
}
}
@unlink($this->tmpPath . 'samplepicture.jpg');
} else {
if ($this->_echoCLI) {
$this->_echo('f', 'warningOver', false);
}
}
}
}
/**
* Update the list of newsgroups and return an array of messages.
*
* @param string $groupList
* @param int $active
* @param int $backfill
*
* @return array
*/
public function addBulk($groupList, $active = 1, $backfill = 1)
{
if (preg_match('/^\\s*$/m', $groupList)) {
$ret = "No group list provided.";
} else {
$nntp = new NNTP(['Echo' => false]);
if ($nntp->doConnect() !== true) {
return 'Problem connecting to usenet.';
}
$groups = $nntp->getGroups();
$nntp->doQuit();
if ($nntp->isError($groups)) {
return 'Problem fetching groups from usenet.';
}
$regFilter = '/' . $groupList . '/i';
$ret = [];
foreach ($groups as $group) {
if (preg_match($regFilter, $group['group']) > 0) {
$res = $this->pdo->queryOneRow(sprintf('SELECT id FROM groups WHERE name = %s', $this->pdo->escapeString($group['group'])));
if ($res === false) {
$this->pdo->queryInsert(sprintf('INSERT INTO groups (name, active, backfill) VALUES (%s, %d, %d)', $this->pdo->escapeString($group['group']), $active, $backfill));
$ret[] = ['group' => $group['group'], 'msg' => 'Created'];
}
}
}
if (count($ret) === 0) {
$ret = 'No groups found with your regex, try again!';
}
}
return $ret;
}
/**
* Get nzpre data from usenet and parse.
*/
public function nzpreUpdate()
{
if (empty($this->pdo->getSetting('nzpregroup')) || empty($this->pdo->getSetting('nzpresubject')) || empty($this->pdo->getSetting('nzpreposter')) || empty($this->pdo->getSetting('nzprefield')) || empty($this->pdo->getSetting('nzprekey'))) {
return false;
}
if ($this->echooutput) {
echo "Predb : Checking for new pre data ";
}
$nntp = new NNTP();
if (!$nntp->doConnect()) {
echo "Failed to get NNTP connection\n";
return false;
}
$ret = $groupData = $nntp->selectGroup($this->pdo->getSetting('nzpregroup'));
if ($nntp->isError($ret)) {
echo "Predb : Error " . $ret->getMessage() . "\n";
return false;
}
$ret = $groupMsgs = $nntp->getOverview($groupData['last'] - (!empty($this->pdo->getSetting('nzprearticles')) ? $this->pdo->getSetting('nzprearticles') : 500) . '-' . $groupData['last']);
if ($nntp->isError($ret)) {
echo "Predb : Error " . $ret->getMessage() . "\n";
return false;
}
$added_updated = 0;
$nzprekey = $this->pdo->getSetting('nzprekey');
while (strlen($nzprekey) < 1024) {
$nzprekey = $nzprekey . $nzprekey;
}
$cnt = !empty($this->pdo->getSetting('nzprearticles')) ? $this->pdo->getSetting('nzprearticles') : 500;
foreach ($groupMsgs as $groupMsg) {
if ($cnt % 50 == 0 && $cnt != 0 && $this->echooutput) {
echo $cnt . "..";
}
$cnt--;
if (preg_match('/^' . $this->pdo->getSetting('nzpresubject') . '$/', $groupMsg['Subject']) && preg_match('/^' . $this->pdo->getSetting('nzpreposter') . '/', $groupMsg['From'])) {
$ret = $msgHeader = $nntp->getHeader($groupMsg['Message-ID']);
if ($nntp->isError($ret)) {
continue;
}
for ($i = 0; $i < count($msgHeader); $i++) {
if (preg_match('/^' . $this->pdo->getSetting('nzprefield') . ': /', $msgHeader[$i])) {
if ($nzpreParse = $this->nzpreParse(str_replace($this->pdo->getSetting('nzprefield') . ': ', '', $msgHeader[$i]), $nzprekey)) {
if ($this->updatePreDB($this->pdo, $nzpreParse)) {
$added_updated++;
}
}
break;
}
}
}
}
$nntp->disconnect();
if ($this->echooutput) {
echo "\nPredb : Added/Updated " . $added_updated . " records\n";
}
}
require_once dirname(__FILE__) . '/../../../www/config.php';
if (!isset($argv[1])) {
exit;
}
$group = $argv[1];
//$cleaner = new CollectionsCleaning();
$nntp = new NNTP();
$cli = new ColorCLI();
if ($nntp->doConnect() !== true) {
exit($cli->error("Unable to connect to usenet."));
}
$number = 1000000;
//exec("tmux kill-session -t NNTPProxy");
$groupArr = $nntp->selectGroup($group);
if ($nntp->isError($groupArr) || !isset($groupArr['first']) || !isset($groupArr['last'])) {
exit;
}
print_r($groupArr);
if (isset($argv[2]) && is_numeric($argv[2])) {
$first = $argv[2];
} else {
if ($groupArr['last'] - $number > $groupArr['first']) {
$first = $groupArr['last'] - $number;
} else {
$first = $groupArr['first'];
}
}
$last = $groupArr['last'];
@unlink(nZEDb_RES . 'logs' . DS . 'not_yenc' . DS . $group . '.txt');
@unlink(nZEDb_RES . 'logs' . DS . 'not_yenc' . DS . $group . '.failed.regex.txt');
/**
* Backfill single group.
*
* @param array $groupArr
* @param int $left
* @param int|string $articles
*
* @return void
*/
public function backfillGroup($groupArr, $left, $articles = '')
{
// Start time for this group.
$startGroup = microtime(true);
$groupName = str_replace('alt.binaries', 'a.b', $groupArr['name']);
// If our local oldest article 0, it means we never ran update_binaries on the group.
if ($groupArr['first_record'] <= 0) {
$dMessage = "You need to run update_binaries on " . $groupName . ". Otherwise the group is dead, you must disable it.";
if ($this->_debug) {
$this->_debugging->log('Backfill', "backfillGroup", $dMessage, Logger::LOG_ERROR);
}
if ($this->_echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->error($dMessage));
}
return;
}
// Select group, here, only once
$data = $this->_nntp->selectGroup($groupArr['name']);
if ($this->_nntp->isError($data)) {
$data = $this->_nntp->dataError($this->_nntp, $groupArr['name']);
if ($this->_nntp->isError($data)) {
return;
}
}
if ($this->_echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->primary('Processing ' . $groupName), true);
}
// Check if this is days or post backfill.
$postCheck = $articles === '' ? false : true;
// Get target post based on date or user specified number.
$targetpost = (string) ($postCheck ? round($groupArr['first_record'] - $articles) : $this->_binaries->daytopost($groupArr['backfill_target'], $data));
// Check if target post is smaller than server's oldest, set it to oldest if so.
if ($targetpost < $data['first']) {
$targetpost = $data['first'];
}
// Check if our target post is newer than our oldest post or if our local oldest article is older than the servers oldest.
if ($targetpost >= $groupArr['first_record'] || $groupArr['first_record'] <= $data['first']) {
$dMessage = "We have hit the maximum we can backfill for " . $groupName . ", skipping it, consider disabling backfill on it.";
if ($this->_debug) {
$this->_debugging->log('Backfill', "backfillGroup", $dMessage, Logger::LOG_NOTICE);
}
if ($this->_echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->notice($dMessage), true);
}
return;
}
if ($this->_echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->primary('Group ' . $groupName . "'s oldest article is " . number_format($data['first']) . ', newest is ' . number_format($data['last']) . ".\nOur target article is " . number_format($targetpost) . '. Our oldest article is article ' . number_format($groupArr['first_record']) . '.'));
}
// Set first and last, moving the window by max messages.
$last = (string) ($groupArr['first_record'] - 1);
// Set the initial "chunk".
$first = (string) ($last - $this->_binaries->messageBuffer + 1);
// Just in case this is the last chunk we needed.
if ($targetpost > $first) {
$first = $targetpost;
}
$done = false;
while ($done === false) {
if ($this->_echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->set256('Yellow') . "\nGetting " . number_format($last - $first + 1) . " articles from " . $groupName . ", " . $left . " group(s) left. (" . number_format($first - $targetpost) . " articles in queue)." . $this->pdo->log->rsetColor(), true);
}
flush();
$lastMsg = $this->_binaries->scan($groupArr, $first, $last, $this->_safePartRepair);
// Get the oldest date.
if (isset($lastMsg['firstArticleDate'])) {
// Try to get it from the oldest pulled article.
$newdate = strtotime($lastMsg['firstArticleDate']);
} else {
// If above failed, try to get it with postdate method.
$newdate = $this->_binaries->postdate($first, $data);
}
$this->pdo->queryExec(sprintf('
UPDATE groups
SET first_record_postdate = %s, first_record = %s, last_updated = NOW()
WHERE id = %d', $this->pdo->from_unixtime($newdate), $this->pdo->escapeString($first), $groupArr['id']));
if ($first == $targetpost) {
$done = true;
} else {
// Keep going: set new last, new first, check for last chunk.
$last = (string) ($first - 1);
$first = (string) ($last - $this->_binaries->messageBuffer + 1);
if ($targetpost > $first) {
$first = $targetpost;
}
}
}
if ($this->_echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->primary(PHP_EOL . 'Group ' . $groupName . ' processed in ' . number_format(microtime(true) - $startGroup, 2) . " seconds."), true);
}
}
} else {
$groups = $pdo->query(sprintf('SELECT * FROM groups WHERE name = %s', $pdo->escapeString($argv[1])));
}
if (count($groups) === 0) {
if ($argv[1] === 'all') {
exit('ERROR! No groups were found with backfill enabled!' . PHP_EOL);
} else {
exit('ERROR! Group (' . $argv[1] . ') not found!' . PHP_EOL);
}
}
$nntp = new NNTP(['Settings' => $pdo]);
$nntp->doConnect() or exit('Could not connect to Usenet!' . PHP_EOL);
$binaries = new Binaries(['NNTP' => $nntp, 'Settings' => $pdo]);
foreach ($groups as $group) {
$groupNNTP = $nntp->selectGroup($group['name']);
if ($nntp->isError($groupNNTP)) {
echo 'ERROR! Could not fetch information from NNTP for group (' . $group['name'] . ')' . PHP_EOL;
continue;
}
$postDate = $pdo->queryOneRow(sprintf('SELECT UNIX_TIMESTAMP(postdate) AS postdate FROM releases WHERE group_id = %d ORDER BY postdate ASC LIMIT 1', $group['id']));
if ($postDate === false) {
echo 'ERROR! Could not find any existing releases for group (' . $group['name'] . ')' . PHP_EOL;
continue;
}
$articleNumber = (int) $binaries->daytopost(round((time() - $postDate['postdate']) / 86400), $groupNNTP);
if ($group['last_record'] != 0 && $articleNumber >= $group['last_record']) {
echo 'ERROR! Could not determine the article number for this date: (' . $postDate['postdate'] . ') on group (' . $group['name'] . ')' . PHP_EOL;
continue;
}
$articleDate = $binaries->postdate($articleNumber, $groupNNTP);
$pdo->queryExec(sprintf('
public function fetchTestBinaries($groupname, $numarticles, $clearexistingbins)
{
$nntp = new NNTP();
$binaries = new Binaries();
$groups = new Groups();
$ret = [];
if ($clearexistingbins == true) {
$this->pdo->queryExec('truncate releaseregextesting');
}
$nntp->doConnect();
$groupsToFetch = [];
if (preg_match('/^[a-z]{2,3}(\\.[a-z0-9\\-]+)+$/', $groupname)) {
$groupsToFetch[] = array('name' => $groupname);
} elseif ($groupname === 0) {
$groupsToFetch = $groups->getAll();
} else {
$newsgroups = $nntp->getGroups();
foreach ($newsgroups as $ngroup) {
if (preg_match('/' . $groupname . '/', $ngroup['group'])) {
$groupsToFetch[] = array('name' => $ngroup['group']);
}
}
}
foreach ($groupsToFetch as $groupArr) {
$group = $groupArr['name'];
$data = $nntp->selectGroup($group);
if (NNTP::isError($data)) {
$ret[] = "Could not select group (doesnt exist on USP): {$group}";
continue;
} else {
$rangeStart = $data['last'] - $numarticles;
$rangeEnd = $groupEnd = $data['last'];
$rangeTotal = $rangeEnd - $rangeStart;
$done = false;
while ($done === false) {
if ($rangeTotal > $binaries->messageBuffer) {
if ($rangeStart + $binaries->messageBuffer > $groupEnd) {
$rangeEnd = $groupEnd;
} else {
$rangeEnd = $rangeStart + $binaries->messageBuffer;
}
}
$msgs = $nntp->getXOver($rangeStart . "-" . $rangeEnd, true, false);
if (NNTP::isError($msgs)) {
$ret[] = "Error {$msgs->code}: {$msgs->message} on " . $group;
continue 2;
}
$headers = [];
if (is_array($msgs)) {
//loop headers, figure out parts
foreach ($msgs as $msg) {
if (!isset($msg['Number'])) {
continue;
}
$msgPart = $msgTotalParts = 0;
$pattern = '|\\((\\d+)[\\/](\\d+)\\)|i';
preg_match_all($pattern, $msg['Subject'], $matches, PREG_PATTERN_ORDER);
$matchcnt = sizeof($matches[0]);
for ($i = 0; $i < $matchcnt; $i++) {
//not (int)'d here because of the preg_replace later on
$msgPart = $matches[1][$i];
$msgTotalParts = $matches[2][$i];
}
if (!isset($msg['Subject']) || $matchcnt == 0) {
// not a binary post most likely.. continue
continue;
}
if ((int) $msgPart > 0 && (int) $msgTotalParts > 0) {
$subject = utf8_encode(trim(preg_replace('|\\(' . $msgPart . '[\\/]' . $msgTotalParts . '\\)|i', '', $msg['Subject'])));
if (!isset($headers[$subject])) {
$headers[$subject]['Subject'] = $subject;
$headers[$subject]['From'] = $msg['From'];
$headers[$subject]['Date'] = strtotime($msg['Date']);
$headers[$subject]['Message-ID'] = $msg['Message-ID'];
$headers[$subject]['Size'] = $msg['Bytes'];
} else {
$headers[$subject]['Size'] += $msg['Bytes'];
}
}
}
unset($msgs);
if (isset($headers) && count($headers)) {
$groupRegexes = $this->getForGroup($group);
$binSetData = [];
foreach ($headers as $subject => $data) {
$binData = array('name' => $subject, 'fromname' => $data['From'], 'date' => $data['Date'], 'binaryhash' => md5($subject . $data['From'] . $group), 'groupname' => $group, 'regexid' => "null", 'categoryid' => "null", 'reqid' => "null", 'blacklistid' => 0, 'size' => $data['Size'], 'relname' => "null", 'relpart' => "null", 'reltotalpart' => "null");
//Filter binaries based on black/white list
if ($binaries->isBlackListed($data, $group)) {
//binary is blacklisted
$binData['blacklistid'] = 1;
}
//Apply Regexes
$regexMatches = [];
foreach ($groupRegexes as $groupRegex) {
$regexCheck = $this->performMatch($groupRegex, $subject, $data['From']);
if ($regexCheck !== false) {
$regexMatches = $regexCheck;
$binData['regexid'] = $regexCheck['regexid'];
$binData['categoryid'] = $regexCheck['regcatid'];
$binData['reqid'] = empty($regexCheck['reqid']) ? "null" : $regexCheck['reqid'];
$binData['relname'] = $regexCheck['name'];
break;
}
}
$binSetData[] = $binData;
}
//insert 500 bins at a time
$binChunks = array_chunk($binSetData, 500);
foreach ($binChunks as $binChunk) {
foreach ($binChunk as $chunk) {
$binParams[] = sprintf("(%s, %s, FROM_UNIXTIME(%s), %s, %s, %s, %s, %s, %d, %d, now())", $this->pdo->escapeString($chunk['name']), $this->pdo->escapeString($chunk['fromname']), $this->pdo->escapeString($chunk['date']), $this->pdo->escapeString($chunk['binaryhash']), $this->pdo->escapeString($chunk['groupname']), $chunk['regexid'], $chunk['categoryid'], $chunk['reqid'], $chunk['blacklistid'], $chunk['size']);
}
$binSql = "INSERT IGNORE INTO releaseregextesting (name, fromname, date, binaryhash, groupname, regexid, categoryid, reqid, blacklistid, size, dateadded) VALUES " . implode(', ', $binParams);
//echo $binSql;
$this->pdo->queryExec($binSql);
}
$ret[] = "Fetched " . number_format($numarticles) . " articles from " . $group;
} else {
$ret[] = "No headers found on " . $group;
continue;
}
} else {
$ret[] = "Can't get parts from server (msgs not array) on " . $group;
continue;
}
if ($rangeEnd == $groupEnd) {
$done = true;
}
$rangeStart = $rangeEnd + 1;
}
}
}
$nntp->doQuit();
return $ret;
}
