<?php
require_once './config.php';
use nzedb\Binaries;
use nzedb\Category;
$page = new AdminPage();
$bin = new Binaries(['Settings' => $page->settings]);
$error = '';
$regex = ['id' => '', 'groupname' => '', 'regex' => '', 'description' => ''];
switch (isset($_REQUEST['action']) ? $_REQUEST['action'] : 'view') {
case 'submit':
if ($_POST["groupname"] == '') {
$error = "Group must be a valid usenet group";
break;
}
if ($_POST["regex"] == '') {
$error = "Regex cannot be empty";
break;
}
if ($_POST["id"] == '') {
$bin->addBlacklist($_POST);
} else {
$ret = $bin->updateBlacklist($_POST);
}
header("Location:" . WWW_TOP . "/binaryblacklist-list.php");
break;
case 'addtest':
if (isset($_GET['regex']) && isset($_GET['groupname'])) {
$regex += ['groupname' => $_GET['groupname'], 'regex' => $_GET['regex'], 'ordinal' => '1', 'status' => '1'];
}
break;
<?php
require_once './config.php';
use nzedb\Binaries;
use nzedb\Category;
$page = new AdminPage();
$bin = new Binaries(['Settings' => $page->settings]);
$id = 0;
// Set the current action.
$action = isset($_REQUEST['action']) ? $_REQUEST['action'] : 'view';
switch ($action) {
case 'submit':
if ($_POST["groupname"] == "") {
$page->smarty->assign('error', "Group must be a valid usenet group");
break;
}
if ($_POST["regex"] == "") {
$page->smarty->assign('error', "Regex cannot be empty");
break;
}
if ($_POST["id"] == "") {
$bin->addBlacklist($_POST);
} else {
$ret = $bin->updateBlacklist($_POST);
}
header("Location:" . WWW_TOP . "/binaryblacklist-list.php");
break;
case 'addtest':
if (isset($_GET['regex']) && isset($_GET['groupname'])) {
$r = ['groupname' => $_GET['groupname'], 'regex' => $_GET['regex'], 'ordinal' => '1', 'status' => '1'];
$page->smarty->assign('regex', $r);
* $options[3] => (string) Group name.
* $options[4] => (int) First article number in range.
* $options[5] => (int) Last article number in range.
* $options[6] => (int) Number of threads.
*/
case 'get_range':
$pdo = new Settings();
$nntp = nntp($pdo);
$groups = new Groups(['Settings' => $pdo]);
$groupMySQL = $groups->getByName($options[3]);
if ($nntp->isError($nntp->selectGroup($groupMySQL['name']))) {
if ($nntp->isError($nntp->dataError($nntp, $groupMySQL['name']))) {
return;
}
}
$binaries = new Binaries(['NNTP' => $nntp, 'Settings' => $pdo, 'Groups' => $groups]);
$return = $binaries->scan($groupMySQL, $options[4], $options[5], $pdo->getSetting('safepartrepair') == 1 ? 'update' : 'backfill');
if (empty($return)) {
exit;
}
$columns = [];
switch ($options[2]) {
case 'binaries':
if ($return['lastArticleNumber'] <= $groupMySQL['last_record']) {
exit;
}
$columns[1] = sprintf('last_record_postdate = %s', $pdo->from_unixtime(is_numeric($return['lastArticleDate']) ? $return['lastArticleDate'] : strtotime($return['lastArticleDate'])));
$columns[2] = sprintf('last_record = %s', $return['lastArticleNumber']);
$query = sprintf('UPDATE groups SET %s, %s, last_updated = NOW() WHERE id = %d AND last_record < %s', $columns[1], $columns[2], $groupMySQL['id'], $return['lastArticleNumber']);
break;
case 'backfill':
/**
* Backfill single group.
*
* @param array $groupArr
* @param int $left
* @param int|string $articles
*
* @return void
*/
public function backfillGroup($groupArr, $left, $articles = '')
{
// Start time for this group.
$startGroup = microtime(true);
$groupName = str_replace('alt.binaries', 'a.b', $groupArr['name']);
// If our local oldest article 0, it means we never ran update_binaries on the group.
if ($groupArr['first_record'] <= 0) {
$dMessage = "You need to run update_binaries on " . $groupName . ". Otherwise the group is dead, you must disable it.";
if ($this->_debug) {
$this->_debugging->log(get_class(), __FUNCTION__, $dMessage, Logger::LOG_ERROR);
}
if ($this->_echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->error($dMessage));
}
return;
}
// Select group, here, only once
$data = $this->_nntp->selectGroup($groupArr['name']);
if ($this->_nntp->isError($data)) {
$data = $this->_nntp->dataError($this->_nntp, $groupArr['name']);
if ($this->_nntp->isError($data)) {
return;
}
}
if ($this->_echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->primary('Processing ' . $groupName), true);
}
// Check if this is days or post backfill.
$postCheck = $articles === '' ? false : true;
// Get target post based on date or user specified number.
$targetpost = (string) ($postCheck ? round($groupArr['first_record'] - $articles) : $this->_binaries->daytopost($groupArr['backfill_target'], $data));
// Check if target post is smaller than server's oldest, set it to oldest if so.
if ($targetpost < $data['first']) {
$targetpost = $data['first'];
}
// Check if our target post is newer than our oldest post or if our local oldest article is older than the servers oldest.
if ($targetpost >= $groupArr['first_record'] || $groupArr['first_record'] <= $data['first']) {
$dMessage = "We have hit the maximum we can backfill for " . $groupName . ($this->_disableBackfillGroup ? ", disabling backfill on it." : ", skipping it, consider disabling backfill on it.");
if ($this->_debug) {
$this->_debugging->log(get_class(), __FUNCTION__, $dMessage, Logger::LOG_NOTICE);
}
if ($this->_disableBackfillGroup) {
$this->pdo->queryExec(sprintf('
UPDATE groups
SET backfill = 0
WHERE id = %d', $groupArr['id']));
}
if ($this->_echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->notice($dMessage), true);
}
return;
}
if ($this->_echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->primary('Group ' . $groupName . "'s oldest article is " . number_format($data['first']) . ', newest is ' . number_format($data['last']) . ".\nOur target article is " . number_format($targetpost) . '. Our oldest article is article ' . number_format($groupArr['first_record']) . '.'));
}
// Set first and last, moving the window by max messages.
$last = (string) ($groupArr['first_record'] - 1);
// Set the initial "chunk".
$first = (string) ($last - $this->_binaries->messageBuffer + 1);
// Just in case this is the last chunk we needed.
if ($targetpost > $first) {
$first = $targetpost;
}
$done = false;
while ($done === false) {
if ($this->_echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->set256('Yellow') . "\nGetting " . number_format($last - $first + 1) . " articles from " . $groupName . ", " . $left . " group(s) left. (" . number_format($first - $targetpost) . " articles in queue)." . $this->pdo->log->rsetColor(), true);
}
flush();
$lastMsg = $this->_binaries->scan($groupArr, $first, $last, $this->_safePartRepair);
// Get the oldest date.
if (isset($lastMsg['firstArticleDate'])) {
// Try to get it from the oldest pulled article.
$newdate = strtotime($lastMsg['firstArticleDate']);
} else {
// If above failed, try to get it with postdate method.
$newdate = $this->_binaries->postdate($first, $data);
}
$this->pdo->queryExec(sprintf('
UPDATE groups
SET first_record_postdate = %s, first_record = %s, last_updated = NOW()
WHERE id = %d', $this->pdo->from_unixtime($newdate), $this->pdo->escapeString($first), $groupArr['id']));
if ($first == $targetpost) {
$done = true;
} else {
// Keep going: set new last, new first, check for last chunk.
$last = (string) ($first - 1);
$first = (string) ($last - $this->_binaries->messageBuffer + 1);
if ($targetpost > $first) {
$first = $targetpost;
}
}
}
if ($this->_echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->primary(PHP_EOL . 'Group ' . $groupName . ' processed in ' . number_format(microtime(true) - $startGroup, 2) . " seconds."), true);
}
}
<?php
// To troubleshoot what's actually on usenet.
require_once realpath(dirname(dirname(dirname(__DIR__))) . DIRECTORY_SEPARATOR . 'indexer.php');
use nzedb\Binaries;
use nzedb\ColorCLI;
use nzedb\NNTP;
$cli = new ColorCLI();
if (!isset($argv[2]) || !is_numeric($argv[2])) {
exit($cli->error("\nTest your nntp connection, get group information and postdate for specific article.\n\n" . "php {$argv['0']} alt.binaries.teevee 595751142 ...: To test nntp on alt.binaries.teevee with artivle 595751142.\n"));
}
$nntp = new NNTP();
if ($nntp->doConnect() !== true) {
exit;
}
$first = $argv[2];
$group = $argv[1];
// Select a group.
$groupArr = $nntp->selectGroup($group);
print_r($groupArr);
// Insert actual local part numbers here.
$msg = $nntp->getXOVER($first . '-' . $first);
// Print out the array of headers.
print_r($msg);
// get postdate for an article
$binaries = new Binaries(['NNTP' => $nntp]);
$newdate = $binaries->postdate($first, $groupArr);
echo $cli->primary("The posted date for " . $group . ", article " . $first . " is " . date('Y-m-d H:i:s', $newdate));
<?php
require_once dirname(__FILE__) . '/../../../../www/config.php';
use nzedb\Binaries;
use nzedb\Categorize;
use nzedb\ConsoleTools;
use nzedb\NZB;
use nzedb\ReleaseCleaning;
use nzedb\Releases;
use nzedb\db\Settings;
$pdo = new Settings();
if (!isset($argv[1])) {
exit($pdo->log->error("\nYou must supply a path as the first argument. Two additional, optional arguments can also be used.\n\n" . "php {$argv['0']} /path/to/import true 1000 ...: To import using the filename as release searchname, limited to 1000\n" . "php {$argv['0']} /path/to/import false ...: To import using the subject as release searchname\n"));
}
$consoleTools = new ConsoleTools(['ColorCLI' => $pdo->log]);
$binaries = new Binaries(['Settings' => $pdo]);
$crosspostt = $pdo->getSetting('crossposttime');
$crosspostt = !empty($crosspostt) ? $crosspostt : 2;
$releasecleaning = new ReleaseCleaning($pdo);
$categorize = new Categorize(['Settings' => $pdo]);
$nzb = new NZB($pdo);
$releases = new Releases(['Settings' => $pdo]);
$nzbsperhour = $nzbSkipped = $maxtoprocess = 0;
if (isset($argv[2]) && is_numeric($argv[2])) {
exit($pdo->log->error("\nTo use a max number to process, it must be the third argument. \nTo run:\nphp nzb-import.php /path [true, false] 1000\n"));
}
if (!isset($argv[2])) {
$pieces = explode(" ", $argv[1]);
$usenzbname = isset($pieces[1]) && $pieces[1] == 'true' ? true : false;
$path = $pieces[0];
} else {
function daytopost($nntp, $group, $days, $debug = true, $bfcheck = true)
{
global $pdo;
$st = false;
if ($debug && $bfcheck) {
echo $pdo->log->primary('Finding start and end articles for ' . $group . '.');
}
if (!isset($nntp)) {
$nntp = new NNTP(['Settings' => $pdo]);
if ($nntp->doConnect(false) !== true) {
return;
}
$st = true;
}
$binaries = new Binaries(['NNTP' => $nntp, 'Settings' => $pdo]);
$data = $nntp->selectGroup($group);
if ($nntp->isError($data)) {
$data = $nntp->dataError($nntp, $group, false);
if ($data === false) {
return;
}
}
// Goal timestamp.
$goaldate = date('U') - 86400 * $days;
$totalnumberofarticles = $data['last'] - $data['first'];
$upperbound = $data['last'];
$lowerbound = $data['first'];
if ($debug && $bfcheck) {
echo $pdo->log->header('Total Articles: ' . number_format($totalnumberofarticles) . ' Newest: ' . number_format($upperbound) . ' Oldest: ' . number_format($lowerbound));
}
if ($data['last'] == PHP_INT_MAX) {
exit($pdo->log->error("Group data is coming back as php's max value. You should not see this since we use a patched Net_NNTP that fixes this bug."));
}
$firstDate = $binaries->postdate($data['first'], $data);
$lastDate = $binaries->postdate($data['last'], $data);
if ($goaldate < $firstDate && $bfcheck) {
if ($st === true) {
$nntp->doQuit();
}
echo $pdo->log->warning("The oldest post indexed from {$days} day(s) ago is older than the first article stored on your news server.\nSetting to First available article of (date('r', {$firstDate}) or daysOld({$firstDate}) days).");
return $data['first'];
} else {
if ($goaldate > $lastDate && $bfcheck) {
if ($st === true) {
$nntp->doQuit();
}
echo $pdo->log->error("ERROR: The oldest post indexed from {$days} day(s) ago is newer than the last article stored on your news server.\nTo backfill this group you need to set Backfill Days to at least ceil(daysOld({$lastDate})+1) days (date('r', {$lastDate}-86400).");
return '';
}
}
if ($debug && $bfcheck) {
echo $pdo->log->primary("Searching for postdates.\nGroup's Firstdate: " . $firstDate . ' (' . (is_int($firstDate) ? date('r', $firstDate) : 'n/a') . ").\nGroup's Lastdate: " . $lastDate . ' (' . date('r', $lastDate) . ").");
}
$interval = floor(($upperbound - $lowerbound) * 0.5);
$templowered = '';
$dateofnextone = $lastDate;
// Match on days not timestamp to speed things up.
while (daysOld($dateofnextone) < $days) {
while (($tmpDate = $binaries->postdate($upperbound - $interval, $data)) > $goaldate) {
$upperbound = $upperbound - $interval;
}
if (!$templowered) {
$interval = ceil($interval / 2);
}
$dateofnextone = $binaries->postdate($upperbound - 1, $data);
while (!$dateofnextone) {
$dateofnextone = $binaries->postdate($upperbound - 1, $data);
}
}
if ($st === true) {
$nntp->doQuit();
}
if ($bfcheck) {
echo $pdo->log->header("\nBackfill article determined to be " . $upperbound . " " . $pdo->log->setColor('Yellow') . "(" . date('r', $dateofnextone) . ")" . $pdo->log->rsetcolor());
} else {
echo $pdo->log->header('Current article determined to be ' . $upperbound . " " . $pdo->log->setColor('Yellow') . "(" . date('r', $dateofnextone) . ")" . $pdo->log->rsetcolor());
}
// which is '.daysOld($dateofnextone)." days old.\n";
return $upperbound;
}
/**
* @param object $nzbXML Reference of simpleXmlObject with NZB contents.
* @param bool|string $useNzbName Use the NZB file name as release name?
* @return bool
*
* @access protected
*/
protected function scanNZBFile(&$nzbXML, $useNzbName = false)
{
$totalFiles = $totalSize = $groupID = 0;
$isBlackListed = $groupName = $firstName = $posterName = $postDate = false;
// Go through the NZB, get the details, look if it's blacklisted, look if we have the groups.
foreach ($nzbXML->file as $file) {
$totalFiles++;
$groupID = -1;
// Get the nzb info.
if ($firstName === false) {
$firstName = (string) $file->attributes()->subject;
}
if ($posterName === false) {
$posterName = (string) $file->attributes()->poster;
}
if ($postDate === false) {
$postDate = date("Y-m-d H:i:s", (string) $file->attributes()->date);
}
// Make a fake message array to use to check the blacklist.
$msg = ["Subject" => (string) $file->attributes()->subject, "From" => (string) $file->attributes()->poster, "Message-ID" => ""];
// Get the group names, group_id, check if it's blacklisted.
$groupArr = [];
foreach ($file->groups->group as $group) {
$group = (string) $group;
// If group_id is -1 try to get a group_id.
if ($groupID === -1) {
if (array_key_exists($group, $this->allGroups)) {
$groupID = $this->allGroups[$group];
if (!$groupName) {
$groupName = $group;
}
}
}
// Add all the found groups to an array.
$groupArr[] = $group;
// Check if this NZB is blacklisted.
if ($this->binaries->isBlacklisted($msg, $group)) {
$isBlackListed = true;
break;
}
}
// If we found a group and it's not blacklisted.
if ($groupID !== -1 && !$isBlackListed) {
// Get the size of the release.
if (count($file->segments->segment) > 0) {
foreach ($file->segments->segment as $segment) {
$totalSize += (int) $segment->attributes()->bytes;
}
}
} else {
if ($isBlackListed) {
$errorMessage = "Subject is blacklisted: " . utf8_encode(trim($firstName));
} else {
$errorMessage = "No group found for " . $firstName . " (one of " . implode(', ', $groupArr) . " are missing";
}
$this->echoOut($errorMessage);
return false;
}
}
// Try to insert the NZB details into the DB.
return $this->insertNZB(['subject' => $firstName, 'useFName' => $useNzbName, 'postDate' => empty($postDate) ? date("Y-m-d H:i:s") : $postDate, 'from' => empty($posterName) ? '' : $posterName, 'group_id' => $groupID, 'groupName' => $groupName, 'totalFiles' => $totalFiles, 'totalSize' => $totalSize]);
}
$pdo = new DB();
if ($argv[1] === 'all') {
$groups = $pdo->query('SELECT * FROM groups WHERE backfill = 1');
} else {
$groups = $pdo->query(sprintf('SELECT * FROM groups WHERE name = %s', $pdo->escapeString($argv[1])));
}
if (count($groups) === 0) {
if ($argv[1] === 'all') {
exit('ERROR! No groups were found with backfill enabled!' . PHP_EOL);
} else {
exit('ERROR! Group (' . $argv[1] . ') not found!' . PHP_EOL);
}
}
$nntp = new NNTP(['Settings' => $pdo]);
$nntp->doConnect() or exit('Could not connect to Usenet!' . PHP_EOL);
$binaries = new Binaries(['NNTP' => $nntp, 'Settings' => $pdo]);
foreach ($groups as $group) {
$groupNNTP = $nntp->selectGroup($group['name']);
if ($nntp->isError($groupNNTP)) {
echo 'ERROR! Could not fetch information from NNTP for group (' . $group['name'] . ')' . PHP_EOL;
continue;
}
$postDate = $pdo->queryOneRow(sprintf('SELECT UNIX_TIMESTAMP(postdate) AS postdate FROM releases WHERE group_id = %d ORDER BY postdate ASC LIMIT 1', $group['id']));
if ($postDate === false) {
echo 'ERROR! Could not find any existing releases for group (' . $group['name'] . ')' . PHP_EOL;
continue;
}
$articleNumber = (int) $binaries->daytopost(round((time() - $postDate['postdate']) / 86400), $groupNNTP);
if ($group['last_record'] != 0 && $articleNumber >= $group['last_record']) {
echo 'ERROR! Could not determine the article number for this date: (' . $postDate['postdate'] . ') on group (' . $group['name'] . ')' . PHP_EOL;
continue;
<?php
/* Argument 1 is optional string, group name. Or numeric, number of header max to download.
* Argument 2 is optional int, max number of headers to download.
*/
require_once dirname(__FILE__) . '/config.php';
use nzedb\Binaries;
use nzedb\Groups;
use nzedb\NNTP;
use nzedb\db\Settings;
$pdo = new Settings();
// Create the connection here and pass
$nntp = new NNTP(['Settings' => $pdo]);
if ($nntp->doConnect() !== true) {
exit($pdo->log->error("Unable to connect to usenet."));
}
$binaries = new Binaries(['NNTP' => $nntp, 'Settings' => $pdo]);
if (isset($argv[1]) && !is_numeric($argv[1])) {
$groupName = $argv[1];
echo $pdo->log->header("Updating group: {$groupName}");
$grp = new Groups(['Settings' => $pdo]);
$group = $grp->getByName($groupName);
if (is_array($group)) {
$binaries->updateGroup($group, isset($argv[2]) && is_numeric($argv[2]) && $argv[2] > 0 ? $argv[2] : 0);
}
} else {
$binaries->updateAllGroups(isset($argv[1]) && is_numeric($argv[1]) && $argv[1] > 0 ? $argv[1] : 0);
}
<?php
require_once './config.php';
use nzedb\Binaries;
$page = new AdminPage();
$bin = new Binaries(['Settings' => $page->settings]);
$page->title = "Binary Black/Whitelist List";
$binlist = $bin->getBlacklist(false);
$page->smarty->assign('binlist', $binlist);
$page->content = $page->smarty->fetch('binaryblacklist-list.tpl');
$page->render();
