/**
* Update all active groups categories and descriptions.
*/
function backfillAllGroups($groupName = '', $backfillDate = null, $regexOnly = false)
{
$n = $this->n;
$groups = new Groups();
$res = false;
if ($groupName != '') {
$grp = $groups->getByName($groupName);
if ($grp) {
$res = array($grp);
}
} else {
$res = $groups->getActive();
}
if ($res) {
foreach ($res as $groupArr) {
$this->backfillGroup($groupArr, $backfillDate, $regexOnly);
}
} else {
echo "No groups specified. Ensure groups are added to newznab's database for updating.{$n}";
}
}
function backfillAllGroups($groupName = '', $backfillDate = null)
{
$n = $this->n;
$groups = new Groups();
if ($groupName != '') {
$grp = $groups->getByName($groupName);
if ($grp) {
$res = array($grp);
}
} else {
$res = $groups->getActive();
}
if ($res) {
$nntp = new Nntp();
$nntp->doConnect();
foreach ($res as $groupArr) {
$this->backfillGroup($nntp, $groupArr, $backfillDate);
}
$nntp->doQuit();
} else {
echo "No groups specified. Ensure groups are added to newznab's database for updating.{$n}";
}
}
/**
* @param int $categorize
* @param int $postProcess
* @param string $groupName (optional)
* @param \NNTP $nntp
* @param bool $echooutput
*
* @return int
*/
public function processReleases($categorize, $postProcess, $groupName, &$nntp, $echooutput)
{
$this->echoCLI = $echooutput && NN_ECHOCLI;
$page = new Page();
$groupID = '';
$s = new Sites();
echo $s->getLicense();
if (!empty($groupName)) {
$groupInfo = $this->groups->getByName($groupName);
$groupID = $groupInfo['id'];
}
$processReleases = microtime(true);
if ($this->echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->header("Starting release update process (" . date('Y-m-d H:i:s') . ")"), true);
}
if (!file_exists($page->settings->getSetting('nzbpath'))) {
$this->pdo->log->doEcho($this->pdo->log->primary('Bad or missing nzb directory - ' . $page->settings->getSetting('nzbpath')), true);
return -1;
}
$this->checkRegexesUptoDate($page->settings->getSetting('latestregexurl'), $page->settings->getSetting('latestregexrevision'), $page->settings->getSetting('newznabID'));
$this->applyRegex($groupID);
$this->processIncompleteBinaries($groupID);
$DIR = NN_MISC;
$PYTHON = shell_exec('which python3 2>/dev/null');
$PYTHON = empty($PYTHON) ? 'python -OOu' : 'python3 -OOu';
$processRequestIDs = (int) $page->settings->getSetting('lookup_reqids');
$consoleTools = new ConsoleTools(['ColorCLI' => $this->pdo->log]);
$totalReleasesAdded = 0;
do {
$releasesCount = $this->createReleases($groupID);
$totalReleasesAdded += $releasesCount['added'];
if ($processRequestIDs === 0) {
$this->processRequestIDs($groupID, 5000, true);
} else {
if ($processRequestIDs === 1) {
$this->processRequestIDs($groupID, 5000, true);
$this->processRequestIDs($groupID, 1000, false);
} else {
if ($processRequestIDs === 2) {
$requestIDTime = time();
if ($this->echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->header("Process Releases -> Request id Threaded lookup."));
}
passthru("{$PYTHON} {$DIR}update_scripts/nix_scripts/tmux/python/requestid_threaded.py");
if ($this->echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->primary("\nReleases updated in " . $consoleTools->convertTime(time() - $requestIDTime)));
}
}
}
}
$this->categorizeReleases($categorize, $groupID);
$this->postProcessReleases($postProcess, $nntp);
$this->deleteBinaries($groupID);
// This loops as long as there were releases created or 3 loops, otherwise, you could loop indefinately
} while ($releasesCount['added'] + $releasesCount['dupes'] >= $this->releaseCreationLimit);
$this->deletedReleasesByGroup($groupID);
$this->deleteReleases();
//
// User/Request housekeeping, should ideally move this to its own section, but it needs to be done automatically.
//
$users = new Users();
$users->pruneRequestHistory($page->settings->getSetting('userdownloadpurgedays'));
//Print amount of added releases and time it took.
if ($this->echoCLI && $this->tablePerGroup === false) {
$countID = $this->pdo->queryOneRow('SELECT COUNT(id) AS count FROM binaries ' . (!empty($groupID) ? ' WHERE groupid = ' . $groupID : ''));
$this->pdo->log->doEcho($this->pdo->log->primary('Completed adding ' . number_format($totalReleasesAdded) . ' releases in ' . $this->consoleTools->convertTime(number_format(microtime(true) - $processReleases, 2)) . '. ' . number_format($countID === false ? 0 : $countID['count']) . ' binaries waiting to be converted (still incomplete or in queue for creation)'), true);
}
return $totalReleasesAdded;
}
<?php
/* Argument 1 is optional string, group name. Or numeric, number of header max to download.
* Argument 2 is optional int, max number of headers to download.
*/
require_once dirname(__FILE__) . '/config.php';
$pdo = new \nzedb\db\Settings();
// Create the connection here and pass
$nntp = new NNTP(['Settings' => $pdo]);
if ($nntp->doConnect() !== true) {
exit($pdo->log->error("Unable to connect to usenet."));
}
$binaries = new Binaries(['NNTP' => $nntp, 'Settings' => $pdo]);
if (isset($argv[1]) && !is_numeric($argv[1])) {
$groupName = $argv[1];
echo $pdo->log->header("Updating group: {$groupName}");
$grp = new Groups(['Settings' => $pdo]);
$group = $grp->getByName($groupName);
if (is_array($group)) {
$binaries->updateGroup($group, isset($argv[2]) && is_numeric($argv[2]) && $argv[2] > 0 ? $argv[2] : 0);
}
} else {
$binaries->updateAllGroups(isset($argv[1]) && is_numeric($argv[1]) && $argv[1] > 0 ? $argv[1] : 0);
}
/**
* Backfill all the groups up to user specified time/date.
*
* @param string $groupName
* @param string|int $articles
* @param string $type
*
* @return void
*/
public function backfillAllGroups($groupName = '', $articles = '', $type = '')
{
$res = [];
if ($groupName !== '') {
$grp = $this->_groups->getByName($groupName);
if ($grp) {
$res = [$grp];
}
} else {
if ($type === 'normal' || $type === '') {
$res = $this->_groups->getActiveBackfill();
} else {
if ($type === 'date') {
$res = $this->_groups->getActiveByDateBackfill();
}
}
}
$groupCount = count($res);
if ($groupCount > 0) {
$counter = 1;
$allTime = microtime(true);
$dMessage = 'Backfilling: ' . $groupCount . ' group(s) - Using compression? ' . ($this->_compressedHeaders ? 'Yes' : 'No');
if ($this->_debug) {
$this->_debugging->log('Backfill', "backfillAllGroups", $dMessage, Logger::LOG_INFO);
}
if ($this->_echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->header($dMessage), true);
}
$this->_binaries = new \Binaries(['NNTP' => $this->_nntp, 'Echo' => $this->_echoCLI, 'Settings' => $this->pdo, 'Groups' => $this->_groups]);
if ($articles !== '' && !is_numeric($articles)) {
$articles = 20000;
}
// Loop through groups.
foreach ($res as $groupArr) {
if ($groupName === '') {
$dMessage = "Starting group " . $counter . ' of ' . $groupCount;
if ($this->_debug) {
$this->_debugging->log('Backfill', "backfillAllGroups", $dMessage, Logger::LOG_INFO);
}
if ($this->_echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->header($dMessage), true);
}
}
$this->backfillGroup($groupArr, $groupCount - $counter, $articles);
$counter++;
}
$dMessage = 'Backfilling completed in ' . number_format(microtime(true) - $allTime, 2) . " seconds.";
if ($this->_debug) {
$this->_debugging->log('Backfill', "backfillAllGroups", $dMessage, Logger::LOG_INFO);
}
if ($this->_echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->primary($dMessage));
}
} else {
$dMessage = "No groups specified. Ensure groups are added to nZEDb's database for updating.";
if ($this->_debug) {
$this->_debugging->log('Backfill', "backfillAllGroups", $dMessage, Logger::LOG_FATAL);
}
if ($this->_echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->warning($dMessage), true);
}
}
}
<?php
require_once dirname(__FILE__) . "/config.php";
$time = TIME();
$c = new ColorCLI();
if (isset($argv[1])) {
$group = $argv[1];
echo $c->header("Updating group {$group}");
$g = new Groups();
$group = $g->getByName($group);
$bin = new Binaries();
$bin->updateGroup($group);
} else {
$binaries = new Binaries();
$binaries->updateAllGroups();
}
function relativeTime($_time)
{
$d = array();
$d[0] = array(1, "sec");
$d[1] = array(60, "min");
$d[2] = array(3600, "hr");
$d[3] = array(86400, "day");
$d[4] = array(31104000, "yr");
$w = array();
$return = "";
$now = TIME();
$diff = $now - $_time;
$secondsLeft = $diff;
for ($i = 4; $i > -1; $i--) {
$w[$i] = intval($secondsLeft / $d[$i][0]);
(new \RequestIDLocal(['Echo' => true]))->lookupRequestIDs(['GroupID' => $options[2], 'limit' => 5000]);
}
break;
/* Update a single group's article headers.
*
* $options[2] => (string) Group name.
*/
/* Update a single group's article headers.
*
* $options[2] => (string) Group name.
*/
case 'update_group_headers':
$pdo = new Settings();
$nntp = nntp($pdo);
$groups = new \Groups();
$groupMySQL = $groups->getByName($options[2]);
(new \Binaries(['NNTP' => $nntp, 'Groups' => $groups, 'Settings' => $pdo]))->updateGroup($groupMySQL);
break;
// Do a single group (update_binaries/backFill/update_releases/postprocess).
// $options[2] => (int)groupid, group to work on
// Do a single group (update_binaries/backFill/update_releases/postprocess).
// $options[2] => (int)groupid, group to work on
case 'update_per_group':
if (is_numeric($options[2])) {
$pdo = new Settings();
// Get the group info from MySQL.
$groupMySQL = $pdo->queryOneRow(sprintf('SELECT * FROM groups WHERE id = %d', $options[2]));
if ($groupMySQL === false) {
exit('ERROR: Group not found with id ' . $options[2] . PHP_EOL);
}
// Connect to NNTP.
/**
* Main method for creating releases/NZB files from collections.
*
* @param int $categorize
* @param int $postProcess
* @param string $groupName (optional)
* @param NNTP $nntp
* @param bool $echooutput
*
* @return int
*/
public function processReleases($categorize, $postProcess, $groupName, &$nntp, $echooutput)
{
$this->echoCLI = $echooutput && nZEDb_ECHOCLI;
$groupID = '';
if (!empty($groupName)) {
$groupInfo = $this->groups->getByName($groupName);
$groupID = $groupInfo['id'];
}
$processReleases = microtime(true);
if ($this->echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->header("Starting release update process (" . date('Y-m-d H:i:s') . ")"), true);
}
if (!file_exists($this->pdo->getSetting('nzbpath'))) {
if ($this->echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->error('Bad or missing nzb directory - ' . $this->pdo->getSetting('nzbpath')), true);
}
return 0;
}
$this->processIncompleteCollections($groupID);
$this->processCollectionSizes($groupID);
$this->deleteUnwantedCollections($groupID);
$DIR = nZEDb_MISC;
$PYTHON = shell_exec('which python3 2>/dev/null');
$PYTHON = empty($PYTHON) ? 'python -OOu' : 'python3 -OOu';
$totalReleasesAdded = $loops = 0;
do {
$releasesAdded = $this->createReleases($groupID);
$totalReleasesAdded += $releasesAdded;
$nzbFilesAdded = $this->createNZBs($groupID);
if ($this->processRequestIDs === 0) {
$this->processRequestIDs($groupID, 5000, true);
} else {
if ($this->processRequestIDs === 1) {
$this->processRequestIDs($groupID, 5000, true);
$this->processRequestIDs($groupID, 1000, false);
} else {
if ($this->processRequestIDs === 2) {
$requestIDTime = time();
if ($this->echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->header("Process Releases -> Request ID Threaded lookup."));
}
passthru("{$PYTHON} {$DIR}update/python/requestid_threaded.py");
if ($this->echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->primary("\nReleases updated in " . $this->consoleTools->convertTime(time() - $requestIDTime)));
}
}
}
}
$this->categorizeReleases($categorize, $groupID);
$this->postProcessReleases($postProcess, $nntp);
$this->deleteCollections($groupID);
// This loops as long as there were releases created or 3 loops, otherwise, you could loop indefinately
} while (($nzbFilesAdded > 0 || $releasesAdded > 0) && $loops++ < 3);
$this->deletedReleasesByGroup($groupID);
$this->deleteReleases();
//Print amount of added releases and time it took.
if ($this->echoCLI && $this->tablePerGroup === false) {
$countID = $this->pdo->queryOneRow('SELECT COUNT(id) AS count FROM collections ' . (!empty($groupID) ? ' WHERE group_id = ' . $groupID : ''));
$this->pdo->log->doEcho($this->pdo->log->primary('Completed adding ' . number_format($totalReleasesAdded) . ' releases in ' . $this->consoleTools->convertTime(number_format(microtime(true) - $processReleases, 2)) . '. ' . number_format($countID === false ? 0 : $countID['count']) . ' collections waiting to be created (still incomplete or in queue for creation)'), true);
}
return $totalReleasesAdded;
}
