/** * Backfill all the groups up to user specified time/date. * * @param string $groupName * @param string|int $articles * @param string $type * * @return void */ public function backfillAllGroups($groupName = '', $articles = '', $type = '') { $res = []; if ($groupName !== '') { $grp = $this->_groups->getByName($groupName); if ($grp) { $res = [$grp]; } } else { if ($type === 'normal' || $type === '') { $res = $this->_groups->getActiveBackfill(); } else { if ($type === 'date') { $res = $this->_groups->getActiveByDateBackfill(); } } } $groupCount = count($res); if ($groupCount > 0) { $counter = 1; $allTime = microtime(true); $dMessage = 'Backfilling: ' . $groupCount . ' group(s) - Using compression? ' . ($this->_compressedHeaders ? 'Yes' : 'No'); if ($this->_debug) { $this->_debugging->log(get_class(), __FUNCTION__, $dMessage, Logger::LOG_INFO); } if ($this->_echoCLI) { $this->pdo->log->doEcho($this->pdo->log->header($dMessage), true); } $this->_binaries = new Binaries(['NNTP' => $this->_nntp, 'Echo' => $this->_echoCLI, 'Settings' => $this->pdo, 'Groups' => $this->_groups]); if ($articles !== '' && !is_numeric($articles)) { $articles = 20000; } // Loop through groups. foreach ($res as $groupArr) { if ($groupName === '') { $dMessage = "Starting group " . $counter . ' of ' . $groupCount; if ($this->_debug) { $this->_debugging->log(get_class(), __FUNCTION__, $dMessage, Logger::LOG_INFO); } if ($this->_echoCLI) { $this->pdo->log->doEcho($this->pdo->log->header($dMessage), true); } } $this->backfillGroup($groupArr, $groupCount - $counter, $articles); $counter++; } $dMessage = 'Backfilling completed in ' . number_format(microtime(true) - $allTime, 2) . " seconds."; if ($this->_debug) { $this->_debugging->log(get_class(), __FUNCTION__, $dMessage, Logger::LOG_INFO); } if ($this->_echoCLI) { $this->pdo->log->doEcho($this->pdo->log->primary($dMessage)); } } else { $dMessage = "No groups specified. Ensure groups are added to nZEDb's database for updating."; if ($this->_debug) { $this->_debugging->log(get_class(), __FUNCTION__, $dMessage, Logger::LOG_FATAL); } if ($this->_echoCLI) { $this->pdo->log->doEcho($this->pdo->log->warning($dMessage), true); } } }
/** * Main method for creating releases/NZB files from collections. * * @param int $categorize * @param int $postProcess * @param string $groupName (optional) * @param \nzedb\NNTP $nntp * @param bool $echooutput * * @return int */ public function processReleases($categorize, $postProcess, $groupName, &$nntp, $echooutput) { $this->echoCLI = $echooutput && nZEDb_ECHOCLI; $groupID = ''; if (!empty($groupName)) { $groupInfo = $this->groups->getByName($groupName); $groupID = $groupInfo['id']; } $processReleases = microtime(true); if ($this->echoCLI) { $this->pdo->log->doEcho($this->pdo->log->header("Starting release update process (" . date('Y-m-d H:i:s') . ")"), true); } if (!file_exists($this->pdo->getSetting('nzbpath'))) { if ($this->echoCLI) { $this->pdo->log->doEcho($this->pdo->log->error('Bad or missing nzb directory - ' . $this->pdo->getSetting('nzbpath')), true); } return 0; } $this->processIncompleteCollections($groupID); $this->processCollectionSizes($groupID); $this->deleteUnwantedCollections($groupID); $DIR = nZEDb_MISC; $PYTHON = shell_exec('which python3 2>/dev/null'); $PYTHON = empty($PYTHON) ? 'python -OOu' : 'python3 -OOu'; $totalReleasesAdded = 0; do { $releasesCount = $this->createReleases($groupID); $totalReleasesAdded += $releasesCount['added']; $nzbFilesAdded = $this->createNZBs($groupID); if ($this->processRequestIDs === 0) { $this->processRequestIDs($groupID, 5000, true); } else { if ($this->processRequestIDs === 1) { $this->processRequestIDs($groupID, 5000, true); $this->processRequestIDs($groupID, 1000, false); } else { if ($this->processRequestIDs === 2) { $requestIDTime = time(); if ($this->echoCLI) { $this->pdo->log->doEcho($this->pdo->log->header("Process Releases -> Request ID Threaded lookup.")); } passthru("{$PYTHON} {$DIR}update/python/requestid_threaded.py"); if ($this->echoCLI) { $this->pdo->log->doEcho($this->pdo->log->primary("\nReleases updated in " . $this->consoleTools->convertTime(time() - $requestIDTime))); } } } } $this->categorizeReleases($categorize, $groupID); $this->postProcessReleases($postProcess, $nntp); $this->deleteCollections($groupID); // This loops as long as the number of releases or nzbs added was >= the limit (meaning there are more waiting to be created) } while ($releasesCount['added'] + $releasesCount['dupes'] >= $this->releaseCreationLimit || $nzbFilesAdded >= $this->releaseCreationLimit); $this->deletedReleasesByGroup($groupID); $this->deleteReleases(); //Print amount of added releases and time it took. if ($this->echoCLI && $this->tablePerGroup === false) { $countID = $this->pdo->queryOneRow('SELECT COUNT(id) AS count FROM collections ' . (!empty($groupID) ? ' WHERE group_id = ' . $groupID : '')); $this->pdo->log->doEcho($this->pdo->log->primary('Completed adding ' . number_format($totalReleasesAdded) . ' releases in ' . $this->consoleTools->convertTime(number_format(microtime(true) - $processReleases, 2)) . '. ' . number_format($countID === false ? 0 : $countID['count']) . ' collections waiting to be created (still incomplete or in queue for creation)'), true); } return $totalReleasesAdded; }
(new RequestIDLocal(['Echo' => true]))->lookupRequestIDs(['GroupID' => $options[2], 'limit' => 5000]); } break; /* Update a single group's article headers. * * $options[2] => (string) Group name. */ /* Update a single group's article headers. * * $options[2] => (string) Group name. */ case 'update_group_headers': $pdo = new Settings(); $nntp = nntp($pdo); $groups = new Groups(['Settings' => $pdo]); $groupMySQL = $groups->getByName($options[2]); (new Binaries(['NNTP' => $nntp, 'Groups' => $groups, 'Settings' => $pdo]))->updateGroup($groupMySQL); break; // Do a single group (update_binaries/backFill/update_releases/postprocess). // $options[2] => (int)groupID, group to work on // Do a single group (update_binaries/backFill/update_releases/postprocess). // $options[2] => (int)groupID, group to work on case 'update_per_group': if (is_numeric($options[2])) { $pdo = new Settings(); // Get the group info from MySQL. $groupMySQL = $pdo->queryOneRow(sprintf('SELECT * FROM groups WHERE id = %d', $options[2])); if ($groupMySQL === false) { exit('ERROR: Group not found with ID ' . $options[2] . PHP_EOL); } // Connect to NNTP.
<?php /* Argument 1 is optional string, group name. Or numeric, number of header max to download. * Argument 2 is optional int, max number of headers to download. */ require_once dirname(__FILE__) . '/config.php'; use nzedb\Binaries; use nzedb\Groups; use nzedb\NNTP; use nzedb\db\Settings; $pdo = new Settings(); // Create the connection here and pass $nntp = new NNTP(['Settings' => $pdo]); if ($nntp->doConnect() !== true) { exit($pdo->log->error("Unable to connect to usenet.")); } $binaries = new Binaries(['NNTP' => $nntp, 'Settings' => $pdo]); if (isset($argv[1]) && !is_numeric($argv[1])) { $groupName = $argv[1]; echo $pdo->log->header("Updating group: {$groupName}"); $grp = new Groups(['Settings' => $pdo]); $group = $grp->getByName($groupName); if (is_array($group)) { $binaries->updateGroup($group, isset($argv[2]) && is_numeric($argv[2]) && $argv[2] > 0 ? $argv[2] : 0); } } else { $binaries->updateAllGroups(isset($argv[1]) && is_numeric($argv[1]) && $argv[1] > 0 ? $argv[1] : 0); }