function getBinary($binaryId, $isNfo = false) { $db = new DB(); $bin = new Binaries(); $binary = $bin->getById($binaryId); if (!$binary) { return false; } $summary = $this->selectGroup($binary['groupname']); $message = $dec = ''; if (PEAR::isError($summary)) { echo $summary->getMessage(); return false; } $resparts = $db->query(sprintf("SELECT size, partnumber, messageID FROM parts WHERE binaryID = %d ORDER BY partnumber", $binaryId)); if (sizeof($resparts) > 1 && $isNfo === true) { echo 'NFO is more than 1 part, skipping. '; return false; } foreach ($resparts as $part) { $messageID = '<' . $part['messageID'] . '>'; $body = $this->getBody($messageID, true); if (PEAR::isError($body)) { echo 'Error fetching part number ' . $part['messageID'] . ' in ' . $binary['groupname'] . ' (Server response: ' . $body->getMessage() . ')'; return false; } $dec = $this->decodeYenc($body); if (!$dec) { echo "Yenc decode failure"; return false; } $message .= $dec; } return $message; }
/** * @param array $group * @param Binaries $binaries * * @return bool */ function dogroup($group, $binaries) { $binaries->updateGroup($group); echo "Press enter to continue, type n and press enter to quit.\n"; $cmd = trim(fgets(fopen("php://stdin", "r"))); if ($cmd == '') { return true; } else { if ($cmd == "no") { exit("Done.\n"); } else { return true; } } }
public function purge($id) { $db = new DB(); $releases = new Releases(); $binaries = new Binaries(); $this->reset($id); $rels = $db->query(sprintf("select ID from releases where groupID = %d", $id)); foreach ($rels as $rel) { $releases->delete($rel["ID"]); } $bins = $db->query(sprintf("select ID from binaries where groupID = %d", $id)); foreach ($bins as $bin) { $binaries->delete($bin["ID"]); } }
public function fetchTestBinaries($groupname, $numarticles, $clearexistingbins) { $db = new DB(); $nntp = new Nntp(); $binaries = new Binaries(); $groups = new Groups(); $ret = array(); if ($clearexistingbins == true) { $db->exec('truncate releaseregextesting'); } $nntp->doConnect(); $groupsToFetch = array(); if (preg_match('/^[a-z]{2,3}(\\.[a-z0-9\\-]+)+$/', $groupname)) { $groupsToFetch[] = array('name' => $groupname); } elseif ($groupname === 0) { $groupsToFetch = $groups->getAll(); } else { $newsgroups = $nntp->getGroups(); foreach ($newsgroups as $ngroup) { if (preg_match('/' . $groupname . '/', $ngroup['group'])) { $groupsToFetch[] = array('name' => $ngroup['group']); } } } foreach ($groupsToFetch as $groupArr) { $group = $groupArr['name']; $data = $nntp->selectGroup($group); if ($nntp->isError($data)) { $ret[] = "Could not select group (doesnt exist on USP): {$group}"; continue; } else { $rangeStart = $data['last'] - $numarticles; $rangeEnd = $groupEnd = $data['last']; $rangeTotal = $rangeEnd - $rangeStart; $done = false; while ($done === false) { if ($rangeTotal > $binaries->messagebuffer) { if ($rangeStart + $binaries->messagebuffer > $groupEnd) { $rangeEnd = $groupEnd; } else { $rangeEnd = $rangeStart + $binaries->messagebuffer; } } if ($binaries->compressedHeaders) { $msgs = $nntp->getXOverview($rangeStart . "-" . $rangeEnd, true, false); } else { $msgs = $nntp->getOverview($rangeStart . "-" . $rangeEnd, true, false); } if ($nntp->isError($msgs)) { $ret[] = "Error {$msgs->code}: {$msgs->message} on " . $group; continue 2; } $headers = array(); if (is_array($msgs)) { //loop headers, figure out parts foreach ($msgs as $msg) { if (!isset($msg['Number'])) { continue; } $msgPart = $msgTotalParts = 0; $pattern = '|\\((\\d+)[\\/](\\d+)\\)|i'; preg_match_all($pattern, $msg['Subject'], $matches, PREG_PATTERN_ORDER); $matchcnt = sizeof($matches[0]); for ($i = 0; $i < $matchcnt; $i++) { //not (int)'d here because of the preg_replace later on $msgPart = $matches[1][$i]; $msgTotalParts = $matches[2][$i]; } if (!isset($msg['Subject']) || $matchcnt == 0) { // not a binary post most likely.. continue continue; } if ((int) $msgPart > 0 && (int) $msgTotalParts > 0) { $subject = utf8_encode(trim(preg_replace('|\\(' . $msgPart . '[\\/]' . $msgTotalParts . '\\)|i', '', $msg['Subject']))); if (!isset($headers[$subject])) { $headers[$subject]['Subject'] = $subject; $headers[$subject]['From'] = $msg['From']; $headers[$subject]['Date'] = strtotime($msg['Date']); $headers[$subject]['Message-ID'] = $msg['Message-ID']; $headers[$subject]['Size'] = $msg['Bytes']; } else { $headers[$subject]['Size'] += $msg['Bytes']; } } } unset($msgs); if (isset($headers) && count($headers)) { $groupRegexes = $this->getForGroup($group); $binSetData = array(); foreach ($headers as $subject => $data) { $binData = array('name' => $subject, 'fromname' => $data['From'], 'date' => $data['Date'], 'binaryhash' => md5($subject . $data['From'] . $group), 'groupname' => $group, 'regexID' => "null", 'categoryID' => "null", 'reqID' => "null", 'blacklistID' => 0, 'size' => $data['Size'], 'relname' => "null", 'relpart' => "null", 'reltotalpart' => "null"); //Filter binaries based on black/white list if ($binaries->isBlackListed($data, $group)) { //binary is blacklisted $binData['blacklistID'] = 1; } //Apply Regexes $regexMatches = array(); foreach ($groupRegexes as $groupRegex) { $regexCheck = $this->performMatch($groupRegex, $subject, $data['From']); if ($regexCheck !== false) { $regexMatches = $regexCheck; $binData['regexID'] = $regexCheck['regexID']; $binData['categoryID'] = $regexCheck['regcatid']; $binData['reqID'] = empty($regexCheck['reqID']) ? "null" : $regexCheck['reqID']; $binData['relname'] = $regexCheck['name']; break; } } $binSetData[] = $binData; } //insert 500 bins at a time $binChunks = array_chunk($binSetData, 500); foreach ($binChunks as $binChunk) { foreach ($binChunk as $chunk) { $binParams[] = sprintf("(%s, %s, FROM_UNIXTIME(%s), %s, %s, %s, %s, %s, %d, %d, now())", $db->escapeString($chunk['name']), $db->escapeString($chunk['fromname']), $db->escapeString($chunk['date']), $db->escapeString($chunk['binaryhash']), $db->escapeString($chunk['groupname']), $chunk['regexID'], $chunk['categoryID'], $chunk['reqID'], $chunk['blacklistID'], $chunk['size']); } $binSql = "INSERT IGNORE INTO releaseregextesting (name, fromname, date, binaryhash, groupname, regexID, categoryID, reqID, blacklistID, size, dateadded) VALUES " . implode(', ', $binParams); //echo $binSql; $db->exec($binSql); } $ret[] = "Fetched " . number_format($numarticles) . " articles from " . $group; } else { $ret[] = "No headers found on " . $group; continue; } } else { $ret[] = "Can't get parts from server (msgs not array) on " . $group; continue; } if ($rangeEnd == $groupEnd) { $done = true; } $rangeStart = $rangeEnd + 1; } } } $nntp->doQuit(); return $ret; }
<?php /* Argument 1 is optional string, group name. Or numeric, number of header max to download. * Argument 2 is optional int, max number of headers to download. */ require_once dirname(__FILE__) . '/config.php'; $pdo = new \nzedb\db\Settings(); // Create the connection here and pass $nntp = new NNTP(['Settings' => $pdo]); if ($nntp->doConnect() !== true) { exit($pdo->log->error("Unable to connect to usenet.")); } $binaries = new Binaries(['NNTP' => $nntp, 'Settings' => $pdo]); if (isset($argv[1]) && !is_numeric($argv[1])) { $groupName = $argv[1]; echo $pdo->log->header("Updating group: {$groupName}"); $grp = new Groups(['Settings' => $pdo]); $group = $grp->getByName($groupName); if (is_array($group)) { $binaries->updateGroup($group, isset($argv[2]) && is_numeric($argv[2]) && $argv[2] > 0 ? $argv[2] : 0); } } else { $binaries->updateAllGroups(isset($argv[1]) && is_numeric($argv[1]) && $argv[1] > 0 ? $argv[1] : 0); }
<?php require_once WWW_DIR . "/lib/binaries.php"; $binaries = new Binaries(); if (!$users->isLoggedIn()) { $page->show403(); } $page->meta_title = "Search Binaries"; $page->meta_keywords = "search,binaries,binsearch,nzb,description,details"; $page->meta_description = "Search for Binaries"; $results = array(); $searchStr = ''; //TODO: bug here in javascript cludge to turn request in a get, rather than post is losing + signs in search query. if (isset($_REQUEST["id"])) { $searchStr = $_REQUEST["id"]; $results = $binaries->search($searchStr, 1000, $page->userdata["categoryexclusions"]); } $page->smarty->assign('search', $searchStr); $page->smarty->assign('results', $results); $page->content = $page->smarty->fetch('searchraw.tpl'); $page->render();
<?php require_once "config.php"; require_once WWW_DIR . "/lib/adminpage.php"; require_once WWW_DIR . "/lib/binaries.php"; $page = new AdminPage(); $bin = new Binaries(); $page->title = "Binary Black/Whitelist List"; $binlist = $bin->getBlacklist(false); $page->smarty->assign('binlist', $binlist); $page->content = $page->smarty->fetch('binaryblacklist-list.tpl'); $page->render();
<?php require "config.php"; require_once WWW_DIR . "/lib/binaries.php"; $binaries = new Binaries(); $binaries->updateAllGroups();
/** * @param object $nzbXML Reference of simpleXmlObject with NZB contents. * @param bool|string $useNzbName Use the NZB file name as release name? * @return bool * * @access protected */ protected function scanNZBFile(&$nzbXML, $useNzbName = false) { $totalFiles = $totalSize = $groupID = 0; $isBlackListed = $groupName = $firstName = $posterName = $postDate = false; // Go through the NZB, get the details, look if it's blacklisted, look if we have the groups. foreach ($nzbXML->file as $file) { $totalFiles++; $groupID = -1; // Get the nzb info. if ($firstName === false) { $firstName = (string) $file->attributes()->subject; } if ($posterName === false) { $posterName = (string) $file->attributes()->poster; } if ($postDate === false) { $postDate = date("Y-m-d H:i:s", (string) $file->attributes()->date); } // Make a fake message array to use to check the blacklist. $msg = ["Subject" => (string) $file->attributes()->subject, "From" => (string) $file->attributes()->poster, "Message-ID" => ""]; // Get the group names, group_id, check if it's blacklisted. $groupArr = []; foreach ($file->groups->group as $group) { $group = (string) $group; // If group_id is -1 try to get a group_id. if ($groupID === -1) { if (array_key_exists($group, $this->allGroups)) { $groupID = $this->allGroups[$group]; if (!$groupName) { $groupName = $group; } } else { $groupID = $this->groups->add(['name' => $group, 'description' => 'Added by NZBimport script.', 'backfill_target' => 0, 'first_record' => 0, 'last_record' => 0, 'active' => 0, 'backfill' => 0]); $this->allGroups[$group] = $groupID; $this->echoOut("Adding missing group: ({$group})"); } } // Add all the found groups to an array. $groupArr[] = $group; // Check if this NZB is blacklisted. if ($this->binaries->isBlacklisted($msg, $group)) { $isBlackListed = true; break; } } // If we found a group and it's not blacklisted. if ($groupID !== -1 && !$isBlackListed) { // Get the size of the release. if (count($file->segments->segment) > 0) { foreach ($file->segments->segment as $segment) { $totalSize += (int) $segment->attributes()->bytes; } } } else { if ($isBlackListed) { $errorMessage = "Subject is blacklisted: " . utf8_encode(trim($firstName)); } else { $errorMessage = "No group found for " . $firstName . " (one of " . implode(', ', $groupArr) . " are missing"; } $this->echoOut($errorMessage); return false; } } // Try to insert the NZB details into the DB. return $this->insertNZB(['subject' => $firstName, 'useFName' => $useNzbName, 'postDate' => empty($postDate) ? date("Y-m-d H:i:s") : $postDate, 'from' => empty($posterName) ? '' : $posterName, 'groupid' => $groupID, 'groupName' => $groupName, 'totalFiles' => $totalFiles, 'totalSize' => $totalSize]); }
$pdo = new nzedb\db\DB(); if ($argv[1] === 'all') { $groups = $pdo->query('SELECT * FROM groups WHERE backfill = 1'); } else { $groups = $pdo->query(sprintf('SELECT * FROM groups WHERE name = %s', $pdo->escapeString($argv[1]))); } if (count($groups) === 0) { if ($argv[1] === 'all') { exit('ERROR! No groups were found with backfill enabled!' . PHP_EOL); } else { exit('ERROR! Group (' . $argv[1] . ') not found!' . PHP_EOL); } } $nntp = new NNTP(['Settings' => $pdo]); $nntp->doConnect() or exit('Could not connect to Usenet!' . PHP_EOL); $binaries = new Binaries(['NNTP' => $nntp, 'Settings' => $pdo]); foreach ($groups as $group) { $groupNNTP = $nntp->selectGroup($group['name']); if ($nntp->isError($groupNNTP)) { echo 'ERROR! Could not fetch information from NNTP for group (' . $group['name'] . ')' . PHP_EOL; continue; } $postDate = $pdo->queryOneRow(sprintf('SELECT UNIX_TIMESTAMP(postdate) AS postdate FROM releases WHERE group_id = %d ORDER BY postdate ASC LIMIT 1', $group['id'])); if ($postDate === false) { echo 'ERROR! Could not find any existing releases for group (' . $group['name'] . ')' . PHP_EOL; continue; } $articleNumber = (int) $binaries->daytopost(round((time() - $postDate['postdate']) / 86400), $groupNNTP); if ($group['last_record'] != 0 && $articleNumber >= $group['last_record']) { echo 'ERROR! Could not determine the article number for this date: (' . $postDate['postdate'] . ') on group (' . $group['name'] . ')' . PHP_EOL; continue;
* $options[3] => (string) Group name. * $options[4] => (int) First article number in range. * $options[5] => (int) Last article number in range. * $options[6] => (int) Number of threads. */ case 'get_range': $pdo = new Settings(); $nntp = nntp($pdo); $groups = new \Groups(); $groupMySQL = $groups->getByName($options[3]); if ($nntp->isError($nntp->selectGroup($groupMySQL['name']))) { if ($nntp->isError($nntp->dataError($nntp, $groupMySQL['name']))) { return; } } $binaries = new \Binaries(['NNTP' => $nntp, 'Settings' => $pdo, 'Groups' => $groups]); $return = $binaries->scan($groupMySQL, $options[4], $options[5], $pdo->getSetting('safepartrepair') == 1 ? 'update' : 'backfill'); if (empty($return)) { exit; } $columns = []; switch ($options[2]) { case 'binaries': if ($return['lastArticleNumber'] <= $groupMySQL['last_record']) { exit; } $columns[1] = sprintf('last_record_postdate = %s', $pdo->from_unixtime(is_numeric($return['lastArticleDate']) ? $return['lastArticleDate'] : strtotime($return['lastArticleDate']))); $columns[2] = sprintf('last_record = %s', $return['lastArticleNumber']); $query = sprintf('UPDATE groups SET %s, %s, last_updated = NOW() WHERE id = %d AND last_record < %s', $columns[1], $columns[2], $groupMySQL['id'], $return['lastArticleNumber']); break; case 'backfill':
<?php require_once "config.php"; require_once WWW_DIR . "/lib/adminpage.php"; require_once WWW_DIR . "/lib/releases.php"; require_once WWW_DIR . "/lib/users.php"; require_once WWW_DIR . "/lib/binaries.php"; $page = new AdminPage(); $users = new Users(); if (!$users->isLoggedIn()) { $page->show403(); } if (isset($_GET["id"])) { $releases = new Releases(); $rel = $releases->getByGuid($_GET["id"]); if (!$rel) { $page->show404(); } $binaries = new Binaries(); $data = $binaries->getForReleaseId($rel["ID"]); $page->smarty->assign('rel', $rel); $page->smarty->assign('binaries', $data); $page->title = "File List"; $page->meta_title = "View Nzb file list"; $page->meta_keywords = "view,nzb,file,list,description,details"; $page->meta_description = "View Nzb File List"; $page->content = $page->smarty->fetch('release-files.tpl'); $page->render(); }
<?php require_once "config.php"; require_once WWW_DIR . "/lib/framework/db.php"; require_once WWW_DIR . "/lib/binaries.php"; $bin = new Binaries(); $articles = $bin->search("mess"); foreach ($articles as $article) { $pattern = '/()(.*)(\\d{2,3}\\/\\d{1,3})/i'; if (!preg_match($pattern, rtrim($article["name"]), $matches)) { echo "Not matched: " . $article["name"] . "<br/>"; } else { print_r($matches) . ""; } }
<?php // To troubleshoot what's actually on usenet. require_once dirname(__FILE__) . '/../../../www/config.php'; $cli = new ColorCLI(); if (!isset($argv[2]) || !is_numeric($argv[2])) { exit($cli->error("\nTest your nntp connection, get group information and postdate for specific article.\n\n" . "php {$argv['0']} alt.binaries.teevee 595751142 ...: To test nntp on alt.binaries.teevee with artivle 595751142.\n")); } $nntp = new NNTP(); if ($nntp->doConnect() !== true) { exit; } $first = $argv[2]; $group = $argv[1]; // Select a group. $groupArr = $nntp->selectGroup($group); print_r($groupArr); // Insert actual local part numbers here. $msg = $nntp->getXOVER($first . '-' . $first); // Print out the array of headers. print_r($msg); // get postdate for an article $binaries = new Binaries(['NNTP' => $nntp]); $newdate = $binaries->postdate($first, $groupArr); echo $cli->primary("The posted date for " . $group . ", article " . $first . " is " . date('Y-m-d H:i:s', $newdate));
/** * Update a group back to a specified date. */ function backfillGroup($groupArr, $backfillDate = null, $regexOnly = false) { $db = new DB(); $binaries = new Binaries(); $n = $this->n; if ($regexOnly === true) { echo "Only inserting binaries which match regex{$n}"; $binaries->onlyProcessRegexBinaries = true; } $this->startGroup = microtime(true); $nntp = new Nntp(); $nntpc = new Nntp(); //Make sure we actually have a connection going before doing anything. if ($nntp->doConnect(5, false, true)) { echo 'Processing ' . $groupArr['name'] . $n; $data = $nntp->selectGroup($groupArr['name']); if ($nntp->isError($data)) { echo "Could not select group (bad name?): {$groupArr['name']}{$n}"; return; } if ($backfillDate) { $targetpost = $this->daytopost($nntp, $groupArr['name'], $this->dateToDays($backfillDate), TRUE); } else { $targetpost = $this->daytopost($nntp, $groupArr['name'], $groupArr['backfill_target'], TRUE); } //get targetpost based on days target if ($groupArr['first_record'] == 0 || $groupArr['backfill_target'] == 0 && !$backfillDate) { echo "Group " . $groupArr['name'] . " has invalid numbers. Have you run update on it? Have you set the backfill days amount?{$n}"; return; } echo "Group " . $data["group"] . ": server has " . $data['first'] . " - " . $data['last'] . ", or ~"; echo (int) (($this->postdate($nntp, $data['last'], FALSE) - $this->postdate($nntp, $data['first'], FALSE)) / 86400); echo " days." . $n . "Local first = " . $groupArr['first_record'] . " ("; echo (int) ((date('U') - $this->postdate($nntp, $groupArr['first_record'], FALSE)) / 86400); echo " days). Backfill target of " . ($backfillDate ? date('Y-m-d', $backfillDate) : $groupArr['backfill_target'] . " days") . " is post {$targetpost}.{$n}"; if ($targetpost >= $groupArr['first_record']) { echo "Nothing to do, we already have the target post.{$n} {$n}"; return ""; } //get first and last part numbers from newsgroup if ($targetpost < $data['first']) { echo "WARNING: Backfill came back as before server's first. Setting targetpost to server first.{$n}"; echo "Skipping Group {$n}"; return ""; } echo $binaries->onlyProcessRegexBinaries === true ? "Note: Discarding parts that do not match a regex" . $n : ""; //Done with $nntp for now, close it to avoid timeouts. $nntp->doQuit(); $nntpc->doConnect(); $datac = $nntpc->selectGroup($groupArr['name']); if ($nntpc->isError($datac)) { echo "Could not select group (bad name?): {$groupArr['name']}{$n}"; return; } //calculate total number of parts $total = $groupArr['first_record'] - $targetpost; $done = false; //set first and last, moving the window by maxxMssgs $last = $groupArr['first_record'] - 1; $first = $last - $binaries->messagebuffer + 1; //set initial "chunk" if ($targetpost > $first) { //just in case this is the last chunk we needed $first = $targetpost; } while ($done === false) { $binaries->startLoop = microtime(true); echo "Getting " . ($last - $first + 1) . " parts (" . number_format($first - $targetpost) . " in queue)" . $n; flush(); $success = $binaries->scan($nntpc, $groupArr, $first, $last, 'backfill'); if (!$success) { return ""; } $db->exec(sprintf("update groups SET first_record = %s, last_updated = now() WHERE ID = %d", $db->escapeString($first), $groupArr['ID'])); if ($first == $targetpost) { $done = true; } else { //Keep going: set new last, new first, check for last chunk. $last = $first - 1; $first = $last - $binaries->messagebuffer + 1; if ($targetpost > $first) { $first = $targetpost; } } } //Done with $nntpc $nntpc->doQuit(); //Just need $nntp for a quick check on the first_record_postdate $nntp->doConnect(); //$nntp->selectGroup($groupArr['name']); // some users report having this in keeps backfill working $first_record_postdate = $this->postdate($nntp, $first, false); //All done with NNTP. $nntp->doQuit(); if ($first_record_postdate != "") { $db->exec(sprintf("update groups SET first_record_postdate = FROM_UNIXTIME(" . $first_record_postdate . "), last_updated = now() WHERE ID = %d", $groupArr['ID'])); } //Set group's first postdate $timeGroup = number_format(microtime(true) - $this->startGroup, 2); echo "Group processed in {$timeGroup} seconds {$n}"; } else { echo "Failed to get NNTP connection.{$n}"; } }
$total = 0; $offset = 0; # Dictionary $pspelldict = array(); # Completion Refernece Time in past $completionRef = strtotime("-{$completionSec} seconds"); # Scan header for spelling...; at least 1 word has to match the spelling if ($use_spellCheck && !function_exists('pspell_check')) { echo "Note: Disabling spell checking as it is not available in your PHP installation.\n"; echo " Consider the following to enable it:"; echo " ubuntu #> sudo apt-get install php5-pspell\n"; echo " redhat #> yum install php-pspell\n"; echo " other #> # re-compile php with --with-pspell flag\n"; $use_spellCheck = false; } $binaries = new Binaries(); if ($use_NNRegexDB) { # Over-ride Blacklist up top and use database $blacklist = $binaries->getBlacklist(true); } while ($res = get_block($offset, $batch)) { $subtotal = count($res); $total += $subtotal; $release = new Releases(); $category = new Category(); // Track error count changes $_errcnt = $errcnt; foreach ($res as $header) { // if a white list is detected we now are required to // only accept the entry if it matches at least 1 whitelist // while a blacklist will over-ride all
<?php require_once dirname(__FILE__) . '/../../../../www/config.php'; use nzedb\db\Settings; $pdo = new Settings(); if (!isset($argv[1])) { exit($pdo->log->error("\nYou must supply a path as the first argument. Two additional, optional arguments can also be used.\n\n" . "php {$argv['0']} /path/to/import true 1000 ...: To import using the filename as release searchname, limited to 1000\n" . "php {$argv['0']} /path/to/import false ...: To import using the subject as release searchname\n")); } $consoleTools = new ConsoleTools(['ColorCLI' => $pdo->log]); $binaries = new Binaries(['Settings' => $pdo]); $crosspostt = $pdo->getSetting('crossposttime'); $crosspostt = !empty($crosspostt) ? $crosspostt : 2; $releasecleaning = new ReleaseCleaning($pdo); $categorize = new Categorize(['Settings' => $pdo]); $nzb = new NZB($pdo); $releases = new Releases(['Settings' => $pdo]); $nzbsperhour = $nzbSkipped = $maxtoprocess = 0; if (isset($argv[2]) && is_numeric($argv[2])) { exit($pdo->log->error("\nTo use a max number to process, it must be the third argument. \nTo run:\nphp nzb-import.php /path [true, false] 1000\n")); } if (!isset($argv[2])) { $pieces = explode(" ", $argv[1]); $usenzbname = isset($pieces[1]) && $pieces[1] == 'true' ? true : false; $path = $pieces[0]; } else { $path = $argv[1]; $usenzbname = isset($argv[2]) && $argv[2] == 'true' ? true : false; } if (isset($argv[3]) && is_numeric($argv[3])) { $maxtoprocess = $argv[3]; }
function daytopost($nntp, $group, $days, $debug = true, $bfcheck = true) { global $pdo; $st = false; if ($debug && $bfcheck) { echo $pdo->log->primary('Finding start and end articles for ' . $group . '.'); } if (!isset($nntp)) { $nntp = new \NNTP(['Settings' => $pdo]); if ($nntp->doConnect(false) !== true) { return; } $st = true; } $binaries = new \Binaries(['NNTP' => $nntp, 'Settings' => $pdo]); $data = $nntp->selectGroup($group); if ($nntp->isError($data)) { $data = $nntp->dataError($nntp, $group, false); if ($data === false) { return; } } // Goal timestamp. $goaldate = date('U') - 86400 * $days; $totalnumberofarticles = $data['last'] - $data['first']; $upperbound = $data['last']; $lowerbound = $data['first']; if ($debug && $bfcheck) { echo $pdo->log->header('Total Articles: ' . number_format($totalnumberofarticles) . ' Newest: ' . number_format($upperbound) . ' Oldest: ' . number_format($lowerbound)); } if ($data['last'] == PHP_INT_MAX) { exit($pdo->log->error("Group data is coming back as php's max value. You should not see this since we use a patched Net_NNTP that fixes this bug.")); } $firstDate = $binaries->postdate($data['first'], $data); $lastDate = $binaries->postdate($data['last'], $data); if ($goaldate < $firstDate && $bfcheck) { if ($st === true) { $nntp->doQuit(); } echo $pdo->log->warning("The oldest post indexed from {$days} day(s) ago is older than the first article stored on your news server.\nSetting to First available article of (date('r', {$firstDate}) or daysOld({$firstDate}) days)."); return $data['first']; } else { if ($goaldate > $lastDate && $bfcheck) { if ($st === true) { $nntp->doQuit(); } echo $pdo->log->error("ERROR: The oldest post indexed from {$days} day(s) ago is newer than the last article stored on your news server.\nTo backfill this group you need to set Backfill Days to at least ceil(daysOld({$lastDate})+1) days (date('r', {$lastDate}-86400)."); return ''; } } if ($debug && $bfcheck) { echo $pdo->log->primary("Searching for postdates.\nGroup's Firstdate: " . $firstDate . ' (' . (is_int($firstDate) ? date('r', $firstDate) : 'n/a') . ").\nGroup's Lastdate: " . $lastDate . ' (' . date('r', $lastDate) . ")."); } $interval = floor(($upperbound - $lowerbound) * 0.5); $templowered = ''; $dateofnextone = $lastDate; // Match on days not timestamp to speed things up. while (daysOld($dateofnextone) < $days) { while (($tmpDate = $binaries->postdate($upperbound - $interval, $data)) > $goaldate) { $upperbound = $upperbound - $interval; } if (!$templowered) { $interval = ceil($interval / 2); } $dateofnextone = $binaries->postdate($upperbound - 1, $data); while (!$dateofnextone) { $dateofnextone = $binaries->postdate($upperbound - 1, $data); } } if ($st === true) { $nntp->doQuit(); } if ($bfcheck) { echo $pdo->log->header("\nBackfill article determined to be " . $upperbound . " " . $pdo->log->setColor('Yellow') . "(" . date('r', $dateofnextone) . ")" . $pdo->log->rsetcolor()); } else { echo $pdo->log->header('Current article determined to be ' . $upperbound . " " . $pdo->log->setColor('Yellow') . "(" . date('r', $dateofnextone) . ")" . $pdo->log->rsetcolor()); } // which is '.daysOld($dateofnextone)." days old.\n"; return $upperbound; }
/** * Retrieve all NNTP messages associated with a binaries.ID */ function getBinary($binaryId, $isNfo = false) { $db = new DB(); $bin = new Binaries(); $binary = $bin->getById($binaryId); if (!$binary) { printf("NntpPrc: Unable to locate binary: %s\n", $binaryId); return false; } $summary = $this->selectGroup($binary['groupname']); $message = $dec = ''; if ($this->isError($summary)) { echo "NntpPrc : " . substr($summary->getMessage(), 0, 100) . "\n"; return false; } $resparts = $db->query(sprintf("SELECT size, partnumber, messageID FROM parts WHERE binaryID = %d ORDER BY partnumber", $binaryId)); // // Dont attempt to download nfos which are larger than one part. // if (sizeof($resparts) > 1 && $isNfo === true) { //echo 'NntpPrc : Error Nfo is too large... skipping.\n'; return false; } foreach ($resparts as $part) { $messageID = '<' . $part['messageID'] . '>'; $body = $this->getBody($messageID, true); if ($this->isError($body)) { //echo 'NntpPrc : Error fetching part number '.$part['messageID'].' in '.$binary['groupname'].' (Server response: '. $body->getMessage().')'; return false; } $dec = $this->decodeYenc($body); if (!$dec) { printf("NntpPrc: Unable to decode body of binary: %s\n", $binaryId); // // Yenc decode failed // return false; } $message .= $dec; } return $message; }
<?php require_once "config.php"; // login check $admin = new AdminPage(); $bin = new Binaries(); if (isset($_GET['action']) && $_GET['action'] == "2") { $id = (int) $_GET['bin_id']; $bin->deleteBlacklist($id); print "Blacklist {$id} deleted."; }
/** * Backfill single group. * * @param array $groupArr * @param int $left * @param int|string $articles * * @return void */ public function backfillGroup($groupArr, $left, $articles = '') { // Start time for this group. $startGroup = microtime(true); $groupName = str_replace('alt.binaries', 'a.b', $groupArr['name']); // If our local oldest article 0, it means we never ran update_binaries on the group. if ($groupArr['first_record'] <= 0) { $dMessage = "You need to run update_binaries on " . $groupName . ". Otherwise the group is dead, you must disable it."; if ($this->_debug) { $this->_debugging->log('Backfill', "backfillGroup", $dMessage, Logger::LOG_ERROR); } if ($this->_echoCLI) { $this->pdo->log->doEcho($this->pdo->log->error($dMessage)); } return; } // Select group, here, only once $data = $this->_nntp->selectGroup($groupArr['name']); if ($this->_nntp->isError($data)) { $data = $this->_nntp->dataError($this->_nntp, $groupArr['name']); if ($this->_nntp->isError($data)) { return; } } if ($this->_echoCLI) { $this->pdo->log->doEcho($this->pdo->log->primary('Processing ' . $groupName), true); } // Check if this is days or post backfill. $postCheck = $articles === '' ? false : true; // Get target post based on date or user specified number. $targetpost = (string) ($postCheck ? round($groupArr['first_record'] - $articles) : $this->_binaries->daytopost($groupArr['backfill_target'], $data)); // Check if target post is smaller than server's oldest, set it to oldest if so. if ($targetpost < $data['first']) { $targetpost = $data['first']; } // Check if our target post is newer than our oldest post or if our local oldest article is older than the servers oldest. if ($targetpost >= $groupArr['first_record'] || $groupArr['first_record'] <= $data['first']) { $dMessage = "We have hit the maximum we can backfill for " . $groupName . ", skipping it, consider disabling backfill on it."; if ($this->_debug) { $this->_debugging->log('Backfill', "backfillGroup", $dMessage, Logger::LOG_NOTICE); } if ($this->_echoCLI) { $this->pdo->log->doEcho($this->pdo->log->notice($dMessage), true); } return; } if ($this->_echoCLI) { $this->pdo->log->doEcho($this->pdo->log->primary('Group ' . $groupName . "'s oldest article is " . number_format($data['first']) . ', newest is ' . number_format($data['last']) . ".\nOur target article is " . number_format($targetpost) . '. Our oldest article is article ' . number_format($groupArr['first_record']) . '.')); } // Set first and last, moving the window by max messages. $last = (string) ($groupArr['first_record'] - 1); // Set the initial "chunk". $first = (string) ($last - $this->_binaries->messageBuffer + 1); // Just in case this is the last chunk we needed. if ($targetpost > $first) { $first = $targetpost; } $done = false; while ($done === false) { if ($this->_echoCLI) { $this->pdo->log->doEcho($this->pdo->log->set256('Yellow') . "\nGetting " . number_format($last - $first + 1) . " articles from " . $groupName . ", " . $left . " group(s) left. (" . number_format($first - $targetpost) . " articles in queue)." . $this->pdo->log->rsetColor(), true); } flush(); $lastMsg = $this->_binaries->scan($groupArr, $first, $last, $this->_safePartRepair); // Get the oldest date. if (isset($lastMsg['firstArticleDate'])) { // Try to get it from the oldest pulled article. $newdate = strtotime($lastMsg['firstArticleDate']); } else { // If above failed, try to get it with postdate method. $newdate = $this->_binaries->postdate($first, $data); } $this->pdo->queryExec(sprintf(' UPDATE groups SET first_record_postdate = %s, first_record = %s, last_updated = NOW() WHERE id = %d', $this->pdo->from_unixtime($newdate), $this->pdo->escapeString($first), $groupArr['id'])); if ($first == $targetpost) { $done = true; } else { // Keep going: set new last, new first, check for last chunk. $last = (string) ($first - 1); $first = (string) ($last - $this->_binaries->messageBuffer + 1); if ($targetpost > $first) { $first = $targetpost; } } } if ($this->_echoCLI) { $this->pdo->log->doEcho($this->pdo->log->primary(PHP_EOL . 'Group ' . $groupName . ' processed in ' . number_format(microtime(true) - $startGroup, 2) . " seconds."), true); } }
<?php require_once "config.php"; require_once WWW_DIR . "/lib/adminpage.php"; require_once WWW_DIR . "/lib/binaries.php"; $page = new AdminPage(); $bin = new Binaries(); $id = 0; // set the current action $action = isset($_REQUEST['action']) ? $_REQUEST['action'] : 'view'; switch ($action) { case 'submit': if ($_POST["groupname"] == "") { $page->smarty->assign('error', "Group must be a valid usenet group"); break; } if ($_POST["regex"] == "") { $page->smarty->assign('error', "Regex cannot be empty"); break; } if ($_POST["id"] == "") { $bin->addBlacklist($_POST); } else { $ret = $bin->updateBlacklist($_POST); } header("Location:" . WWW_TOP . "/binaryblacklist-list.php"); break; case 'addtest': if (isset($_GET['regex']) && isset($_GET['groupname'])) { $r = array('groupname' => $_GET['groupname'], 'regex' => $_GET['regex'], 'ordinal' => '1', 'status' => '1'); $page->smarty->assign('regex', $r);
function backfillGroup($nntp, $groupArr, $backfillDate = null) { $db = new DB(); $binaries = new Binaries(); $n = $this->n; $this->startGroup = microtime(true); echo 'Processing ' . $groupArr['name'] . $n; $data = $nntp->selectGroup($groupArr['name']); if (PEAR::isError($data)) { echo "Could not select group (bad name?): {$groupArr['name']}{$n}"; return; } if ($backfillDate) { $targetpost = $this->daytopost($nntp, $groupArr['name'], dateToDays($backfillDate), TRUE); } else { $targetpost = $this->daytopost($nntp, $groupArr['name'], $groupArr['backfill_target'], TRUE); } //get targetpost based on days target if ($groupArr['first_record'] == 0 || $groupArr['backfill_target'] == 0) { echo "Group " . $groupArr['name'] . " has invalid numbers. Have you run update on it? Have you set the backfill days amount?{$n}"; return; } echo "Group " . $data["group"] . ": server has " . $data['first'] . " - " . $data['last'] . ", or ~"; echo (int) (($this->postdate($nntp, $data['last'], FALSE) - $this->postdate($nntp, $data['first'], FALSE)) / 86400); echo " days." . $n . "Local first = " . $groupArr['first_record'] . " ("; echo (int) ((date('U') - $this->postdate($nntp, $groupArr['first_record'], FALSE)) / 86400); echo " days). Backfill target of " . $groupArr['backfill_target'] . "days is post {$targetpost}.{$n}"; if ($targetpost >= $groupArr['first_record']) { echo "Nothing to do, we already have the target post.{$n} {$n}"; return ""; } //get first and last part numbers from newsgroup if ($targetpost < $data['first']) { echo "WARNING: Backfill came back as before server's first. Setting targetpost to server first.{$n}"; echo "Skipping Group {$n}"; return ""; } //calculate total number of parts $total = $groupArr['first_record'] - $targetpost; $done = false; //set first and last, moving the window by maxxMssgs $last = $groupArr['first_record'] - 1; $first = $last - $binaries->messagebuffer + 1; //set initial "chunk" if ($targetpost > $first) { //just in case this is the last chunk we needed $first = $targetpost; } while ($done === false) { $binaries->startLoop = microtime(true); echo "Getting " . ($last - $first + 1) . " parts (" . ($first - $targetpost) . " in queue)" . $n; flush(); $binaries->scan($nntp, $groupArr, $first, $last, 'backfill'); $db->query(sprintf("UPDATE groups SET first_record = %s, last_updated = now() WHERE ID = %d", $db->escapeString($first), $groupArr['ID'])); if ($first == $targetpost) { $done = true; } else { //Keep going: set new last, new first, check for last chunk. $last = $first - 1; $first = $last - $binaries->messagebuffer + 1; if ($targetpost > $first) { $first = $targetpost; } } } $first_record_postdate = $this->postdate($nntp, $first, false); $db->query(sprintf("UPDATE groups SET first_record_postdate = FROM_UNIXTIME(" . $first_record_postdate . "), last_updated = now() WHERE ID = %d", $groupArr['ID'])); //Set group's first postdate $timeGroup = number_format(microtime(true) - $this->startGroup, 2); echo "Group processed in {$timeGroup} seconds {$n}"; }