Пример #1
0
 function getBinary($binaryId, $isNfo = false)
 {
     $db = new DB();
     $bin = new Binaries();
     $binary = $bin->getById($binaryId);
     if (!$binary) {
         return false;
     }
     $summary = $this->selectGroup($binary['groupname']);
     $message = $dec = '';
     if (PEAR::isError($summary)) {
         echo $summary->getMessage();
         return false;
     }
     $resparts = $db->query(sprintf("SELECT size, partnumber, messageID FROM parts WHERE binaryID = %d ORDER BY partnumber", $binaryId));
     if (sizeof($resparts) > 1 && $isNfo === true) {
         echo 'NFO is more than 1 part, skipping. ';
         return false;
     }
     foreach ($resparts as $part) {
         $messageID = '<' . $part['messageID'] . '>';
         $body = $this->getBody($messageID, true);
         if (PEAR::isError($body)) {
             echo 'Error fetching part number ' . $part['messageID'] . ' in ' . $binary['groupname'] . ' (Server response: ' . $body->getMessage() . ')';
             return false;
         }
         $dec = $this->decodeYenc($body);
         if (!$dec) {
             echo "Yenc decode failure";
             return false;
         }
         $message .= $dec;
     }
     return $message;
 }
Пример #2
0
/**
 * @param array $group
 * @param Binaries $binaries
 *
 * @return bool
 */
function dogroup($group, $binaries)
{
    $binaries->updateGroup($group);
    echo "Press enter to continue, type n and press enter to quit.\n";
    $cmd = trim(fgets(fopen("php://stdin", "r")));
    if ($cmd == '') {
        return true;
    } else {
        if ($cmd == "no") {
            exit("Done.\n");
        } else {
            return true;
        }
    }
}
Пример #3
0
 public function purge($id)
 {
     $db = new DB();
     $releases = new Releases();
     $binaries = new Binaries();
     $this->reset($id);
     $rels = $db->query(sprintf("select ID from releases where groupID = %d", $id));
     foreach ($rels as $rel) {
         $releases->delete($rel["ID"]);
     }
     $bins = $db->query(sprintf("select ID from binaries where groupID = %d", $id));
     foreach ($bins as $bin) {
         $binaries->delete($bin["ID"]);
     }
 }
 public function fetchTestBinaries($groupname, $numarticles, $clearexistingbins)
 {
     $db = new DB();
     $nntp = new Nntp();
     $binaries = new Binaries();
     $groups = new Groups();
     $ret = array();
     if ($clearexistingbins == true) {
         $db->exec('truncate releaseregextesting');
     }
     $nntp->doConnect();
     $groupsToFetch = array();
     if (preg_match('/^[a-z]{2,3}(\\.[a-z0-9\\-]+)+$/', $groupname)) {
         $groupsToFetch[] = array('name' => $groupname);
     } elseif ($groupname === 0) {
         $groupsToFetch = $groups->getAll();
     } else {
         $newsgroups = $nntp->getGroups();
         foreach ($newsgroups as $ngroup) {
             if (preg_match('/' . $groupname . '/', $ngroup['group'])) {
                 $groupsToFetch[] = array('name' => $ngroup['group']);
             }
         }
     }
     foreach ($groupsToFetch as $groupArr) {
         $group = $groupArr['name'];
         $data = $nntp->selectGroup($group);
         if ($nntp->isError($data)) {
             $ret[] = "Could not select group (doesnt exist on USP): {$group}";
             continue;
         } else {
             $rangeStart = $data['last'] - $numarticles;
             $rangeEnd = $groupEnd = $data['last'];
             $rangeTotal = $rangeEnd - $rangeStart;
             $done = false;
             while ($done === false) {
                 if ($rangeTotal > $binaries->messagebuffer) {
                     if ($rangeStart + $binaries->messagebuffer > $groupEnd) {
                         $rangeEnd = $groupEnd;
                     } else {
                         $rangeEnd = $rangeStart + $binaries->messagebuffer;
                     }
                 }
                 if ($binaries->compressedHeaders) {
                     $msgs = $nntp->getXOverview($rangeStart . "-" . $rangeEnd, true, false);
                 } else {
                     $msgs = $nntp->getOverview($rangeStart . "-" . $rangeEnd, true, false);
                 }
                 if ($nntp->isError($msgs)) {
                     $ret[] = "Error {$msgs->code}: {$msgs->message} on " . $group;
                     continue 2;
                 }
                 $headers = array();
                 if (is_array($msgs)) {
                     //loop headers, figure out parts
                     foreach ($msgs as $msg) {
                         if (!isset($msg['Number'])) {
                             continue;
                         }
                         $msgPart = $msgTotalParts = 0;
                         $pattern = '|\\((\\d+)[\\/](\\d+)\\)|i';
                         preg_match_all($pattern, $msg['Subject'], $matches, PREG_PATTERN_ORDER);
                         $matchcnt = sizeof($matches[0]);
                         for ($i = 0; $i < $matchcnt; $i++) {
                             //not (int)'d here because of the preg_replace later on
                             $msgPart = $matches[1][$i];
                             $msgTotalParts = $matches[2][$i];
                         }
                         if (!isset($msg['Subject']) || $matchcnt == 0) {
                             // not a binary post most likely.. continue
                             continue;
                         }
                         if ((int) $msgPart > 0 && (int) $msgTotalParts > 0) {
                             $subject = utf8_encode(trim(preg_replace('|\\(' . $msgPart . '[\\/]' . $msgTotalParts . '\\)|i', '', $msg['Subject'])));
                             if (!isset($headers[$subject])) {
                                 $headers[$subject]['Subject'] = $subject;
                                 $headers[$subject]['From'] = $msg['From'];
                                 $headers[$subject]['Date'] = strtotime($msg['Date']);
                                 $headers[$subject]['Message-ID'] = $msg['Message-ID'];
                                 $headers[$subject]['Size'] = $msg['Bytes'];
                             } else {
                                 $headers[$subject]['Size'] += $msg['Bytes'];
                             }
                         }
                     }
                     unset($msgs);
                     if (isset($headers) && count($headers)) {
                         $groupRegexes = $this->getForGroup($group);
                         $binSetData = array();
                         foreach ($headers as $subject => $data) {
                             $binData = array('name' => $subject, 'fromname' => $data['From'], 'date' => $data['Date'], 'binaryhash' => md5($subject . $data['From'] . $group), 'groupname' => $group, 'regexID' => "null", 'categoryID' => "null", 'reqID' => "null", 'blacklistID' => 0, 'size' => $data['Size'], 'relname' => "null", 'relpart' => "null", 'reltotalpart' => "null");
                             //Filter binaries based on black/white list
                             if ($binaries->isBlackListed($data, $group)) {
                                 //binary is blacklisted
                                 $binData['blacklistID'] = 1;
                             }
                             //Apply Regexes
                             $regexMatches = array();
                             foreach ($groupRegexes as $groupRegex) {
                                 $regexCheck = $this->performMatch($groupRegex, $subject, $data['From']);
                                 if ($regexCheck !== false) {
                                     $regexMatches = $regexCheck;
                                     $binData['regexID'] = $regexCheck['regexID'];
                                     $binData['categoryID'] = $regexCheck['regcatid'];
                                     $binData['reqID'] = empty($regexCheck['reqID']) ? "null" : $regexCheck['reqID'];
                                     $binData['relname'] = $regexCheck['name'];
                                     break;
                                 }
                             }
                             $binSetData[] = $binData;
                         }
                         //insert 500 bins at a time
                         $binChunks = array_chunk($binSetData, 500);
                         foreach ($binChunks as $binChunk) {
                             foreach ($binChunk as $chunk) {
                                 $binParams[] = sprintf("(%s, %s, FROM_UNIXTIME(%s), %s, %s, %s, %s, %s, %d, %d, now())", $db->escapeString($chunk['name']), $db->escapeString($chunk['fromname']), $db->escapeString($chunk['date']), $db->escapeString($chunk['binaryhash']), $db->escapeString($chunk['groupname']), $chunk['regexID'], $chunk['categoryID'], $chunk['reqID'], $chunk['blacklistID'], $chunk['size']);
                             }
                             $binSql = "INSERT IGNORE INTO releaseregextesting (name, fromname, date, binaryhash, groupname, regexID, categoryID, reqID, blacklistID, size, dateadded) VALUES " . implode(', ', $binParams);
                             //echo $binSql;
                             $db->exec($binSql);
                         }
                         $ret[] = "Fetched " . number_format($numarticles) . " articles from " . $group;
                     } else {
                         $ret[] = "No headers found on " . $group;
                         continue;
                     }
                 } else {
                     $ret[] = "Can't get parts from server (msgs not array) on " . $group;
                     continue;
                 }
                 if ($rangeEnd == $groupEnd) {
                     $done = true;
                 }
                 $rangeStart = $rangeEnd + 1;
             }
         }
     }
     $nntp->doQuit();
     return $ret;
 }
Пример #5
0
<?php

/* Argument 1 is optional string, group name. Or numeric, number of header max to download.
 * Argument 2 is optional int, max number of headers to download.
 */
require_once dirname(__FILE__) . '/config.php';
$pdo = new \nzedb\db\Settings();
// Create the connection here and pass
$nntp = new NNTP(['Settings' => $pdo]);
if ($nntp->doConnect() !== true) {
    exit($pdo->log->error("Unable to connect to usenet."));
}
$binaries = new Binaries(['NNTP' => $nntp, 'Settings' => $pdo]);
if (isset($argv[1]) && !is_numeric($argv[1])) {
    $groupName = $argv[1];
    echo $pdo->log->header("Updating group: {$groupName}");
    $grp = new Groups(['Settings' => $pdo]);
    $group = $grp->getByName($groupName);
    if (is_array($group)) {
        $binaries->updateGroup($group, isset($argv[2]) && is_numeric($argv[2]) && $argv[2] > 0 ? $argv[2] : 0);
    }
} else {
    $binaries->updateAllGroups(isset($argv[1]) && is_numeric($argv[1]) && $argv[1] > 0 ? $argv[1] : 0);
}
<?php

require_once WWW_DIR . "/lib/binaries.php";
$binaries = new Binaries();
if (!$users->isLoggedIn()) {
    $page->show403();
}
$page->meta_title = "Search Binaries";
$page->meta_keywords = "search,binaries,binsearch,nzb,description,details";
$page->meta_description = "Search for Binaries";
$results = array();
$searchStr = '';
//TODO: bug here in javascript cludge to turn request in a get, rather than post is losing + signs in search query.
if (isset($_REQUEST["id"])) {
    $searchStr = $_REQUEST["id"];
    $results = $binaries->search($searchStr, 1000, $page->userdata["categoryexclusions"]);
}
$page->smarty->assign('search', $searchStr);
$page->smarty->assign('results', $results);
$page->content = $page->smarty->fetch('searchraw.tpl');
$page->render();
<?php

require_once "config.php";
require_once WWW_DIR . "/lib/adminpage.php";
require_once WWW_DIR . "/lib/binaries.php";
$page = new AdminPage();
$bin = new Binaries();
$page->title = "Binary Black/Whitelist List";
$binlist = $bin->getBlacklist(false);
$page->smarty->assign('binlist', $binlist);
$page->content = $page->smarty->fetch('binaryblacklist-list.tpl');
$page->render();
Пример #8
0
<?php

require "config.php";
require_once WWW_DIR . "/lib/binaries.php";
$binaries = new Binaries();
$binaries->updateAllGroups();
Пример #9
0
 /**
  * @param object $nzbXML Reference of simpleXmlObject with NZB contents.
  * @param bool|string $useNzbName Use the NZB file name as release name?
  * @return bool
  *
  * @access protected
  */
 protected function scanNZBFile(&$nzbXML, $useNzbName = false)
 {
     $totalFiles = $totalSize = $groupID = 0;
     $isBlackListed = $groupName = $firstName = $posterName = $postDate = false;
     // Go through the NZB, get the details, look if it's blacklisted, look if we have the groups.
     foreach ($nzbXML->file as $file) {
         $totalFiles++;
         $groupID = -1;
         // Get the nzb info.
         if ($firstName === false) {
             $firstName = (string) $file->attributes()->subject;
         }
         if ($posterName === false) {
             $posterName = (string) $file->attributes()->poster;
         }
         if ($postDate === false) {
             $postDate = date("Y-m-d H:i:s", (string) $file->attributes()->date);
         }
         // Make a fake message array to use to check the blacklist.
         $msg = ["Subject" => (string) $file->attributes()->subject, "From" => (string) $file->attributes()->poster, "Message-ID" => ""];
         // Get the group names, group_id, check if it's blacklisted.
         $groupArr = [];
         foreach ($file->groups->group as $group) {
             $group = (string) $group;
             // If group_id is -1 try to get a group_id.
             if ($groupID === -1) {
                 if (array_key_exists($group, $this->allGroups)) {
                     $groupID = $this->allGroups[$group];
                     if (!$groupName) {
                         $groupName = $group;
                     }
                 } else {
                     $groupID = $this->groups->add(['name' => $group, 'description' => 'Added by NZBimport script.', 'backfill_target' => 0, 'first_record' => 0, 'last_record' => 0, 'active' => 0, 'backfill' => 0]);
                     $this->allGroups[$group] = $groupID;
                     $this->echoOut("Adding missing group: ({$group})");
                 }
             }
             // Add all the found groups to an array.
             $groupArr[] = $group;
             // Check if this NZB is blacklisted.
             if ($this->binaries->isBlacklisted($msg, $group)) {
                 $isBlackListed = true;
                 break;
             }
         }
         // If we found a group and it's not blacklisted.
         if ($groupID !== -1 && !$isBlackListed) {
             // Get the size of the release.
             if (count($file->segments->segment) > 0) {
                 foreach ($file->segments->segment as $segment) {
                     $totalSize += (int) $segment->attributes()->bytes;
                 }
             }
         } else {
             if ($isBlackListed) {
                 $errorMessage = "Subject is blacklisted: " . utf8_encode(trim($firstName));
             } else {
                 $errorMessage = "No group found for " . $firstName . " (one of " . implode(', ', $groupArr) . " are missing";
             }
             $this->echoOut($errorMessage);
             return false;
         }
     }
     // Try to insert the NZB details into the DB.
     return $this->insertNZB(['subject' => $firstName, 'useFName' => $useNzbName, 'postDate' => empty($postDate) ? date("Y-m-d H:i:s") : $postDate, 'from' => empty($posterName) ? '' : $posterName, 'groupid' => $groupID, 'groupName' => $groupName, 'totalFiles' => $totalFiles, 'totalSize' => $totalSize]);
 }
Пример #10
0
$pdo = new nzedb\db\DB();
if ($argv[1] === 'all') {
    $groups = $pdo->query('SELECT * FROM groups WHERE backfill = 1');
} else {
    $groups = $pdo->query(sprintf('SELECT * FROM groups WHERE name = %s', $pdo->escapeString($argv[1])));
}
if (count($groups) === 0) {
    if ($argv[1] === 'all') {
        exit('ERROR! No groups were found with backfill enabled!' . PHP_EOL);
    } else {
        exit('ERROR! Group (' . $argv[1] . ') not found!' . PHP_EOL);
    }
}
$nntp = new NNTP(['Settings' => $pdo]);
$nntp->doConnect() or exit('Could not connect to Usenet!' . PHP_EOL);
$binaries = new Binaries(['NNTP' => $nntp, 'Settings' => $pdo]);
foreach ($groups as $group) {
    $groupNNTP = $nntp->selectGroup($group['name']);
    if ($nntp->isError($groupNNTP)) {
        echo 'ERROR! Could not fetch information from NNTP for group (' . $group['name'] . ')' . PHP_EOL;
        continue;
    }
    $postDate = $pdo->queryOneRow(sprintf('SELECT UNIX_TIMESTAMP(postdate) AS postdate FROM releases WHERE group_id = %d ORDER BY postdate ASC LIMIT 1', $group['id']));
    if ($postDate === false) {
        echo 'ERROR! Could not find any existing releases for group (' . $group['name'] . ')' . PHP_EOL;
        continue;
    }
    $articleNumber = (int) $binaries->daytopost(round((time() - $postDate['postdate']) / 86400), $groupNNTP);
    if ($group['last_record'] != 0 && $articleNumber >= $group['last_record']) {
        echo 'ERROR! Could not determine the article number for this date: (' . $postDate['postdate'] . ') on group (' . $group['name'] . ')' . PHP_EOL;
        continue;
Пример #11
0
  * $options[3] => (string) Group name.
  * $options[4] => (int)    First article number in range.
  * $options[5] => (int)    Last article number in range.
  * $options[6] => (int)    Number of threads.
  */
 case 'get_range':
     $pdo = new Settings();
     $nntp = nntp($pdo);
     $groups = new \Groups();
     $groupMySQL = $groups->getByName($options[3]);
     if ($nntp->isError($nntp->selectGroup($groupMySQL['name']))) {
         if ($nntp->isError($nntp->dataError($nntp, $groupMySQL['name']))) {
             return;
         }
     }
     $binaries = new \Binaries(['NNTP' => $nntp, 'Settings' => $pdo, 'Groups' => $groups]);
     $return = $binaries->scan($groupMySQL, $options[4], $options[5], $pdo->getSetting('safepartrepair') == 1 ? 'update' : 'backfill');
     if (empty($return)) {
         exit;
     }
     $columns = [];
     switch ($options[2]) {
         case 'binaries':
             if ($return['lastArticleNumber'] <= $groupMySQL['last_record']) {
                 exit;
             }
             $columns[1] = sprintf('last_record_postdate = %s', $pdo->from_unixtime(is_numeric($return['lastArticleDate']) ? $return['lastArticleDate'] : strtotime($return['lastArticleDate'])));
             $columns[2] = sprintf('last_record = %s', $return['lastArticleNumber']);
             $query = sprintf('UPDATE groups SET %s, %s, last_updated = NOW() WHERE id = %d AND last_record < %s', $columns[1], $columns[2], $groupMySQL['id'], $return['lastArticleNumber']);
             break;
         case 'backfill':
Пример #12
0
<?php

require_once "config.php";
require_once WWW_DIR . "/lib/adminpage.php";
require_once WWW_DIR . "/lib/releases.php";
require_once WWW_DIR . "/lib/users.php";
require_once WWW_DIR . "/lib/binaries.php";
$page = new AdminPage();
$users = new Users();
if (!$users->isLoggedIn()) {
    $page->show403();
}
if (isset($_GET["id"])) {
    $releases = new Releases();
    $rel = $releases->getByGuid($_GET["id"]);
    if (!$rel) {
        $page->show404();
    }
    $binaries = new Binaries();
    $data = $binaries->getForReleaseId($rel["ID"]);
    $page->smarty->assign('rel', $rel);
    $page->smarty->assign('binaries', $data);
    $page->title = "File List";
    $page->meta_title = "View Nzb file list";
    $page->meta_keywords = "view,nzb,file,list,description,details";
    $page->meta_description = "View Nzb File List";
    $page->content = $page->smarty->fetch('release-files.tpl');
    $page->render();
}
Пример #13
0
<?php

require_once "config.php";
require_once WWW_DIR . "/lib/framework/db.php";
require_once WWW_DIR . "/lib/binaries.php";
$bin = new Binaries();
$articles = $bin->search("mess");
foreach ($articles as $article) {
    $pattern = '/()(.*)(\\d{2,3}\\/\\d{1,3})/i';
    if (!preg_match($pattern, rtrim($article["name"]), $matches)) {
        echo "Not matched: " . $article["name"] . "<br/>";
    } else {
        print_r($matches) . "";
    }
}
Пример #14
0
<?php

// To troubleshoot what's actually on usenet.
require_once dirname(__FILE__) . '/../../../www/config.php';
$cli = new ColorCLI();
if (!isset($argv[2]) || !is_numeric($argv[2])) {
    exit($cli->error("\nTest your nntp connection, get group information and postdate for specific article.\n\n" . "php {$argv['0']} alt.binaries.teevee 595751142    ...: To test nntp on alt.binaries.teevee with artivle 595751142.\n"));
}
$nntp = new NNTP();
if ($nntp->doConnect() !== true) {
    exit;
}
$first = $argv[2];
$group = $argv[1];
// Select a group.
$groupArr = $nntp->selectGroup($group);
print_r($groupArr);
// Insert actual local part numbers here.
$msg = $nntp->getXOVER($first . '-' . $first);
// Print out the array of headers.
print_r($msg);
// get postdate for an article
$binaries = new Binaries(['NNTP' => $nntp]);
$newdate = $binaries->postdate($first, $groupArr);
echo $cli->primary("The posted date for " . $group . ", article " . $first . " is " . date('Y-m-d H:i:s', $newdate));
 /**
  * Update a group back to a specified date.
  */
 function backfillGroup($groupArr, $backfillDate = null, $regexOnly = false)
 {
     $db = new DB();
     $binaries = new Binaries();
     $n = $this->n;
     if ($regexOnly === true) {
         echo "Only inserting binaries which match regex{$n}";
         $binaries->onlyProcessRegexBinaries = true;
     }
     $this->startGroup = microtime(true);
     $nntp = new Nntp();
     $nntpc = new Nntp();
     //Make sure we actually have a connection going before doing anything.
     if ($nntp->doConnect(5, false, true)) {
         echo 'Processing ' . $groupArr['name'] . $n;
         $data = $nntp->selectGroup($groupArr['name']);
         if ($nntp->isError($data)) {
             echo "Could not select group (bad name?): {$groupArr['name']}{$n}";
             return;
         }
         if ($backfillDate) {
             $targetpost = $this->daytopost($nntp, $groupArr['name'], $this->dateToDays($backfillDate), TRUE);
         } else {
             $targetpost = $this->daytopost($nntp, $groupArr['name'], $groupArr['backfill_target'], TRUE);
         }
         //get targetpost based on days target
         if ($groupArr['first_record'] == 0 || $groupArr['backfill_target'] == 0 && !$backfillDate) {
             echo "Group " . $groupArr['name'] . " has invalid numbers.  Have you run update on it?  Have you set the backfill days amount?{$n}";
             return;
         }
         echo "Group " . $data["group"] . ": server has " . $data['first'] . " - " . $data['last'] . ", or ~";
         echo (int) (($this->postdate($nntp, $data['last'], FALSE) - $this->postdate($nntp, $data['first'], FALSE)) / 86400);
         echo " days." . $n . "Local first = " . $groupArr['first_record'] . " (";
         echo (int) ((date('U') - $this->postdate($nntp, $groupArr['first_record'], FALSE)) / 86400);
         echo " days).  Backfill target of " . ($backfillDate ? date('Y-m-d', $backfillDate) : $groupArr['backfill_target'] . " days") . " is post {$targetpost}.{$n}";
         if ($targetpost >= $groupArr['first_record']) {
             echo "Nothing to do, we already have the target post.{$n} {$n}";
             return "";
         }
         //get first and last part numbers from newsgroup
         if ($targetpost < $data['first']) {
             echo "WARNING: Backfill came back as before server's first.  Setting targetpost to server first.{$n}";
             echo "Skipping Group {$n}";
             return "";
         }
         echo $binaries->onlyProcessRegexBinaries === true ? "Note: Discarding parts that do not match a regex" . $n : "";
         //Done with $nntp for now, close it to avoid timeouts.
         $nntp->doQuit();
         $nntpc->doConnect();
         $datac = $nntpc->selectGroup($groupArr['name']);
         if ($nntpc->isError($datac)) {
             echo "Could not select group (bad name?): {$groupArr['name']}{$n}";
             return;
         }
         //calculate total number of parts
         $total = $groupArr['first_record'] - $targetpost;
         $done = false;
         //set first and last, moving the window by maxxMssgs
         $last = $groupArr['first_record'] - 1;
         $first = $last - $binaries->messagebuffer + 1;
         //set initial "chunk"
         if ($targetpost > $first) {
             //just in case this is the last chunk we needed
             $first = $targetpost;
         }
         while ($done === false) {
             $binaries->startLoop = microtime(true);
             echo "Getting " . ($last - $first + 1) . " parts (" . number_format($first - $targetpost) . " in queue)" . $n;
             flush();
             $success = $binaries->scan($nntpc, $groupArr, $first, $last, 'backfill');
             if (!$success) {
                 return "";
             }
             $db->exec(sprintf("update groups SET first_record = %s, last_updated = now() WHERE ID = %d", $db->escapeString($first), $groupArr['ID']));
             if ($first == $targetpost) {
                 $done = true;
             } else {
                 //Keep going: set new last, new first, check for last chunk.
                 $last = $first - 1;
                 $first = $last - $binaries->messagebuffer + 1;
                 if ($targetpost > $first) {
                     $first = $targetpost;
                 }
             }
         }
         //Done with $nntpc
         $nntpc->doQuit();
         //Just need $nntp for a quick check on the first_record_postdate
         $nntp->doConnect();
         //$nntp->selectGroup($groupArr['name']); // some users report having this in keeps backfill working
         $first_record_postdate = $this->postdate($nntp, $first, false);
         //All done with NNTP.
         $nntp->doQuit();
         if ($first_record_postdate != "") {
             $db->exec(sprintf("update groups SET first_record_postdate = FROM_UNIXTIME(" . $first_record_postdate . "), last_updated = now() WHERE ID = %d", $groupArr['ID']));
         }
         //Set group's first postdate
         $timeGroup = number_format(microtime(true) - $this->startGroup, 2);
         echo "Group processed in {$timeGroup} seconds {$n}";
     } else {
         echo "Failed to get NNTP connection.{$n}";
     }
 }
$total = 0;
$offset = 0;
# Dictionary
$pspelldict = array();
# Completion Refernece Time in past
$completionRef = strtotime("-{$completionSec} seconds");
# Scan header for spelling...; at least 1 word has to match the spelling
if ($use_spellCheck && !function_exists('pspell_check')) {
    echo "Note: Disabling spell checking as it is not available in your PHP installation.\n";
    echo "      Consider the following to enable it:";
    echo "             ubuntu #> sudo apt-get install php5-pspell\n";
    echo "             redhat #> yum install php-pspell\n";
    echo "             other  #> # re-compile php with --with-pspell flag\n";
    $use_spellCheck = false;
}
$binaries = new Binaries();
if ($use_NNRegexDB) {
    # Over-ride Blacklist up top and use database
    $blacklist = $binaries->getBlacklist(true);
}
while ($res = get_block($offset, $batch)) {
    $subtotal = count($res);
    $total += $subtotal;
    $release = new Releases();
    $category = new Category();
    // Track error count changes
    $_errcnt = $errcnt;
    foreach ($res as $header) {
        // if a white list is detected we now are required to
        // only accept the entry if it matches at least 1 whitelist
        // while a blacklist will over-ride all
Пример #17
0
<?php

require_once dirname(__FILE__) . '/../../../../www/config.php';
use nzedb\db\Settings;
$pdo = new Settings();
if (!isset($argv[1])) {
    exit($pdo->log->error("\nYou must supply a path as the first argument. Two additional, optional arguments can also be used.\n\n" . "php {$argv['0']} /path/to/import true 1000            ...: To import using the filename as release searchname, limited to 1000\n" . "php {$argv['0']} /path/to/import false                ...: To import using the subject as release searchname\n"));
}
$consoleTools = new ConsoleTools(['ColorCLI' => $pdo->log]);
$binaries = new Binaries(['Settings' => $pdo]);
$crosspostt = $pdo->getSetting('crossposttime');
$crosspostt = !empty($crosspostt) ? $crosspostt : 2;
$releasecleaning = new ReleaseCleaning($pdo);
$categorize = new Categorize(['Settings' => $pdo]);
$nzb = new NZB($pdo);
$releases = new Releases(['Settings' => $pdo]);
$nzbsperhour = $nzbSkipped = $maxtoprocess = 0;
if (isset($argv[2]) && is_numeric($argv[2])) {
    exit($pdo->log->error("\nTo use a max number to process, it must be the third argument. \nTo run:\nphp nzb-import.php /path [true, false] 1000\n"));
}
if (!isset($argv[2])) {
    $pieces = explode("   ", $argv[1]);
    $usenzbname = isset($pieces[1]) && $pieces[1] == 'true' ? true : false;
    $path = $pieces[0];
} else {
    $path = $argv[1];
    $usenzbname = isset($argv[2]) && $argv[2] == 'true' ? true : false;
}
if (isset($argv[3]) && is_numeric($argv[3])) {
    $maxtoprocess = $argv[3];
}
Пример #18
0
function daytopost($nntp, $group, $days, $debug = true, $bfcheck = true)
{
    global $pdo;
    $st = false;
    if ($debug && $bfcheck) {
        echo $pdo->log->primary('Finding start and end articles for ' . $group . '.');
    }
    if (!isset($nntp)) {
        $nntp = new \NNTP(['Settings' => $pdo]);
        if ($nntp->doConnect(false) !== true) {
            return;
        }
        $st = true;
    }
    $binaries = new \Binaries(['NNTP' => $nntp, 'Settings' => $pdo]);
    $data = $nntp->selectGroup($group);
    if ($nntp->isError($data)) {
        $data = $nntp->dataError($nntp, $group, false);
        if ($data === false) {
            return;
        }
    }
    // Goal timestamp.
    $goaldate = date('U') - 86400 * $days;
    $totalnumberofarticles = $data['last'] - $data['first'];
    $upperbound = $data['last'];
    $lowerbound = $data['first'];
    if ($debug && $bfcheck) {
        echo $pdo->log->header('Total Articles: ' . number_format($totalnumberofarticles) . ' Newest: ' . number_format($upperbound) . ' Oldest: ' . number_format($lowerbound));
    }
    if ($data['last'] == PHP_INT_MAX) {
        exit($pdo->log->error("Group data is coming back as php's max value. You should not see this since we use a patched Net_NNTP that fixes this bug."));
    }
    $firstDate = $binaries->postdate($data['first'], $data);
    $lastDate = $binaries->postdate($data['last'], $data);
    if ($goaldate < $firstDate && $bfcheck) {
        if ($st === true) {
            $nntp->doQuit();
        }
        echo $pdo->log->warning("The oldest post indexed from {$days} day(s) ago is older than the first article stored on your news server.\nSetting to First available article of (date('r', {$firstDate}) or daysOld({$firstDate}) days).");
        return $data['first'];
    } else {
        if ($goaldate > $lastDate && $bfcheck) {
            if ($st === true) {
                $nntp->doQuit();
            }
            echo $pdo->log->error("ERROR: The oldest post indexed from {$days} day(s) ago is newer than the last article stored on your news server.\nTo backfill this group you need to set Backfill Days to at least ceil(daysOld({$lastDate})+1) days (date('r', {$lastDate}-86400).");
            return '';
        }
    }
    if ($debug && $bfcheck) {
        echo $pdo->log->primary("Searching for postdates.\nGroup's Firstdate: " . $firstDate . ' (' . (is_int($firstDate) ? date('r', $firstDate) : 'n/a') . ").\nGroup's Lastdate: " . $lastDate . ' (' . date('r', $lastDate) . ").");
    }
    $interval = floor(($upperbound - $lowerbound) * 0.5);
    $templowered = '';
    $dateofnextone = $lastDate;
    // Match on days not timestamp to speed things up.
    while (daysOld($dateofnextone) < $days) {
        while (($tmpDate = $binaries->postdate($upperbound - $interval, $data)) > $goaldate) {
            $upperbound = $upperbound - $interval;
        }
        if (!$templowered) {
            $interval = ceil($interval / 2);
        }
        $dateofnextone = $binaries->postdate($upperbound - 1, $data);
        while (!$dateofnextone) {
            $dateofnextone = $binaries->postdate($upperbound - 1, $data);
        }
    }
    if ($st === true) {
        $nntp->doQuit();
    }
    if ($bfcheck) {
        echo $pdo->log->header("\nBackfill article determined to be " . $upperbound . " " . $pdo->log->setColor('Yellow') . "(" . date('r', $dateofnextone) . ")" . $pdo->log->rsetcolor());
    } else {
        echo $pdo->log->header('Current article determined to be ' . $upperbound . " " . $pdo->log->setColor('Yellow') . "(" . date('r', $dateofnextone) . ")" . $pdo->log->rsetcolor());
    }
    // which is '.daysOld($dateofnextone)." days old.\n";
    return $upperbound;
}
Пример #19
0
 /**
  * Retrieve all NNTP messages associated with a binaries.ID
  */
 function getBinary($binaryId, $isNfo = false)
 {
     $db = new DB();
     $bin = new Binaries();
     $binary = $bin->getById($binaryId);
     if (!$binary) {
         printf("NntpPrc: Unable to locate binary: %s\n", $binaryId);
         return false;
     }
     $summary = $this->selectGroup($binary['groupname']);
     $message = $dec = '';
     if ($this->isError($summary)) {
         echo "NntpPrc : " . substr($summary->getMessage(), 0, 100) . "\n";
         return false;
     }
     $resparts = $db->query(sprintf("SELECT size, partnumber, messageID FROM parts WHERE binaryID = %d ORDER BY partnumber", $binaryId));
     //
     // Dont attempt to download nfos which are larger than one part.
     //
     if (sizeof($resparts) > 1 && $isNfo === true) {
         //echo 'NntpPrc : Error Nfo is too large... skipping.\n';
         return false;
     }
     foreach ($resparts as $part) {
         $messageID = '<' . $part['messageID'] . '>';
         $body = $this->getBody($messageID, true);
         if ($this->isError($body)) {
             //echo 'NntpPrc : Error fetching part number '.$part['messageID'].' in '.$binary['groupname'].' (Server response: '. $body->getMessage().')';
             return false;
         }
         $dec = $this->decodeYenc($body);
         if (!$dec) {
             printf("NntpPrc: Unable to decode body of binary: %s\n", $binaryId);
             //
             // Yenc decode failed
             //
             return false;
         }
         $message .= $dec;
     }
     return $message;
 }
<?php

require_once "config.php";
// login check
$admin = new AdminPage();
$bin = new Binaries();
if (isset($_GET['action']) && $_GET['action'] == "2") {
    $id = (int) $_GET['bin_id'];
    $bin->deleteBlacklist($id);
    print "Blacklist {$id} deleted.";
}
Пример #21
0
    /**
     * Backfill single group.
     *
     * @param array $groupArr
     * @param int $left
     * @param int|string $articles
     *
     * @return void
     */
    public function backfillGroup($groupArr, $left, $articles = '')
    {
        // Start time for this group.
        $startGroup = microtime(true);
        $groupName = str_replace('alt.binaries', 'a.b', $groupArr['name']);
        // If our local oldest article 0, it means we never ran update_binaries on the group.
        if ($groupArr['first_record'] <= 0) {
            $dMessage = "You need to run update_binaries on " . $groupName . ". Otherwise the group is dead, you must disable it.";
            if ($this->_debug) {
                $this->_debugging->log('Backfill', "backfillGroup", $dMessage, Logger::LOG_ERROR);
            }
            if ($this->_echoCLI) {
                $this->pdo->log->doEcho($this->pdo->log->error($dMessage));
            }
            return;
        }
        // Select group, here, only once
        $data = $this->_nntp->selectGroup($groupArr['name']);
        if ($this->_nntp->isError($data)) {
            $data = $this->_nntp->dataError($this->_nntp, $groupArr['name']);
            if ($this->_nntp->isError($data)) {
                return;
            }
        }
        if ($this->_echoCLI) {
            $this->pdo->log->doEcho($this->pdo->log->primary('Processing ' . $groupName), true);
        }
        // Check if this is days or post backfill.
        $postCheck = $articles === '' ? false : true;
        // Get target post based on date or user specified number.
        $targetpost = (string) ($postCheck ? round($groupArr['first_record'] - $articles) : $this->_binaries->daytopost($groupArr['backfill_target'], $data));
        // Check if target post is smaller than server's oldest, set it to oldest if so.
        if ($targetpost < $data['first']) {
            $targetpost = $data['first'];
        }
        // Check if our target post is newer than our oldest post or if our local oldest article is older than the servers oldest.
        if ($targetpost >= $groupArr['first_record'] || $groupArr['first_record'] <= $data['first']) {
            $dMessage = "We have hit the maximum we can backfill for " . $groupName . ", skipping it, consider disabling backfill on it.";
            if ($this->_debug) {
                $this->_debugging->log('Backfill', "backfillGroup", $dMessage, Logger::LOG_NOTICE);
            }
            if ($this->_echoCLI) {
                $this->pdo->log->doEcho($this->pdo->log->notice($dMessage), true);
            }
            return;
        }
        if ($this->_echoCLI) {
            $this->pdo->log->doEcho($this->pdo->log->primary('Group ' . $groupName . "'s oldest article is " . number_format($data['first']) . ', newest is ' . number_format($data['last']) . ".\nOur target article is " . number_format($targetpost) . '. Our oldest article is article ' . number_format($groupArr['first_record']) . '.'));
        }
        // Set first and last, moving the window by max messages.
        $last = (string) ($groupArr['first_record'] - 1);
        // Set the initial "chunk".
        $first = (string) ($last - $this->_binaries->messageBuffer + 1);
        // Just in case this is the last chunk we needed.
        if ($targetpost > $first) {
            $first = $targetpost;
        }
        $done = false;
        while ($done === false) {
            if ($this->_echoCLI) {
                $this->pdo->log->doEcho($this->pdo->log->set256('Yellow') . "\nGetting " . number_format($last - $first + 1) . " articles from " . $groupName . ", " . $left . " group(s) left. (" . number_format($first - $targetpost) . " articles in queue)." . $this->pdo->log->rsetColor(), true);
            }
            flush();
            $lastMsg = $this->_binaries->scan($groupArr, $first, $last, $this->_safePartRepair);
            // Get the oldest date.
            if (isset($lastMsg['firstArticleDate'])) {
                // Try to get it from the oldest pulled article.
                $newdate = strtotime($lastMsg['firstArticleDate']);
            } else {
                // If above failed, try to get it with postdate method.
                $newdate = $this->_binaries->postdate($first, $data);
            }
            $this->pdo->queryExec(sprintf('
					UPDATE groups
					SET first_record_postdate = %s, first_record = %s, last_updated = NOW()
					WHERE id = %d', $this->pdo->from_unixtime($newdate), $this->pdo->escapeString($first), $groupArr['id']));
            if ($first == $targetpost) {
                $done = true;
            } else {
                // Keep going: set new last, new first, check for last chunk.
                $last = (string) ($first - 1);
                $first = (string) ($last - $this->_binaries->messageBuffer + 1);
                if ($targetpost > $first) {
                    $first = $targetpost;
                }
            }
        }
        if ($this->_echoCLI) {
            $this->pdo->log->doEcho($this->pdo->log->primary(PHP_EOL . 'Group ' . $groupName . ' processed in ' . number_format(microtime(true) - $startGroup, 2) . " seconds."), true);
        }
    }
Пример #22
0
<?php

require_once "config.php";
require_once WWW_DIR . "/lib/adminpage.php";
require_once WWW_DIR . "/lib/binaries.php";
$page = new AdminPage();
$bin = new Binaries();
$id = 0;
// set the current action
$action = isset($_REQUEST['action']) ? $_REQUEST['action'] : 'view';
switch ($action) {
    case 'submit':
        if ($_POST["groupname"] == "") {
            $page->smarty->assign('error', "Group must be a valid usenet group");
            break;
        }
        if ($_POST["regex"] == "") {
            $page->smarty->assign('error', "Regex cannot be empty");
            break;
        }
        if ($_POST["id"] == "") {
            $bin->addBlacklist($_POST);
        } else {
            $ret = $bin->updateBlacklist($_POST);
        }
        header("Location:" . WWW_TOP . "/binaryblacklist-list.php");
        break;
    case 'addtest':
        if (isset($_GET['regex']) && isset($_GET['groupname'])) {
            $r = array('groupname' => $_GET['groupname'], 'regex' => $_GET['regex'], 'ordinal' => '1', 'status' => '1');
            $page->smarty->assign('regex', $r);
Пример #23
0
 function backfillGroup($nntp, $groupArr, $backfillDate = null)
 {
     $db = new DB();
     $binaries = new Binaries();
     $n = $this->n;
     $this->startGroup = microtime(true);
     echo 'Processing ' . $groupArr['name'] . $n;
     $data = $nntp->selectGroup($groupArr['name']);
     if (PEAR::isError($data)) {
         echo "Could not select group (bad name?): {$groupArr['name']}{$n}";
         return;
     }
     if ($backfillDate) {
         $targetpost = $this->daytopost($nntp, $groupArr['name'], dateToDays($backfillDate), TRUE);
     } else {
         $targetpost = $this->daytopost($nntp, $groupArr['name'], $groupArr['backfill_target'], TRUE);
     }
     //get targetpost based on days target
     if ($groupArr['first_record'] == 0 || $groupArr['backfill_target'] == 0) {
         echo "Group " . $groupArr['name'] . " has invalid numbers.  Have you run update on it?  Have you set the backfill days amount?{$n}";
         return;
     }
     echo "Group " . $data["group"] . ": server has " . $data['first'] . " - " . $data['last'] . ", or ~";
     echo (int) (($this->postdate($nntp, $data['last'], FALSE) - $this->postdate($nntp, $data['first'], FALSE)) / 86400);
     echo " days." . $n . "Local first = " . $groupArr['first_record'] . " (";
     echo (int) ((date('U') - $this->postdate($nntp, $groupArr['first_record'], FALSE)) / 86400);
     echo " days).  Backfill target of " . $groupArr['backfill_target'] . "days is post {$targetpost}.{$n}";
     if ($targetpost >= $groupArr['first_record']) {
         echo "Nothing to do, we already have the target post.{$n} {$n}";
         return "";
     }
     //get first and last part numbers from newsgroup
     if ($targetpost < $data['first']) {
         echo "WARNING: Backfill came back as before server's first.  Setting targetpost to server first.{$n}";
         echo "Skipping Group {$n}";
         return "";
     }
     //calculate total number of parts
     $total = $groupArr['first_record'] - $targetpost;
     $done = false;
     //set first and last, moving the window by maxxMssgs
     $last = $groupArr['first_record'] - 1;
     $first = $last - $binaries->messagebuffer + 1;
     //set initial "chunk"
     if ($targetpost > $first) {
         //just in case this is the last chunk we needed
         $first = $targetpost;
     }
     while ($done === false) {
         $binaries->startLoop = microtime(true);
         echo "Getting " . ($last - $first + 1) . " parts (" . ($first - $targetpost) . " in queue)" . $n;
         flush();
         $binaries->scan($nntp, $groupArr, $first, $last, 'backfill');
         $db->query(sprintf("UPDATE groups SET first_record = %s, last_updated = now() WHERE ID = %d", $db->escapeString($first), $groupArr['ID']));
         if ($first == $targetpost) {
             $done = true;
         } else {
             //Keep going: set new last, new first, check for last chunk.
             $last = $first - 1;
             $first = $last - $binaries->messagebuffer + 1;
             if ($targetpost > $first) {
                 $first = $targetpost;
             }
         }
     }
     $first_record_postdate = $this->postdate($nntp, $first, false);
     $db->query(sprintf("UPDATE groups SET first_record_postdate = FROM_UNIXTIME(" . $first_record_postdate . "), last_updated = now() WHERE ID = %d", $groupArr['ID']));
     //Set group's first postdate
     $timeGroup = number_format(microtime(true) - $this->startGroup, 2);
     echo "Group processed in {$timeGroup} seconds {$n}";
 }