public function fetchTestBinaries($groupname, $numarticles, $clearexistingbins)
{
$db = new DB();
$nntp = new Nntp();
$binaries = new Binaries();
$groups = new Groups();
$ret = array();
if ($clearexistingbins == true) {
$db->exec('truncate releaseregextesting');
}
$nntp->doConnect();
$groupsToFetch = array();
if (preg_match('/^[a-z]{2,3}(\\.[a-z0-9\\-]+)+$/', $groupname)) {
$groupsToFetch[] = array('name' => $groupname);
} elseif ($groupname === 0) {
$groupsToFetch = $groups->getAll();
} else {
$newsgroups = $nntp->getGroups();
foreach ($newsgroups as $ngroup) {
if (preg_match('/' . $groupname . '/', $ngroup['group'])) {
$groupsToFetch[] = array('name' => $ngroup['group']);
}
}
}
foreach ($groupsToFetch as $groupArr) {
$group = $groupArr['name'];
$data = $nntp->selectGroup($group);
if ($nntp->isError($data)) {
$ret[] = "Could not select group (doesnt exist on USP): {$group}";
continue;
} else {
$rangeStart = $data['last'] - $numarticles;
$rangeEnd = $groupEnd = $data['last'];
$rangeTotal = $rangeEnd - $rangeStart;
$done = false;
while ($done === false) {
if ($rangeTotal > $binaries->messagebuffer) {
if ($rangeStart + $binaries->messagebuffer > $groupEnd) {
$rangeEnd = $groupEnd;
} else {
$rangeEnd = $rangeStart + $binaries->messagebuffer;
}
}
if ($binaries->compressedHeaders) {
$msgs = $nntp->getXOverview($rangeStart . "-" . $rangeEnd, true, false);
} else {
$msgs = $nntp->getOverview($rangeStart . "-" . $rangeEnd, true, false);
}
if ($nntp->isError($msgs)) {
$ret[] = "Error {$msgs->code}: {$msgs->message} on " . $group;
continue 2;
}
$headers = array();
if (is_array($msgs)) {
//loop headers, figure out parts
foreach ($msgs as $msg) {
if (!isset($msg['Number'])) {
continue;
}
$msgPart = $msgTotalParts = 0;
$pattern = '|\\((\\d+)[\\/](\\d+)\\)|i';
preg_match_all($pattern, $msg['Subject'], $matches, PREG_PATTERN_ORDER);
$matchcnt = sizeof($matches[0]);
for ($i = 0; $i < $matchcnt; $i++) {
//not (int)'d here because of the preg_replace later on
$msgPart = $matches[1][$i];
$msgTotalParts = $matches[2][$i];
}
if (!isset($msg['Subject']) || $matchcnt == 0) {
// not a binary post most likely.. continue
continue;
}
if ((int) $msgPart > 0 && (int) $msgTotalParts > 0) {
$subject = utf8_encode(trim(preg_replace('|\\(' . $msgPart . '[\\/]' . $msgTotalParts . '\\)|i', '', $msg['Subject'])));
if (!isset($headers[$subject])) {
$headers[$subject]['Subject'] = $subject;
$headers[$subject]['From'] = $msg['From'];
$headers[$subject]['Date'] = strtotime($msg['Date']);
$headers[$subject]['Message-ID'] = $msg['Message-ID'];
$headers[$subject]['Size'] = $msg['Bytes'];
} else {
$headers[$subject]['Size'] += $msg['Bytes'];
}
}
}
unset($msgs);
if (isset($headers) && count($headers)) {
$groupRegexes = $this->getForGroup($group);
$binSetData = array();
foreach ($headers as $subject => $data) {
$binData = array('name' => $subject, 'fromname' => $data['From'], 'date' => $data['Date'], 'binaryhash' => md5($subject . $data['From'] . $group), 'groupname' => $group, 'regexID' => "null", 'categoryID' => "null", 'reqID' => "null", 'blacklistID' => 0, 'size' => $data['Size'], 'relname' => "null", 'relpart' => "null", 'reltotalpart' => "null");
//Filter binaries based on black/white list
if ($binaries->isBlackListed($data, $group)) {
//binary is blacklisted
$binData['blacklistID'] = 1;
}
//Apply Regexes
$regexMatches = array();
foreach ($groupRegexes as $groupRegex) {
$regexCheck = $this->performMatch($groupRegex, $subject, $data['From']);
if ($regexCheck !== false) {
$regexMatches = $regexCheck;
$binData['regexID'] = $regexCheck['regexID'];
$binData['categoryID'] = $regexCheck['regcatid'];
$binData['reqID'] = empty($regexCheck['reqID']) ? "null" : $regexCheck['reqID'];
$binData['relname'] = $regexCheck['name'];
break;
}
}
$binSetData[] = $binData;
}
//insert 500 bins at a time
$binChunks = array_chunk($binSetData, 500);
foreach ($binChunks as $binChunk) {
foreach ($binChunk as $chunk) {
$binParams[] = sprintf("(%s, %s, FROM_UNIXTIME(%s), %s, %s, %s, %s, %s, %d, %d, now())", $db->escapeString($chunk['name']), $db->escapeString($chunk['fromname']), $db->escapeString($chunk['date']), $db->escapeString($chunk['binaryhash']), $db->escapeString($chunk['groupname']), $chunk['regexID'], $chunk['categoryID'], $chunk['reqID'], $chunk['blacklistID'], $chunk['size']);
}
$binSql = "INSERT IGNORE INTO releaseregextesting (name, fromname, date, binaryhash, groupname, regexID, categoryID, reqID, blacklistID, size, dateadded) VALUES " . implode(', ', $binParams);
//echo $binSql;
$db->exec($binSql);
}
$ret[] = "Fetched " . number_format($numarticles) . " articles from " . $group;
} else {
$ret[] = "No headers found on " . $group;
continue;
}
} else {
$ret[] = "Can't get parts from server (msgs not array) on " . $group;
continue;
}
if ($rangeEnd == $groupEnd) {
$done = true;
}
$rangeStart = $rangeEnd + 1;
}
}
}
$nntp->doQuit();
return $ret;
}
/**
* Process headers and store in database for a group.
*/
function updateGroup($nntp = null, $groupArr)
{
$blnDoDisconnect = false;
if ($nntp == null) {
$nntp = new Nntp();
if (!$nntp->doConnect()) {
echo "Failed to get NNTP connection.";
return;
}
$this->message = array();
$blnDoDisconnect = true;
}
$db = new DB();
$backfill = new Backfill();
$n = $this->n;
$this->startGroup = microtime(true);
$this->startLoop = microtime(true);
echo 'Processing ' . $groupArr['name'] . $n;
// Connect to server
$data = $nntp->selectGroup($groupArr['name']);
if ($nntp->isError($data)) {
echo "Could not select group (bad name?): {$groupArr['name']}{$n} {$n}";
return;
}
if ($groupArr['regexmatchonly'] == 1) {
$this->onlyProcessRegexBinaries = true;
echo "Note: Discarding parts that do not match a regex" . $n;
} else {
$this->onlyProcessRegexBinaries = false;
}
//Attempt to repair any missing parts before grabbing new ones
$this->partRepair($nntp, $groupArr);
//Get first and last part numbers from newsgroup
$last = $grouplast = $data['last'];
// For new newsgroups - determine here how far you want to go back.
if ($groupArr['last_record'] == 0) {
if ($this->NewGroupScanByDays) {
$first = $backfill->daytopost($nntp, $groupArr['name'], $this->NewGroupDaysToScan, true);
if ($first == '') {
echo "Skipping group: {$groupArr['name']}{$n}";
return;
}
} else {
if ($data['first'] > $data['last'] - $this->NewGroupMsgsToScan) {
$first = $data['first'];
} else {
$first = $data['last'] - $this->NewGroupMsgsToScan;
}
}
$first_record_postdate = $backfill->postdate($nntp, $first, false);
if ($first_record_postdate != "") {
$db->exec(sprintf("update groups SET first_record = %s, first_record_postdate = FROM_UNIXTIME(" . $first_record_postdate . ") WHERE ID = %d", $db->escapeString($first), $groupArr['ID']));
}
} else {
if ($data['last'] < $groupArr['last_record']) {
echo "Warning: Server's last num {$data['last']} is lower than the local last num {$groupArr['last_record']}" . $n;
return;
}
$first = $groupArr['last_record'] + 1;
}
// Generate postdates for first and last records, for those that upgraded
if ((is_null($groupArr['first_record_postdate']) || is_null($groupArr['last_record_postdate'])) && ($groupArr['last_record'] != "0" && $groupArr['first_record'] != "0")) {
$db->exec(sprintf("update groups SET first_record_postdate = FROM_UNIXTIME(" . $backfill->postdate($nntp, $groupArr['first_record'], false) . "), last_record_postdate = FROM_UNIXTIME(" . $backfill->postdate($nntp, $groupArr['last_record'], false) . ") WHERE ID = %d", $groupArr['ID']));
}
// Deactivate empty groups
if ($data['last'] - $data['first'] <= 5) {
$db->exec(sprintf("update groups SET active = %s, last_updated = now() WHERE ID = %d", $db->escapeString('0'), $groupArr['ID']));
}
// Calculate total number of parts
$total = $grouplast - $first + 1;
// If total is bigger than 0 it means we have new parts in the newsgroup
if ($total > 0) {
echo "Group " . $data["group"] . " has " . number_format($total) . " new parts." . $n;
if ($total > $this->MaxMsgsPerRun) {
$grouplast = $first + $this->MaxMsgsPerRun;
echo "NOTICE: Only processing first " . number_format($this->MaxMsgsPerRun) . " parts." . $n;
}
echo "First: " . $data['first'] . " Last: " . $data['last'] . " Local last: " . $groupArr['last_record'] . $n;
if ($groupArr['last_record'] == 0) {
echo "New group starting with " . ($this->NewGroupScanByDays ? $this->NewGroupDaysToScan . " days" : $this->NewGroupMsgsToScan . " messages") . " worth." . $n;
}
$done = false;
// Get all the parts (in portions of $this->messagebuffer to not use too much memory)
while ($done === false) {
$this->startLoop = microtime(true);
if ($total > $this->messagebuffer) {
if ($first + $this->messagebuffer > $grouplast) {
$last = $grouplast;
} else {
$last = $first + $this->messagebuffer;
}
}
echo "Getting " . number_format($last - $first + 1) . " parts (" . $first . " to " . $last . ") - " . number_format($grouplast - $last) . " in queue" . $n;
flush();
//get headers from newsgroup
$lastId = $this->scan($nntp, $groupArr, $first, $last);
if ($lastId === false) {
//scan failed - skip group
return;
}
$db->exec(sprintf("update groups SET last_record = %s, last_updated = now() WHERE ID = %d", $db->escapeString($lastId), $groupArr['ID']));
if ($last == $grouplast) {
$done = true;
} else {
$last = $lastId;
$first = $last + 1;
}
}
$last_record_postdate = $backfill->postdate($nntp, $last, false);
if ($last_record_postdate != "") {
$db->exec(sprintf("update groups SET last_record_postdate = FROM_UNIXTIME(" . $last_record_postdate . "), last_updated = now() WHERE ID = %d", $groupArr['ID']));
//Set group's last postdate
}
$timeGroup = number_format(microtime(true) - $this->startGroup, 2);
echo "Group processed in {$timeGroup} seconds {$n} {$n}";
} else {
echo "No new records for " . $data["group"] . " (first {$first} last {$last} total {$total}) grouplast " . $groupArr['last_record'] . $n . $n;
}
if ($blnDoDisconnect) {
$nntp->doQuit();
}
}
/**
* Update a group back to a specified date.
*/
function backfillGroup($groupArr, $backfillDate = null, $regexOnly = false)
{
$db = new DB();
$binaries = new Binaries();
$n = $this->n;
if ($regexOnly === true) {
echo "Only inserting binaries which match regex{$n}";
$binaries->onlyProcessRegexBinaries = true;
}
$this->startGroup = microtime(true);
$nntp = new Nntp();
$nntpc = new Nntp();
//Make sure we actually have a connection going before doing anything.
if ($nntp->doConnect(5, false, true)) {
echo 'Processing ' . $groupArr['name'] . $n;
$data = $nntp->selectGroup($groupArr['name']);
if ($nntp->isError($data)) {
echo "Could not select group (bad name?): {$groupArr['name']}{$n}";
return;
}
if ($backfillDate) {
$targetpost = $this->daytopost($nntp, $groupArr['name'], $this->dateToDays($backfillDate), TRUE);
} else {
$targetpost = $this->daytopost($nntp, $groupArr['name'], $groupArr['backfill_target'], TRUE);
}
//get targetpost based on days target
if ($groupArr['first_record'] == 0 || $groupArr['backfill_target'] == 0 && !$backfillDate) {
echo "Group " . $groupArr['name'] . " has invalid numbers. Have you run update on it? Have you set the backfill days amount?{$n}";
return;
}
echo "Group " . $data["group"] . ": server has " . $data['first'] . " - " . $data['last'] . ", or ~";
echo (int) (($this->postdate($nntp, $data['last'], FALSE) - $this->postdate($nntp, $data['first'], FALSE)) / 86400);
echo " days." . $n . "Local first = " . $groupArr['first_record'] . " (";
echo (int) ((date('U') - $this->postdate($nntp, $groupArr['first_record'], FALSE)) / 86400);
echo " days). Backfill target of " . ($backfillDate ? date('Y-m-d', $backfillDate) : $groupArr['backfill_target'] . " days") . " is post {$targetpost}.{$n}";
if ($targetpost >= $groupArr['first_record']) {
echo "Nothing to do, we already have the target post.{$n} {$n}";
return "";
}
//get first and last part numbers from newsgroup
if ($targetpost < $data['first']) {
echo "WARNING: Backfill came back as before server's first. Setting targetpost to server first.{$n}";
echo "Skipping Group {$n}";
return "";
}
echo $binaries->onlyProcessRegexBinaries === true ? "Note: Discarding parts that do not match a regex" . $n : "";
//Done with $nntp for now, close it to avoid timeouts.
$nntp->doQuit();
$nntpc->doConnect();
$datac = $nntpc->selectGroup($groupArr['name']);
if ($nntpc->isError($datac)) {
echo "Could not select group (bad name?): {$groupArr['name']}{$n}";
return;
}
//calculate total number of parts
$total = $groupArr['first_record'] - $targetpost;
$done = false;
//set first and last, moving the window by maxxMssgs
$last = $groupArr['first_record'] - 1;
$first = $last - $binaries->messagebuffer + 1;
//set initial "chunk"
if ($targetpost > $first) {
//just in case this is the last chunk we needed
$first = $targetpost;
}
while ($done === false) {
$binaries->startLoop = microtime(true);
echo "Getting " . ($last - $first + 1) . " parts (" . number_format($first - $targetpost) . " in queue)" . $n;
flush();
$success = $binaries->scan($nntpc, $groupArr, $first, $last, 'backfill');
if (!$success) {
return "";
}
$db->exec(sprintf("update groups SET first_record = %s, last_updated = now() WHERE ID = %d", $db->escapeString($first), $groupArr['ID']));
if ($first == $targetpost) {
$done = true;
} else {
//Keep going: set new last, new first, check for last chunk.
$last = $first - 1;
$first = $last - $binaries->messagebuffer + 1;
if ($targetpost > $first) {
$first = $targetpost;
}
}
}
//Done with $nntpc
$nntpc->doQuit();
//Just need $nntp for a quick check on the first_record_postdate
$nntp->doConnect();
//$nntp->selectGroup($groupArr['name']); // some users report having this in keeps backfill working
$first_record_postdate = $this->postdate($nntp, $first, false);
//All done with NNTP.
$nntp->doQuit();
if ($first_record_postdate != "") {
$db->exec(sprintf("update groups SET first_record_postdate = FROM_UNIXTIME(" . $first_record_postdate . "), last_updated = now() WHERE ID = %d", $groupArr['ID']));
}
//Set group's first postdate
$timeGroup = number_format(microtime(true) - $this->startGroup, 2);
echo "Group processed in {$timeGroup} seconds {$n}";
} else {
echo "Failed to get NNTP connection.{$n}";
}
}
/**
* Get nzpre data from usenet and parse.
*/
public function nzpreUpdate()
{
require_once WWW_DIR . "/lib/nntp.php";
$s = new Sites();
$site = $s->get();
if (empty($site->nzpregroup) || empty($site->nzpresubject) || empty($site->nzpreposter) || empty($site->nzprefield) || empty($site->nzprekey)) {
return false;
}
if ($this->echooutput) {
echo "Predb : Checking for new pre data ";
}
$db = new DB();
$nntp = new Nntp();
if (!$nntp->doConnect()) {
echo "Failed to get NNTP connection\n";
return false;
}
$ret = $groupData = $nntp->selectGroup($site->nzpregroup);
if ($nntp->isError($ret)) {
echo "Predb : Error " . $ret->getMessage() . "\n";
return false;
}
$ret = $groupMsgs = $nntp->getOverview($groupData['last'] - (!empty($site->nzprearticles) ? $site->nzprearticles : 500) . '-' . $groupData['last']);
if ($nntp->isError($ret)) {
echo "Predb : Error " . $ret->getMessage() . "\n";
return false;
}
$added_updated = 0;
$nzprekey = $site->nzprekey;
while (strlen($nzprekey) < 1024) {
$nzprekey = $nzprekey . $nzprekey;
}
$cnt = !empty($site->nzprearticles) ? $site->nzprearticles : 500;
foreach ($groupMsgs as $groupMsg) {
if ($cnt % 50 == 0 && $cnt != 0 && $this->echooutput) {
echo $cnt . "..";
}
$cnt--;
if (preg_match('/^' . $site->nzpresubject . '$/', $groupMsg['Subject']) && preg_match('/^' . $site->nzpreposter . '$/', $groupMsg['From'])) {
$ret = $msgHeader = $nntp->getHeader($groupMsg['Message-ID']);
if ($nntp->isError($ret)) {
continue;
}
for ($i = 0; $i < count($msgHeader); $i++) {
if (preg_match('/^' . $site->nzprefield . ': /', $msgHeader[$i])) {
if ($nzpreParse = $this->nzpreParse(str_replace($site->nzprefield . ': ', '', $msgHeader[$i]), $nzprekey)) {
if ($this->updatePreDB($db, $nzpreParse)) {
$added_updated++;
}
}
break;
}
}
}
}
$nntp->disconnect();
if ($this->echooutput) {
echo "\nPredb : Added/Updated " . $added_updated . " records\n";
}
}
