/**
* Update the list of newsgroups from nntp provider matching a regex and return an array of messages.
*/
function addBulk($groupList, $active = 1)
{
require_once WWW_DIR . "/lib/binaries.php";
require_once WWW_DIR . "/lib/nntp.php";
$ret = array();
if ($groupList == "") {
$ret[] = "No group list provided.";
} else {
$db = new DB();
$nntp = new Nntp();
if (!$nntp->doConnect()) {
$ret[] = "Failed to get NNTP connection";
return $ret;
}
$groups = $nntp->getGroups();
$nntp->doQuit();
$regfilter = "/(" . str_replace(array('.', '*'), array('\\.', '.*?'), $groupList) . ")\$/";
foreach ($groups as $group) {
if (preg_match($regfilter, $group['group']) > 0) {
$res = $db->queryOneRow(sprintf("SELECT ID FROM groups WHERE name = %s ", $db->escapeString($group['group'])));
if ($res) {
$db->exec(sprintf("update groups SET active = %d where ID = %d", $active, $res["ID"]));
$ret[] = array('group' => $group['group'], 'msg' => 'Updated');
} else {
$desc = "";
$db->queryInsert(sprintf("INSERT INTO groups (name, description, active) VALUES (%s, %s, %d)", $db->escapeString($group['group']), $db->escapeString($desc), $active));
$ret[] = array('group' => $group['group'], 'msg' => 'Created');
}
}
}
}
return $ret;
}
public function fetchTestBinaries($groupname, $numarticles, $clearexistingbins)
{
$db = new DB();
$nntp = new Nntp();
$binaries = new Binaries();
$groups = new Groups();
$ret = array();
if ($clearexistingbins == true) {
$db->exec('truncate releaseregextesting');
}
$nntp->doConnect();
$groupsToFetch = array();
if (preg_match('/^[a-z]{2,3}(\\.[a-z0-9\\-]+)+$/', $groupname)) {
$groupsToFetch[] = array('name' => $groupname);
} elseif ($groupname === 0) {
$groupsToFetch = $groups->getAll();
} else {
$newsgroups = $nntp->getGroups();
foreach ($newsgroups as $ngroup) {
if (preg_match('/' . $groupname . '/', $ngroup['group'])) {
$groupsToFetch[] = array('name' => $ngroup['group']);
}
}
}
foreach ($groupsToFetch as $groupArr) {
$group = $groupArr['name'];
$data = $nntp->selectGroup($group);
if ($nntp->isError($data)) {
$ret[] = "Could not select group (doesnt exist on USP): {$group}";
continue;
} else {
$rangeStart = $data['last'] - $numarticles;
$rangeEnd = $groupEnd = $data['last'];
$rangeTotal = $rangeEnd - $rangeStart;
$done = false;
while ($done === false) {
if ($rangeTotal > $binaries->messagebuffer) {
if ($rangeStart + $binaries->messagebuffer > $groupEnd) {
$rangeEnd = $groupEnd;
} else {
$rangeEnd = $rangeStart + $binaries->messagebuffer;
}
}
if ($binaries->compressedHeaders) {
$msgs = $nntp->getXOverview($rangeStart . "-" . $rangeEnd, true, false);
} else {
$msgs = $nntp->getOverview($rangeStart . "-" . $rangeEnd, true, false);
}
if ($nntp->isError($msgs)) {
$ret[] = "Error {$msgs->code}: {$msgs->message} on " . $group;
continue 2;
}
$headers = array();
if (is_array($msgs)) {
//loop headers, figure out parts
foreach ($msgs as $msg) {
if (!isset($msg['Number'])) {
continue;
}
$msgPart = $msgTotalParts = 0;
$pattern = '|\\((\\d+)[\\/](\\d+)\\)|i';
preg_match_all($pattern, $msg['Subject'], $matches, PREG_PATTERN_ORDER);
$matchcnt = sizeof($matches[0]);
for ($i = 0; $i < $matchcnt; $i++) {
//not (int)'d here because of the preg_replace later on
$msgPart = $matches[1][$i];
$msgTotalParts = $matches[2][$i];
}
if (!isset($msg['Subject']) || $matchcnt == 0) {
// not a binary post most likely.. continue
continue;
}
if ((int) $msgPart > 0 && (int) $msgTotalParts > 0) {
$subject = utf8_encode(trim(preg_replace('|\\(' . $msgPart . '[\\/]' . $msgTotalParts . '\\)|i', '', $msg['Subject'])));
if (!isset($headers[$subject])) {
$headers[$subject]['Subject'] = $subject;
$headers[$subject]['From'] = $msg['From'];
$headers[$subject]['Date'] = strtotime($msg['Date']);
$headers[$subject]['Message-ID'] = $msg['Message-ID'];
$headers[$subject]['Size'] = $msg['Bytes'];
} else {
$headers[$subject]['Size'] += $msg['Bytes'];
}
}
}
unset($msgs);
if (isset($headers) && count($headers)) {
$groupRegexes = $this->getForGroup($group);
$binSetData = array();
foreach ($headers as $subject => $data) {
$binData = array('name' => $subject, 'fromname' => $data['From'], 'date' => $data['Date'], 'binaryhash' => md5($subject . $data['From'] . $group), 'groupname' => $group, 'regexID' => "null", 'categoryID' => "null", 'reqID' => "null", 'blacklistID' => 0, 'size' => $data['Size'], 'relname' => "null", 'relpart' => "null", 'reltotalpart' => "null");
//Filter binaries based on black/white list
if ($binaries->isBlackListed($data, $group)) {
//binary is blacklisted
$binData['blacklistID'] = 1;
}
//Apply Regexes
$regexMatches = array();
foreach ($groupRegexes as $groupRegex) {
$regexCheck = $this->performMatch($groupRegex, $subject, $data['From']);
if ($regexCheck !== false) {
$regexMatches = $regexCheck;
$binData['regexID'] = $regexCheck['regexID'];
$binData['categoryID'] = $regexCheck['regcatid'];
$binData['reqID'] = empty($regexCheck['reqID']) ? "null" : $regexCheck['reqID'];
$binData['relname'] = $regexCheck['name'];
break;
}
}
$binSetData[] = $binData;
}
//insert 500 bins at a time
$binChunks = array_chunk($binSetData, 500);
foreach ($binChunks as $binChunk) {
foreach ($binChunk as $chunk) {
$binParams[] = sprintf("(%s, %s, FROM_UNIXTIME(%s), %s, %s, %s, %s, %s, %d, %d, now())", $db->escapeString($chunk['name']), $db->escapeString($chunk['fromname']), $db->escapeString($chunk['date']), $db->escapeString($chunk['binaryhash']), $db->escapeString($chunk['groupname']), $chunk['regexID'], $chunk['categoryID'], $chunk['reqID'], $chunk['blacklistID'], $chunk['size']);
}
$binSql = "INSERT IGNORE INTO releaseregextesting (name, fromname, date, binaryhash, groupname, regexID, categoryID, reqID, blacklistID, size, dateadded) VALUES " . implode(', ', $binParams);
//echo $binSql;
$db->exec($binSql);
}
$ret[] = "Fetched " . number_format($numarticles) . " articles from " . $group;
} else {
$ret[] = "No headers found on " . $group;
continue;
}
} else {
$ret[] = "Can't get parts from server (msgs not array) on " . $group;
continue;
}
if ($rangeEnd == $groupEnd) {
$done = true;
}
$rangeStart = $rangeEnd + 1;
}
}
}
$nntp->doQuit();
return $ret;
}