/**
* Download a range of usenet messages. Store binaries with subjects matching a
* specific pattern in the database.
*
* @param $groupArr
* @param $first
* @param $last
* @param string $type
*
* @return array
*/
function scan($groupArr, $first, $last, $type = 'update')
{
$db = new Settings();
$releaseRegex = new ReleaseRegex();
$n = $this->n;
// Check if MySQL tables exist, create if they do not, get their names at the same time.
$tableNames = $this->_groups->getCBPTableNames($this->_tablePerGroup, $groupArr['id']);
$partRepair = $type === 'partrepair';
$returnArray = [];
// Download the headers.
if ($partRepair === true) {
// This is slower but possibly is better with missing headers.
$msgs = $this->_nntp->getOverview($first . '-' . $last, true, false);
} else {
$msgs = $this->_nntp->getXOVER($first . '-' . $last);
}
// If there was an error, try to reconnect.
if ($this->_nntp->isError($msgs)) {
// Increment if part repair and return false.
if ($partRepair === true) {
$this->_pdo->queryExec(sprintf('UPDATE %s SET attempts = attempts + 1 WHERE groupid = %d AND numberid %s', $tableNames['prname'], $groupArr['id'], $first == $last ? '= ' . $first : 'IN (' . implode(',', range($first, $last)) . ')'));
return $returnArray;
}
// This is usually a compression error, so try disabling compression.
$this->_nntp->doQuit();
if ($this->_nntp->doConnect(false) !== true) {
return $returnArray;
}
// Re-select group, download headers again without compression and re-enable compression.
$this->_nntp->selectGroup($groupArr['name']);
$msgs = $this->_nntp->getXOVER($first . '-' . $last);
$this->_nntp->enableCompression();
// Check if the non-compression headers have an error.
if ($this->_nntp->isError($msgs)) {
$this->log("Code {$msgs->code}: {$msgs->message}\nSkipping group: {$groupArr['name']}", 'scan', \Logger::LOG_WARNING, 'error');
return $returnArray;
}
}
$rangerequested = range($first, $last);
$msgsreceived = array();
$msgsblacklisted = array();
$msgsignored = array();
$msgsinserted = array();
$msgsnotinserted = array();
$timeHeaders = number_format(microtime(true) - $this->startHeaders, 2);
if ($this->_nntp->isError($msgs)) {
echo "Error {$msgs->code}: {$msgs->message}{$n}";
echo "Skipping group{$n}";
return $returnArray;
}
// Check if we got headers.
$msgCount = count($msgs);
if ($msgCount < 1) {
return $returnArray;
}
// Get highest and lowest article numbers/dates.
$iterator1 = 0;
$iterator2 = $msgCount - 1;
while (true) {
if (!isset($returnArray['firstArticleNumber']) && isset($msgs[$iterator1]['Number'])) {
$returnArray['firstArticleNumber'] = $msgs[$iterator1]['Number'];
$returnArray['firstArticleDate'] = $msgs[$iterator1]['Date'];
}
if (!isset($returnArray['lastArticleNumber']) && isset($msgs[$iterator2]['Number'])) {
$returnArray['lastArticleNumber'] = $msgs[$iterator2]['Number'];
$returnArray['lastArticleDate'] = $msgs[$iterator2]['Date'];
}
// Break if we found non empty articles.
if (isset($returnArray['firstArticleNumber']) && isset($returnArray['lastArticleNumber'])) {
break;
}
// Break out if we couldn't find anything.
if ($iterator1++ >= $msgCount - 1 || $iterator2-- <= 0) {
break;
}
}
if (is_array($msgs)) {
//loop headers, figure out parts
foreach ($msgs as $msg) {
if (!isset($msg['Number'])) {
continue;
}
$msgsreceived[] = $msg['Number'];
$msgPart = $msgTotalParts = 0;
$pattern = '|\\((\\d+)[\\/](\\d+)\\)|i';
preg_match_all($pattern, $msg['Subject'], $matches, PREG_PATTERN_ORDER);
$matchcnt = sizeof($matches[0]);
for ($i = 0; $i < $matchcnt; $i++) {
$msgPart = $matches[1][$i];
$msgTotalParts = $matches[2][$i];
}
if (!isset($msg['Subject']) || $matchcnt == 0) {
$msgsignored[] = $msg['Number'];
continue;
}
if ((int) $msgPart > 0 && (int) $msgTotalParts > 0) {
$subject = utf8_encode(trim(preg_replace('|\\(' . $msgPart . '[\\/]' . $msgTotalParts . '\\)|i', '', $msg['Subject'])));
if (!isset($this->message[$subject])) {
$this->message[$subject] = $msg;
$this->message[$subject]['MaxParts'] = (int) $msgTotalParts;
$this->message[$subject]['Date'] = strtotime($this->message[$subject]['Date']);
}
if ((int) $msgPart > 0) {
$this->message[$subject]['Parts'][(int) $msgPart] = array('Message-ID' => substr($msg['Message-ID'], 1, -1), 'number' => $msg['Number'], 'part' => (int) $msgPart, 'size' => $msg['Bytes']);
$this->message[$subject]['PartNumbers'][(int) $msgPart] = $msg['Number'];
}
}
}
unset($msg);
unset($msgs);
$count = 0;
$updatecount = 0;
$partcount = 0;
$rangenotreceived = array_diff($rangerequested, $msgsreceived);
if ($type != 'partrepair') {
echo "Received " . sizeof($msgsreceived) . " articles of " . ($last - $first + 1) . " requested, " . sizeof($msgsignored) . " not binaries {$n}";
}
if ($type == 'update' && sizeof($msgsreceived) == 0) {
echo "Error: Server did not return any articles.{$n}";
echo "Skipping group{$n}";
return $returnArray;
}
if (sizeof($rangenotreceived) > 0) {
switch ($type) {
case 'backfill':
//don't add missing articles
break;
case 'partrepair':
case 'update':
default:
$this->addMissingParts($rangenotreceived, $tableNames['prname'], $groupArr['id']);
break;
}
echo "Server did not return " . count($rangenotreceived) . " article(s).{$n}";
}
if (isset($this->message) && count($this->message)) {
$groupRegexes = $releaseRegex->getForGroup($groupArr['name']);
//insert binaries and parts into database. when binary already exists; only insert new parts
foreach ($this->message as $subject => $data) {
//Filter binaries based on black/white list
if ($this->isBlackListed($data, $groupArr['name'])) {
$msgsblacklisted[] = count($data['Parts']);
if ($type == 'partrepair') {
$partIds = array();
foreach ($data['Parts'] as $partdata) {
$partIds[] = $partdata['number'];
}
$db->queryExec(sprintf("DELETE FROM %s WHERE numberid IN (%s) AND groupid=%d", $tableNames['prname'], implode(',', $partIds), $groupArr['id']));
}
continue;
}
if (isset($data['Parts']) && count($data['Parts']) > 0 && $subject != '') {
//Check for existing binary
$binaryID = 0;
$binaryHash = md5($subject . $data['From'] . $groupArr['id']);
$res = $db->queryOneRow(sprintf("SELECT id FROM %s WHERE binaryhash = %s", $tableNames['bname'], $db->escapeString($binaryHash)));
if (!$res) {
//Apply Regexes
$regexMatches = array();
foreach ($groupRegexes as $groupRegex) {
$regexCheck = $releaseRegex->performMatch($groupRegex, $subject);
if ($regexCheck !== false) {
$regexMatches = $regexCheck;
break;
}
}
$sql = '';
if (!empty($regexMatches)) {
$relparts = explode("/", $regexMatches['parts']);
$sql = sprintf('INSERT INTO %s (name, fromname, date, xref, totalparts, groupid, procstat, categoryid, regexid, reqid, relpart, reltotalpart, binaryhash, relname, dateadded) VALUES (%s, %s, FROM_UNIXTIME(%s), %s, %s, %d, %d, %s, %d, %s, %d, %d, %s, %s, now())', $tableNames['bname'], $db->escapeString($subject), $db->escapeString(utf8_encode($data['From'])), $db->escapeString($data['Date']), $db->escapeString($data['Xref']), $db->escapeString($data['MaxParts']), $groupArr['id'], Releases::PROCSTAT_TITLEMATCHED, $regexMatches['regcatid'], $regexMatches['regexid'], $db->escapeString($regexMatches['reqid']), $relparts[0], $relparts[1], $db->escapeString($binaryHash), $db->escapeString(str_replace('_', ' ', $regexMatches['name'])));
} elseif ($this->onlyProcessRegexBinaries === false) {
$sql = sprintf('INSERT INTO %s (name, fromname, date, xref, totalparts, groupid, binaryhash, dateadded) VALUES (%s, %s, FROM_UNIXTIME(%s), %s, %s, %d, %s, now())', $tableNames['bname'], $db->escapeString($subject), $db->escapeString(utf8_encode($data['From'])), $db->escapeString($data['Date']), $db->escapeString($data['Xref']), $db->escapeString($data['MaxParts']), $groupArr['id'], $db->escapeString($binaryHash));
} elseif ($type == 'partrepair') {
$partIds = array();
foreach ($data['Parts'] as $partdata) {
$partIds[] = $partdata['number'];
}
$db->queryExec(sprintf('DELETE FROM %s WHERE numberid IN (%s) AND groupid = %d', $tableNames['prname'], implode(',', $partIds), $groupArr['id']));
continue;
}
if ($sql != '') {
$binaryID = $db->queryInsert($sql);
$count++;
//if ($count % 500 == 0) echo "$count bin adds...";
}
} else {
$binaryID = $res["id"];
$updatecount++;
//if ($updatecount % 500 == 0) echo "$updatecount bin updates...";
}
if ($binaryID != 0) {
$partParams = array();
$partNumbers = array();
foreach ($data['Parts'] as $partdata) {
$partcount++;
$partParams[] = sprintf('(%d, %s, %s, %s, %s)', $binaryID, $db->escapeString($partdata['Message-ID']), $db->escapeString($partdata['number']), $db->escapeString(round($partdata['part'])), $db->escapeString($partdata['size']));
$partNumbers[] = $partdata['number'];
}
$partSql = 'INSERT INTO ' . $tableNames['pname'] . ' (binaryid, messageid, number, partnumber, size) VALUES ' . implode(', ', $partParams);
$pidata = $db->queryInsert($partSql);
if (!$pidata) {
$msgsnotinserted = array_merge($msgsnotinserted, $partNumbers);
} else {
$msgsinserted = array_merge($msgsinserted, $partNumbers);
}
}
}
}
if (!empty($this->_binaryBlacklistIdsToUpdate)) {
$this->_pdo->queryExec(sprintf('UPDATE binaryblacklist SET last_activity = NOW() WHERE id IN (%s)', implode(',', $this->_binaryBlacklistIdsToUpdate)));
$this->_binaryBlacklistIdsToUpdate = [];
}
//TODO: determine whether to add to missing articles if insert failed
if (sizeof($msgsnotinserted) > 0) {
echo 'WARNING: ' . count($msgsnotinserted) . ' Parts failed to insert' . $n;
$this->addMissingParts($msgsnotinserted, $tableNames['prname'], $groupArr['id']);
}
if ($count >= 500 || $updatecount >= 500) {
echo $n;
}
//line break for bin adds output
}
$timeUpdate = number_format(microtime(true) - $this->startUpdate, 2);
$timeLoop = number_format(microtime(true) - $this->startLoop, 2);
if (sizeof($msgsblacklisted) > 0) {
echo "Blacklisted " . array_sum($msgsblacklisted) . " parts in " . sizeof($msgsblacklisted) . " binaries" . $n;
}
if ($type != 'partrepair') {
if ($this->_echoCLI) {
$this->_colorCLI->doEcho($this->_colorCLI->alternateOver(number_format($count)) . $this->_colorCLI->primaryOver(' new, ') . $this->_colorCLI->alternateOver(number_format($updatecount)) . $this->_colorCLI->primaryOver(' updated, ') . $this->_colorCLI->alternateOver(number_format($partcount)) . $this->_colorCLI->primaryOver(' parts, ') . $this->_colorCLI->alternateOver($timeHeaders . 's') . $this->_colorCLI->primaryOver(' to download articles, ') . $this->_colorCLI->alternateOver($timeUpdate . 's') . $this->_colorCLI->primaryOver(' to insert binaries/parts, ') . $this->_colorCLI->alternateOver($timeLoop . 's') . $this->_colorCLI->primary(' total.'));
}
}
unset($this->message);
unset($data);
return $returnArray;
} else {
echo "Error: Can't get parts from server (msgs not array) {$n}";
echo "Skipping group{$n}";
return $returnArray;
}
}
<?php
require_once "config.php";
require_once WWW_DIR . "/lib/adminpage.php";
require_once WWW_DIR . "/lib/releaseregex.php";
require_once WWW_DIR . "/lib/category.php";
$page = new AdminPage();
$category = new Category();
$reg = new ReleaseRegex();
$id = 0;
// set the current action
$action = isset($_REQUEST['action']) ? $_REQUEST['action'] : 'view';
switch ($action) {
case 'submit':
if ($_POST["id"] == "") {
$reg->add($_POST);
} else {
$ret = $reg->update($_POST);
}
header("Location:" . WWW_TOP . "/regex-list.php");
break;
case 'addtest':
if (isset($_GET['regex']) && isset($_GET['groupname'])) {
$r = array('groupname' => $_GET['groupname'], 'regex' => $_GET['regex'], 'ordinal' => '1', 'status' => '1');
$page->smarty->assign('regex', $r);
}
break;
case 'view':
default:
$page->title = "Release Regex Add";
if (isset($_GET["id"])) {
<?php
require_once "config.php";
$page = new AdminPage();
$reg = new ReleaseRegex();
$page->title = "Release Regex List";
$reggrouplist = $reg->getGroupsForSelect();
$page->smarty->assign('reggrouplist', $reggrouplist);
$group = ".*";
if (isset($_REQUEST["group"])) {
$group = $_REQUEST["group"];
}
$page->smarty->assign('selectedgroup', $group);
$regexlist = $reg->get(false, $group, true);
$page->smarty->assign('regexlist', $regexlist);
$page->content = $page->smarty->fetch('regex-list.tpl');
$page->render();
function processReleases()
{
require_once WWW_DIR . "/lib/binaries.php";
$db = new DB();
$currTime_ori = $db->queryOneRow("SELECT NOW() as now");
$cat = new Category();
$nzb = new Nzb();
$s = new Sites();
$releaseRegex = new ReleaseRegex();
$page = new Page();
$groups = new Groups();
$retcount = 0;
echo $s->getLicense();
echo "\n\nStarting release update process (" . date("Y-m-d H:i:s") . ")\n";
if (!file_exists($page->site->nzbpath)) {
echo "Bad or missing nzb directory - " . $page->site->nzbpath;
return -1;
}
//
// Get all regexes for all groups which are to be applied to new binaries
// in order of how they should be applied
//
$releaseRegex->get();
echo "Stage 1 : Applying regex to binaries\n";
$activeCategories = $cat->get();
$catbasedsizes = $db->getLookupAsArray($activeCategories, "ID");
$activeGroups = $groups->getActive(false);
$groupbasedminsizes = $db->getLookupAsArray($groups->getAllNoReleases(), "ID");
foreach ($activeGroups as $groupArr) {
//check if regexes have already been applied during update binaries
if ($groupArr['regexmatchonly'] == 1) {
continue;
}
$groupRegexes = $releaseRegex->getForGroup($groupArr['name']);
echo "Stage 1 : Applying " . sizeof($groupRegexes) . " regexes to group " . $groupArr['name'] . "\n";
// Get out all binaries of STAGE0 for current group
$newUnmatchedBinaries = array();
$ressql = sprintf("SELECT binaries.ID, binaries.name, binaries.date, binaries.totalParts, binaries.procstat, binaries.fromname from binaries where groupID = %d and procstat IN (%d,%d) and regexID IS NULL order by binaries.date asc", $groupArr['ID'], Releases::PROCSTAT_NEW, Releases::PROCSTAT_TITLENOTMATCHED);
$resbin = $db->queryDirect($ressql);
$matchedbins = 0;
while ($rowbin = $db->getAssocArray($resbin)) {
$regexMatches = array();
foreach ($groupRegexes as $groupRegex) {
$regexCheck = $releaseRegex->performMatch($groupRegex, $rowbin['name']);
if ($regexCheck !== false) {
$regexMatches = $regexCheck;
break;
}
}
if (!empty($regexMatches)) {
$matchedbins++;
$relparts = explode("/", $regexMatches['parts']);
$db->exec(sprintf("update binaries set relname = replace(%s, '_', ' '), relpart = %d, reltotalpart = %d, procstat=%d, categoryID=%s, regexID=%d, reqID=%s where ID = %d", $db->escapeString($regexMatches['name']), $relparts[0], $relparts[1], Releases::PROCSTAT_TITLEMATCHED, $regexMatches['regcatid'], $regexMatches['regexID'], $db->escapeString($regexMatches['reqID']), $rowbin["ID"]));
} else {
if ($rowbin['procstat'] == Releases::PROCSTAT_NEW) {
$newUnmatchedBinaries[] = $rowbin['ID'];
}
}
}
//mark as not matched
if (!empty($newUnmatchedBinaries)) {
$db->exec(sprintf("update binaries set procstat=%d where ID IN (%s)", Releases::PROCSTAT_TITLENOTMATCHED, implode(',', $newUnmatchedBinaries)));
}
}
//
// Move all binaries from releases which have the correct number of files on to the next stage.
//
echo "Stage 2 : Marking binaries where all parts are available";
$result = $db->queryDirect(sprintf("SELECT relname, date, SUM(reltotalpart) AS reltotalpart, groupID, reqID, fromname, SUM(num) AS num, coalesce(g.minfilestoformrelease, s.minfilestoformrelease) as minfilestoformrelease FROM ( SELECT relname, reltotalpart, groupID, reqID, fromname, max(date) as date, COUNT(ID) AS num FROM binaries WHERE procstat = %s GROUP BY relname, reltotalpart, groupID, reqID, fromname ORDER BY NULL ) x left outer join groups g on g.ID = x.groupID inner join ( select value as minfilestoformrelease from site where setting = 'minfilestoformrelease' ) s GROUP BY relname, groupID, reqID, fromname, minfilestoformrelease ORDER BY NULL", Releases::PROCSTAT_TITLEMATCHED));
while ($row = $db->getAssocArray($result)) {
$retcount++;
//
// Less than the site permitted number of files in a release. Dont discard it, as it may
// be part of a set being uploaded.
//
if ($row["num"] < $row["minfilestoformrelease"]) {
//echo "Number of files in release ".$row["relname"]." less than site/group setting (".$row['num']."/".$row["minfilestoformrelease"].")\n";
//$db->exec(sprintf("update binaries set procattempts = procattempts + 1 where relname = %s and procstat = %d and groupID = %d and fromname = %s", $db->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $db->escapeString($row["fromname"]) ));
} elseif ($row["num"] >= $row["reltotalpart"]) {
$incomplete = false;
if ($row['reltotalpart'] == 0 && strtotime($currTime_ori['now']) - strtotime($row['date']) < 14400) {
$incomplete = true;
} else {
// Check that the binary is complete
$binlist = $db->query(sprintf("SELECT binaries.ID, totalParts, date, COUNT(DISTINCT parts.messageID) AS num FROM binaries, parts WHERE binaries.ID=parts.binaryID AND binaries.relname = %s AND binaries.procstat = %d AND binaries.groupID = %d AND binaries.fromname = %s GROUP BY binaries.ID ORDER BY NULL", $db->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $db->escapeString($row["fromname"])));
foreach ($binlist as $rowbin) {
if ($rowbin['num'] < $rowbin['totalParts']) {
// Allow to binary to release if posted to usenet longer than four hours ago and we still don't have all the parts
if (!(strtotime($currTime_ori['now']) - strtotime($rowbin['date']) > 14400)) {
$incomplete = true;
break;
}
}
}
}
if (!$incomplete) {
//
// Right number of files, but see if the binary is a allfilled/reqid post, in which case it needs its name looked up
//
if ($row['reqID'] != '' && $page->site->reqidurl != "") {
//
// Try and get the name using the group
//
$binGroup = $db->queryOneRow(sprintf("SELECT name FROM groups WHERE ID = %d", $row["groupID"]));
$newtitle = $this->getReleaseNameForReqId($page->site->reqidurl, $page->site->newznabID, $binGroup["name"], $row["reqID"]);
//
// if the feed/group wasnt supported by the scraper, then just use the release name as the title.
//
if ($newtitle == "no feed") {
$newtitle = $row["relname"];
}
//
// Valid release with right number of files and title now, so move it on
//
if ($newtitle != "") {
$db->exec(sprintf("update binaries set relname = %s, procstat=%d where relname = %s and procstat = %d and groupID = %d and fromname=%s", $db->escapeString($newtitle), Releases::PROCSTAT_READYTORELEASE, $db->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $db->escapeString($row["fromname"])));
} else {
//
// Item not found, if the binary was added to the index yages ago, then give up.
//
$maxaddeddate = $db->queryOneRow(sprintf("SELECT NOW() as now, MAX(dateadded) as dateadded FROM binaries WHERE relname = %s and procstat = %d and groupID = %d and fromname=%s", $db->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $db->escapeString($row["fromname"])));
//
// If added to the index over 48 hours ago, give up trying to determine the title
//
if (strtotime($maxaddeddate['now']) - strtotime($maxaddeddate['dateadded']) > 60 * 60 * 48) {
$db->exec(sprintf("update binaries set procstat=%d where relname = %s and procstat = %d and groupID = %d and fromname=%s", Releases::PROCSTAT_NOREQIDNAMELOOKUPFOUND, $db->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $db->escapeString($row["fromname"])));
}
}
} else {
$db->exec(sprintf("update binaries set procstat=%d where relname = %s and procstat = %d and groupID = %d and fromname=%s", Releases::PROCSTAT_READYTORELEASE, $db->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $db->escapeString($row["fromname"])));
}
}
}
if ($retcount % 100 == 0) {
echo ".";
}
}
$retcount = 0;
echo "\nStage 3 : Creating releases from complete binaries\n";
//
// Get out all distinct relname, group from binaries of STAGE2
//
$result = $db->queryDirect(sprintf("SELECT relname, groupID, g.name as group_name, fromname, max(categoryID) as categoryID, max(regexID) as regexID, max(reqID) as reqID, MAX(date) as date, count(binaries.ID) as parts, coalesce(sum(binaries.size),0) as size from binaries inner join groups g on g.ID = binaries.groupID where procstat = %d and relname is not null group by relname, g.name, groupID, fromname ORDER BY COUNT(binaries.ID) desc", Releases::PROCSTAT_READYTORELEASE));
while ($row = $db->getAssocArray($result)) {
$relguid = md5(uniqid());
//
// Get categoryID if one has been allocated to this
//
if ($row["categoryID"] != "") {
$catId = $row["categoryID"];
} else {
$catId = $cat->determineCategory($row["group_name"], $row["relname"]);
}
//
// Determine if size matches permitted boundaries and discard here if not.
//
$gsize = $groupbasedminsizes[$row["groupID"]][0]["minsizetoformrelease"];
if ($gsize == "" || $gsize == 0) {
$gsize = $row["size"];
}
$ssize = $page->site->minsizetoformrelease;
if ($ssize == "" || $ssize == 0) {
$ssize = $row["size"];
}
$csize = $catbasedsizes[$catId][0]["minsizetoformrelease"];
if ($csize == "" || $csize == 0) {
$csize = $row["size"];
}
$cpsize = $catbasedsizes[$catId][0]["parentminsizetoformrelease"];
if ($cpsize == "" || $cpsize == 0) {
$cpsize = $row["size"];
}
$cmaxsize = $catbasedsizes[$catId][0]["maxsizetoformrelease"];
if ($cmaxsize == "" || $cmaxsize == 0) {
$cmaxsize = $row["size"];
}
$cpmaxsize = $catbasedsizes[$catId][0]["parentmaxsizetoformrelease"];
if ($cpmaxsize == "" || $cpmaxsize == 0) {
$cpmaxsize = $row["size"];
}
$overallminsize = max($gsize, $ssize, $csize, $cpsize);
$overallmaxsize = min($cmaxsize, $cpmaxsize);
if ($row["size"] < $overallminsize || $row["size"] > $overallmaxsize) {
echo sprintf("Stage 3 : Discarding - %s (Size %s outside permitted range of %s%s)\n", $row["relname"], formatBytes($row["size"]), $overallminsize != $row["size"] ? formatBytes($overallminsize) : "", $overallmaxsize != $row["size"] ? formatBytes($overallmaxsize) : "");
$db->exec(sprintf("DELETE parts, binaries FROM parts JOIN binaries ON binaries.ID = parts.binaryID WHERE relname=%s and groupID=%d and fromname=%s and procstat=%d", $db->escapeString($row["relname"]), $row["groupID"], $db->escapeString($row["fromname"]), Releases::PROCSTAT_READYTORELEASE));
} else {
// Clean release name
$cleanRelName = $this->cleanReleaseName($row['relname']);
$relid = $this->insertRelease($cleanRelName, $row["parts"], $row["groupID"], $relguid, $catId, $row["regexID"], $row["date"], $row["fromname"], $row["reqID"], $page->site);
//
// Tag every binary for this release with its parent release id
//
$db->exec(sprintf("update binaries set procstat = %d, releaseID = %d where relname = %s and procstat = %d and groupID = %d and fromname=%s", Releases::PROCSTAT_RELEASED, $relid, $db->escapeString($row["relname"]), Releases::PROCSTAT_READYTORELEASE, $row["groupID"], $db->escapeString($row["fromname"])));
//
// Write the nzb to disk
//
$nzbfile = $nzb->getNZBPath($relguid, $page->site->nzbpath, true);
$nzb->writeNZBforReleaseId($relid, $cleanRelName, $catId, $nzbfile);
//
// Remove used binaries
//
$db->exec(sprintf("DELETE parts, binaries FROM parts JOIN binaries ON binaries.ID = parts.binaryID WHERE releaseID = %d ", $relid));
//
// If nzb successfully written, then load it and get size completion from it
//
$nzbInfo = new nzbInfo();
if (!$nzbInfo->loadFromFile($nzbfile)) {
echo "Stage 3 : Failed to write nzb file (bad perms?) " . $nzbfile . "\n";
//copy($nzbfile, "./ERRORNZB_".$relguid);
$this->delete($relid);
} else {
// Check if gid already exists
$dupes = $db->queryOneRow(sprintf("SELECT EXISTS(SELECT 1 FROM releases WHERE gid = %s) as total", $db->escapeString($nzbInfo->gid)));
if ($dupes['total'] > 0) {
echo "Stage 3 : Duplicate - " . $cleanRelName . " -" . $nzbInfo->gid . "-\n";
$this->delete($relid);
} else {
$db->exec(sprintf("update releases set totalpart = %d, size = %s, completion = %d, GID=%s where ID = %d", $nzbInfo->filecount, $nzbInfo->filesize, $nzbInfo->completion, $db->escapeString($nzbInfo->gid), $relid));
echo "Stage 3 : Added release " . $cleanRelName . "\n";
//Increment new release count
$retcount++;
}
}
}
}
//
// Delete any releases under the minimum completion percent.
//
if ($page->site->completionpercent != 0) {
echo "Stage 4 : Deleting releases less than " . $page->site->completionpercent . " complete\n";
$result = $db->query(sprintf("select ID from releases where completion > 0 and completion < %d", $page->site->completionpercent));
foreach ($result as $row) {
$this->delete($row["ID"]);
}
}
/*
*Potentially leave this in to mop up release when the cat sizes change.
//
// Delete releases whos minsize is less than the site or group minimum
//
$result = $db->query("select releases.ID from releases left outer join (SELECT g.ID, coalesce(g.minsizetoformrelease, s.minsizetoformrelease) as minsizetoformrelease FROM groups g inner join ( select value as minsizetoformrelease from site where setting = 'minsizetoformrelease' ) s ) x on x.ID = releases.groupID where minsizetoformrelease != 0 and releases.size < minsizetoformrelease");
if (count($result) > 0)
{
echo "Stage 4 : Deleting ".count($result)." release(s) where size is smaller than minsize for site/group\n";
foreach ($result as $row)
$this->delete($row["ID"]);
}
$result = $db->query("select releases.ID, name, categoryID, size FROM releases JOIN (
select
catc.ID,
case when catc.minsizetoformrelease = 0 then catp.minsizetoformrelease else catc.minsizetoformrelease end as minsizetoformrelease,
case when catc.maxsizetoformrelease = 0 then catp.maxsizetoformrelease else catc.maxsizetoformrelease end as maxsizetoformrelease
from category catp join category catc on catc.parentID = catp.ID
where (catc.minsizetoformrelease != 0 or catc.maxsizetoformrelease != 0) or (catp.minsizetoformrelease != 0 or catp.maxsizetoformrelease != 0)
) x on x.ID = releases.categoryID
where
(size < minsizetoformrelease and minsizetoformrelease != 0) or
(size > maxsizetoformrelease and maxsizetoformrelease != 0)");
if(count($result) > 0)
{
echo "Stage 4 : Deleting release(s) not matching category min/max size ...\n";
foreach ($result as $r){
$this->delete($r['ID']);
}
}
*/
echo "Stage 5 : Post processing started\n";
$postprocess = new PostProcess(true);
$postprocess->processAll();
//
// aggregate the releasefiles upto the releases.
//
echo "Stage 6 : Aggregating Files\n";
$db->exec("update releases INNER JOIN (SELECT releaseID, COUNT(ID) AS num FROM releasefiles GROUP BY releaseID) b ON b.releaseID = releases.ID and releases.rarinnerfilecount = 0 SET rarinnerfilecount = b.num");
// Remove the binaries and parts used to form releases, or that are duplicates.
//
if ($page->site->partsdeletechunks > 0) {
echo "Stage 7 : Chunk deleting unused binaries and parts";
$query = sprintf("SELECT parts.ID as partsID,binaries.ID as binariesID FROM parts\r\n\t\t\t\t\t\tLEFT JOIN binaries ON binaries.ID = parts.binaryID\r\n\t\t\t\t\t\tWHERE binaries.dateadded < %s - INTERVAL %d HOUR LIMIT 0,%d", $db->escapeString($currTime_ori["now"]), ceil($page->site->rawretentiondays * 24), $page->site->partsdeletechunks);
$cc = 0;
$done = false;
while (!$done) {
$dd = $cc;
$result = $db->query($query);
if (count($result) > 0) {
$pID = array();
$bID = array();
foreach ($result as $row) {
$pID[] = $row['partsID'];
$bID[] = $row['binariesID'];
}
$pID = '(' . implode(',', $pID) . ')';
$bID = '(' . implode(',', $bID) . ')';
$fr = $db->exec("DELETE FROM parts WHERE ID IN {$pID}");
if ($fr > 0) {
$cc += $fr;
$cc += $db->exec("DELETE FROM binaries WHERE ID IN {$bID}");
}
unset($pID);
unset($bID);
if ($cc == $dd) {
$done = true;
}
echo $cc % 10000 ? '.' : '';
} else {
$done = true;
}
}
echo "\nStage 7 : Complete - " . $cc . " rows affected\n";
} else {
echo "Stage 7 : Deleting unused binaries and parts\n";
$db->exec(sprintf("DELETE parts, binaries FROM parts JOIN binaries ON binaries.ID = parts.binaryID\r\n\t\t\tWHERE binaries.dateadded < %s - INTERVAL %d HOUR", $db->escapeString($currTime_ori["now"]), ceil($page->site->rawretentiondays * 24)));
}
//
// User/Request housekeeping, should ideally move this to its own section, but it needs to be done automatically.
//
$users = new Users();
$users->pruneRequestHistory($page->site->userdownloadpurgedays);
echo "Done : Added " . $retcount . " releases\n\n";
return $retcount;
}
<?php
require_once "config.php";
require_once WWW_DIR . "/lib/adminpage.php";
require_once WWW_DIR . "/lib/releaseregex.php";
// login check
$admin = new AdminPage();
$regex = new ReleaseRegex();
if (isset($_GET['action']) && $_GET['action'] == "2") {
$id = (int) $_GET['regex_id'];
$regex->delete($id);
print "Regex {$id} deleted.";
}
<?php
require_once 'config.php';
$page = new AdminPage();
$page->title = "Submit your regex expressions to newznab";
$regex = new ReleaseRegex();
$regexList = $regex->get(false, -1, true, true);
if (count($regexList)) {
$regexSerialize = serialize($regexList);
$regexFilename = 'releaseregex-' . time() . '.regex';
// User wants to submit their regex's
if (isset($_POST['regex_submit_please'])) {
// Submit
$ch = curl_init();
curl_setopt($ch, CURLOPT_HEADER, 0);
curl_setopt($ch, CURLOPT_VERBOSE, 0);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/4.0 (newznab / compatible;)");
curl_setopt($ch, CURLOPT_URL, "http://newznab.com/regex/uploadregex.php");
curl_setopt($ch, CURLOPT_POST, true);
$post = array("regex" => $regexSerialize);
curl_setopt($ch, CURLOPT_POSTFIELDS, $post);
$response = curl_exec($ch);
curl_close($ch);
if ($response == 'OK') {
$page->smarty->assign('upload_status', 'OK');
} else {
$page->smarty->assign('upload_status', 'BAD');
}
}
} else {
<?php
require_once dirname(__FILE__) . '/../../www/config.php';
use newznab\db\Settings;
use newznab\utility\Utility;
$releases = new Releases();
$db = new Settings();
$cat = new Categorize();
$releaseRegex = new ReleaseRegex();
$nzb = new NZB();
$page = new Page();
$num = 0;
$usefilename = true;
$dupecheck = true;
$movefiles = true;
$categoryoverride = -1;
if (empty($argc) || $argc <= 1) {
$path = "./";
} else {
$util = new Utility();
$path = !$util->endsWith($argv[1], "/") ? $argv[1] . "/" : $argv[1];
if (isset($argv[2])) {
$usefilename = strtolower($argv[2]) == 'true';
}
if (isset($argv[3])) {
$dupecheck = strtolower($argv[3]) == 'true';
}
if (isset($argv[4])) {
$movefiles = strtolower($argv[4]) == 'true';
}
if (isset($argv[5]) && is_numeric($argv[5])) {
/**
* Download a range of usenet messages. Store binaries with subjects matching a
* specific pattern in the database.
*/
function scan($nntp, $groupArr, $first, $last, $type = 'update')
{
$db = new Db();
$releaseRegex = new ReleaseRegex();
$n = $this->n;
$this->startHeaders = microtime(true);
if ($this->compressedHeaders) {
$nntpn = new Nntp();
$nntpn->doConnect(5, false, true);
$response = $nntpn->_sendCommand('XFEATURE COMPRESS GZIP');
if ($nntpn->isError($response) || $response != 290) {
$response2 = $nntpn->_sendCommand('XZVER');
if ($nntpn->isError($response2) || $response2 != 412) {
$msgs = $nntp->getOverview($first . "-" . $last, true, false);
$nntpn->doQuit();
} else {
$msgs = $nntp->getXOverview($first . "-" . $last, true, false);
$nntpn->doQuit();
}
} else {
$msgs = $nntp->getOverview($first . "-" . $last, true, false);
$nntpn->doQuit();
}
} else {
$msgs = $nntp->getOverview($first . "-" . $last, true, false);
}
if ($nntp->isError($msgs) && ($msgs->code == 400 || $msgs->code == 503)) {
echo "NNTP connection timed out. Reconnecting...{$n}";
if (!$nntp->doConnect()) {
// TODO: What now?
echo "Failed to get NNTP connection.{$n}";
return;
}
$nntp->selectGroup($groupArr['name']);
if ($this->compressedHeaders) {
$nntpn = new Nntp();
$nntpn->doConnect(5, false, true);
$response = $nntpn->_sendCommand('XFEATURE COMPRESS GZIP');
if ($nntpn->isError($response) || $response != 290) {
$response2 = $nntpn->_sendCommand('XZVER');
if ($nntpn->isError($response2) || $response2 != 412) {
$msgs = $nntp->getOverview($first . "-" . $last, true, false);
$nntpn->doQuit();
} else {
$msgs = $nntp->getXOverview($first . "-" . $last, true, false);
$nntpn->doQuit();
}
} else {
$msgs = $nntp->getOverview($first . "-" . $last, true, false);
$nntpn->doQuit();
}
} else {
$msgs = $nntp->getOverview($first . "-" . $last, true, false);
}
}
$rangerequested = range($first, $last);
$msgsreceived = array();
$msgsblacklisted = array();
$msgsignored = array();
$msgsinserted = array();
$msgsnotinserted = array();
$timeHeaders = number_format(microtime(true) - $this->startHeaders, 2);
if ($nntp->isError($msgs)) {
echo "Error {$msgs->code}: {$msgs->message}{$n}";
echo "Skipping group{$n}";
return false;
}
$this->startUpdate = microtime(true);
if (is_array($msgs)) {
//loop headers, figure out parts
foreach ($msgs as $msg) {
if (!isset($msg['Number'])) {
continue;
}
$msgsreceived[] = $msg['Number'];
$msgPart = $msgTotalParts = 0;
$pattern = '|\\((\\d+)[\\/](\\d+)\\)|i';
preg_match_all($pattern, $msg['Subject'], $matches, PREG_PATTERN_ORDER);
$matchcnt = sizeof($matches[0]);
for ($i = 0; $i < $matchcnt; $i++) {
$msgPart = $matches[1][$i];
$msgTotalParts = $matches[2][$i];
}
if (!isset($msg['Subject']) || $matchcnt == 0) {
$msgsignored[] = $msg['Number'];
continue;
}
if ((int) $msgPart > 0 && (int) $msgTotalParts > 0) {
$subject = utf8_encode(trim(preg_replace('|\\(' . $msgPart . '[\\/]' . $msgTotalParts . '\\)|i', '', $msg['Subject'])));
if (!isset($this->message[$subject])) {
$this->message[$subject] = $msg;
$this->message[$subject]['MaxParts'] = (int) $msgTotalParts;
$this->message[$subject]['Date'] = strtotime($this->message[$subject]['Date']);
}
if ((int) $msgPart > 0) {
$this->message[$subject]['Parts'][(int) $msgPart] = array('Message-ID' => substr($msg['Message-ID'], 1, -1), 'number' => $msg['Number'], 'part' => (int) $msgPart, 'size' => $msg['Bytes']);
$this->message[$subject]['PartNumbers'][(int) $msgPart] = $msg['Number'];
}
}
}
unset($msg);
unset($msgs);
$count = 0;
$updatecount = 0;
$partcount = 0;
$rangenotreceived = array_diff($rangerequested, $msgsreceived);
if ($type != 'partrepair') {
echo "Received " . sizeof($msgsreceived) . " articles of " . ($last - $first + 1) . " requested, " . sizeof($msgsignored) . " not binaries {$n}";
}
if ($type == 'update' && sizeof($msgsreceived) == 0) {
echo "Error: Server did not return any articles.{$n}";
echo "Skipping group{$n}";
return false;
}
if (sizeof($rangenotreceived) > 0) {
switch ($type) {
case 'backfill':
//don't add missing articles
break;
case 'partrepair':
case 'update':
default:
$this->addMissingParts($rangenotreceived, $groupArr['ID']);
break;
}
echo "Server did not return " . count($rangenotreceived) . " article(s).{$n}";
}
if (isset($this->message) && count($this->message)) {
$groupRegexes = $releaseRegex->getForGroup($groupArr['name']);
//insert binaries and parts into database. when binary already exists; only insert new parts
foreach ($this->message as $subject => $data) {
//Filter binaries based on black/white list
if ($this->isBlackListed($data, $groupArr['name'])) {
$msgsblacklisted[] = count($data['Parts']);
if ($type == 'partrepair') {
$partIds = array();
foreach ($data['Parts'] as $partdata) {
$partIds[] = $partdata['number'];
}
$db->exec(sprintf("DELETE FROM partrepair WHERE numberID IN (%s) AND groupID=%d", implode(',', $partIds), $groupArr['ID']));
}
continue;
}
if (isset($data['Parts']) && count($data['Parts']) > 0 && $subject != '') {
//Check for existing binary
$binaryID = 0;
$binaryHash = md5($subject . $data['From'] . $groupArr['ID']);
$res = $db->queryOneRow(sprintf("SELECT ID FROM binaries WHERE binaryhash = %s", $db->escapeString($binaryHash)));
if (!$res) {
//Apply Regexes
$regexMatches = array();
foreach ($groupRegexes as $groupRegex) {
$regexCheck = $releaseRegex->performMatch($groupRegex, $subject);
if ($regexCheck !== false) {
$regexMatches = $regexCheck;
break;
}
}
$sql = '';
if (!empty($regexMatches)) {
$relparts = explode("/", $regexMatches['parts']);
$sql = sprintf("INSERT INTO binaries (name, fromname, date, xref, totalparts, groupID, procstat, categoryID, regexID, reqID, relpart, reltotalpart, binaryhash, relname, dateadded) VALUES (%s, %s, FROM_UNIXTIME(%s), %s, %s, %d, %d, %s, %d, %s, %d, %d, %s, %s, now())", $db->escapeString($subject), $db->escapeString(utf8_encode($data['From'])), $db->escapeString($data['Date']), $db->escapeString($data['Xref']), $db->escapeString($data['MaxParts']), $groupArr['ID'], Releases::PROCSTAT_TITLEMATCHED, $regexMatches['regcatid'], $regexMatches['regexID'], $db->escapeString($regexMatches['reqID']), $relparts[0], $relparts[1], $db->escapeString($binaryHash), $db->escapeString(str_replace('_', ' ', $regexMatches['name'])));
} elseif ($this->onlyProcessRegexBinaries === false) {
$sql = sprintf("INSERT INTO binaries (name, fromname, date, xref, totalparts, groupID, binaryhash, dateadded) VALUES (%s, %s, FROM_UNIXTIME(%s), %s, %s, %d, %s, now())", $db->escapeString($subject), $db->escapeString(utf8_encode($data['From'])), $db->escapeString($data['Date']), $db->escapeString($data['Xref']), $db->escapeString($data['MaxParts']), $groupArr['ID'], $db->escapeString($binaryHash));
} elseif ($type == 'partrepair') {
$partIds = array();
foreach ($data['Parts'] as $partdata) {
$partIds[] = $partdata['number'];
}
$db->exec(sprintf("DELETE FROM partrepair WHERE numberID IN (%s) AND groupID=%d", implode(',', $partIds), $groupArr['ID']));
continue;
}
if ($sql != '') {
$binaryID = $db->queryInsert($sql);
$count++;
if ($count % 500 == 0) {
echo "{$count} bin adds...";
}
}
} else {
$binaryID = $res["ID"];
$updatecount++;
if ($updatecount % 500 == 0) {
echo "{$updatecount} bin updates...";
}
}
if ($binaryID != 0) {
$partParams = array();
$partNumbers = array();
$totsize = 0;
foreach ($data['Parts'] as $partdata) {
$partcount++;
$totsize += $partdata['size'];
$partParams[] = sprintf("(%d, %s, %s, %s, %s)", $binaryID, $db->escapeString($partdata['Message-ID']), $db->escapeString($partdata['number']), $db->escapeString(round($partdata['part'])), $db->escapeString($partdata['size']));
$partNumbers[] = $partdata['number'];
}
$partSql = "INSERT INTO parts (binaryID, messageID, number, partnumber, size) VALUES " . implode(', ', $partParams);
$pidata = $db->queryInsert($partSql, false);
if (!$pidata) {
$msgsnotinserted = array_merge($msgsnotinserted, $partNumbers);
} else {
$msgsinserted = array_merge($msgsinserted, $partNumbers);
}
// update bin size
$upsql = sprintf("update binaries set size = size + %d where ID = %d", $totsize, $binaryID);
$db->exec($upsql);
}
}
}
//TODO: determine whether to add to missing articles if insert failed
if (sizeof($msgsnotinserted) > 0) {
echo 'WARNING: ' . count($msgsnotinserted) . ' Parts failed to insert' . $n;
$this->addMissingParts($msgsnotinserted, $groupArr['ID']);
}
if ($count >= 500 || $updatecount >= 500) {
echo $n;
}
//line break for bin adds output
}
$timeUpdate = number_format(microtime(true) - $this->startUpdate, 2);
$timeLoop = number_format(microtime(true) - $this->startLoop, 2);
if (sizeof($msgsblacklisted) > 0) {
echo "Blacklisted " . array_sum($msgsblacklisted) . " parts in " . sizeof($msgsblacklisted) . " binaries" . $n;
}
if ($type != 'partrepair') {
echo number_format($count) . ' new, ' . number_format($updatecount) . ' updated, ' . number_format($partcount) . ' parts.';
echo " {$timeHeaders} headers, {$timeUpdate} update, {$timeLoop} range.{$n}";
}
unset($this->message);
unset($data);
return $last;
} else {
echo "Error: Can't get parts from server (msgs not array) {$n}";
echo "Skipping group{$n}";
return false;
}
}
function processReleases()
{
$db = new DB();
$cat = new Category();
$bin = new Binaries();
$nzb = new Nzb();
$s = new Sites();
$relreg = new ReleaseRegex();
$page = new Page();
$nfo = new Nfo(true);
$retcount = 0;
echo $s->getLicense();
echo "\n\nStarting release update process (" . date("Y-m-d H:i:s") . ")\n";
if (!file_exists($page->site->nzbpath)) {
echo "Bad or missing nzb directory - " . $page->site->nzbpath;
return;
}
$this->checkRegexesUptoDate($page->site->latestregexurl, $page->site->latestregexrevision, true);
//
// Get all regexes for all groups which are to be applied to new binaries
// in order of how they should be applied
//
$regexrows = $relreg->get();
foreach ($regexrows as $regexrow) {
echo "Applying regex " . $regexrow["ID"] . " for group " . ($regexrow["groupname"] == "" ? "all" : $regexrow["groupname"]) . "\n";
$groupmatch = "";
//
// Groups ending in * need to be like matched when getting out binaries for groups and children
//
if (preg_match("/\\*\$/i", $regexrow["groupname"])) {
$groupname = substr($regexrow["groupname"], 0, -1);
$resgrps = $db->query(sprintf("select ID from groups where name like %s ", $db->escapeString($groupname . "%")));
foreach ($resgrps as $resgrp) {
$groupmatch .= " groupID = " . $resgrp["ID"] . " or ";
}
$groupmatch .= " 1=2 ";
} elseif ($regexrow["groupname"] != "") {
$resgrp = $db->queryOneRow(sprintf("select ID from groups where name = %s ", $db->escapeString($regexrow["groupname"])));
//
// if group not found, its a regex for a group we arent indexing.
//
if ($resgrp) {
$groupmatch = " groupID = " . $resgrp["ID"];
} else {
$groupmatch = " 1=2 ";
}
} else {
$groupmatch = " 1=1 ";
}
// Get current mysql time for date comparison checks in case php is in a different time zone
$currTime = $db->queryOneRow("SELECT NOW() as now");
// Get out all binaries of STAGE0 for current group
$arrNoPartBinaries = array();
$resbin = $db->queryDirect(sprintf("SELECT binaries.ID, binaries.name, binaries.date, binaries.totalParts from binaries where (%s) and procstat = %d order by binaries.date asc", $groupmatch, Releases::PROCSTAT_NEW));
while ($rowbin = mysql_fetch_assoc($resbin)) {
if (preg_match($regexrow["regex"], $rowbin["name"], $matches)) {
$matches = array_map("trim", $matches);
if (isset($matches['reqid']) && ctype_digit($matches['reqid']) && (!isset($matches['name']) || empty($matches['name']))) {
$matches['name'] = $matches['reqid'];
}
// Check that the regex provided the correct parameters
if (!isset($matches['name']) || empty($matches['name'])) {
echo "regex applied which didnt return right number of capture groups - " . $regexrow["regex"] . "\n";
print_r($matches);
continue;
}
// If theres no number of files data in the subject, put it into a release if it was posted to usenet longer than five hours ago.
if (!isset($matches['parts']) && strtotime($currTime['now']) - strtotime($rowbin['date']) > 18000 || isset($arrNoPartBinaries[$matches['name']])) {
//
// Take a copy of the name of this no-part release found. This can be used
// next time round the loop to find parts of this set, but which have not yet reached 3 hours.
//
$arrNoPartBinaries[$matches['name']] = "1";
$matches['parts'] = "01/01";
}
if (isset($matches['name']) && isset($matches['parts'])) {
if (strpos($matches['parts'], '/') === false) {
$matches['parts'] = str_replace(array('-', '~', ' of '), '/', $matches['parts']);
}
$regcatid = "null ";
if ($regexrow["categoryID"] != "") {
$regcatid = $regexrow["categoryID"];
}
$reqid = " null ";
if (isset($matches['reqid'])) {
$reqid = $matches['reqid'];
}
//check if post is repost
if (preg_match('/(repost\\d?|re\\-?up)/i', $rowbin['name'], $repost) && !preg_match('/repost|re\\-?up/i', $matches['name'])) {
$matches['name'] .= ' ' . $repost[1];
}
$relparts = explode("/", $matches['parts']);
$db->query(sprintf("update binaries set relname = replace(%s, '_', ' '), relpart = %d, reltotalpart = %d, procstat=%d, categoryID=%s, regexID=%d, reqID=%s where ID = %d", $db->escapeString($matches['name']), $relparts[0], $relparts[1], Releases::PROCSTAT_TITLEMATCHED, $regcatid, $regexrow["ID"], $reqid, $rowbin["ID"]));
}
}
}
}
//
// Move all binaries from releases which have the correct number of files on to the next stage.
//
echo "Stage 2\n";
$result = $db->queryDirect(sprintf("SELECT relname, SUM(reltotalpart) AS reltotalpart, groupID, reqID, fromname, SUM(num) AS num, coalesce(g.minfilestoformrelease, s.minfilestoformrelease) as minfilestoformrelease FROM ( SELECT relname, reltotalpart, groupID, reqID, fromname, COUNT(ID) AS num FROM binaries WHERE procstat = %s GROUP BY relname, reltotalpart, groupID, reqID, fromname ) x left outer join groups g on g.ID = x.groupID inner join ( select value as minfilestoformrelease from site where setting = 'minfilestoformrelease' ) s GROUP BY relname, groupID, reqID, fromname", Releases::PROCSTAT_TITLEMATCHED));
while ($row = mysql_fetch_assoc($result)) {
$retcount++;
//
// Less than the site permitted number of files in a release. Dont discard it, as it may
// be part of a set being uploaded.
//
if ($row["num"] < $row["minfilestoformrelease"]) {
echo "Number of files in release " . $row["relname"] . " less than site/group setting (" . $row['num'] . "/" . $row["minfilestoformrelease"] . ")\n";
$db->query(sprintf("update binaries set procattempts = procattempts + 1 where relname = %s and procstat = %d and groupID = %d and fromname = %s", $db->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $db->escapeString($row["fromname"])));
} elseif ($row["num"] >= $row["reltotalpart"]) {
// Check that the binary is complete
$binlist = $db->query(sprintf("SELECT ID, totalParts, date from binaries where relname = %s and procstat = %d and groupID = %d and fromname = %s", $db->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $db->escapeString($row["fromname"])));
$incomplete = false;
foreach ($binlist as $rowbin) {
$binParts = $db->queryOneRow(sprintf("SELECT COUNT(ID) AS num FROM parts WHERE binaryID = %d", $rowbin['ID']));
if ($binParts['num'] < $rowbin['totalParts']) {
echo "binary " . $rowbin['ID'] . " from " . $row['relname'] . " has missing parts - " . $binParts['num'] . "/" . $rowbin['totalParts'] . " (" . number_format($binParts['num'] / $rowbin['totalParts'] * 100, 1) . "% complete)\n";
// Allow to binary to release if posted to usenet longer than four hours ago and we still don't have all the parts
if (strtotime($currTime['now']) - strtotime($rowbin['date']) > 14400) {
echo "allowing incomplete binary " . $rowbin['ID'] . "\n";
} else {
$incomplete = true;
}
}
}
if ($incomplete) {
echo "Incorrect number of parts " . $row["relname"] . "-" . $row["num"] . "-" . $row["reltotalpart"] . "\n";
$db->query(sprintf("update binaries set procattempts = procattempts + 1 where relname = %s and procstat = %d and groupID = %d and fromname = %s", $db->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $db->escapeString($row["fromname"])));
} elseif ($row['reqID'] != '' && $page->site->reqidurl != "") {
//
// Try and get the name using the group
//
$binGroup = $db->queryOneRow(sprintf("SELECT name FROM groups WHERE ID = %d", $row["groupID"]));
echo "Looking up " . $row['reqID'] . " in " . $binGroup['name'] . "... ";
$newtitle = $this->getReleaseNameForReqId($page->site->reqidurl, $binGroup["name"], $row["reqID"], true);
//
// if the feed/group wasnt supported by the scraper, then just use the release name as the title.
//
if ($newtitle == "no feed") {
$newtitle = $row["relname"];
echo "Group not supported\n";
}
//
// Valid release with right number of files and title now, so move it on
//
if ($newtitle != "") {
$db->query(sprintf("update binaries set relname = %s, procstat=%d where relname = %s and procstat = %d and groupID = %d and fromname=%s", $db->escapeString($newtitle), Releases::PROCSTAT_READYTORELEASE, $db->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $db->escapeString($row["fromname"])));
} else {
//
// Item not found, if the binary was added to the index yages ago, then give up.
//
$maxaddeddate = $db->queryOneRow(sprintf("SELECT NOW() as now, MAX(dateadded) as dateadded FROM binaries WHERE relname = %s and procstat = %d and groupID = %d and fromname=%s", $db->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $db->escapeString($row["fromname"])));
//
// If added to the index over 48 hours ago, give up trying to determine the title
//
if (strtotime($maxaddeddate['now']) - strtotime($maxaddeddate['dateadded']) > 60 * 60 * 48) {
$db->query(sprintf("update binaries set procstat=%d where relname = %s and procstat = %d and groupID = %d and fromname=%s", Releases::PROCSTAT_NOREQIDNAMELOOKUPFOUND, $db->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $db->escapeString($row["fromname"])));
echo "Not found in 48 hours\n";
}
}
} else {
$db->query(sprintf("update binaries set procstat=%d where relname = %s and procstat = %d and groupID = %d and fromname=%s", Releases::PROCSTAT_READYTORELEASE, $db->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $db->escapeString($row["fromname"])));
}
} else {
echo "Incorrect number of files for " . $row["relname"] . " (" . $row["num"] . "/" . $row["reltotalpart"] . ")\n";
$db->query(sprintf("update binaries set procattempts = procattempts + 1 where relname = %s and procstat = %d and groupID = %d and fromname=%s", $db->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $db->escapeString($row["fromname"])));
}
if ($retcount % 10 == 0) {
echo "-processed " . $retcount . " binaries stage two\n";
}
}
$retcount = $nfocount = 0;
echo "Stage 3\n";
//
// Get out all distinct relname, group from binaries of STAGE2
//
$result = $db->queryDirect(sprintf("SELECT relname, groupID, g.name as group_name, fromname, count(binaries.ID) as parts from binaries inner join groups g on g.ID = binaries.groupID where procstat = %d and relname is not null group by relname, g.name, groupID, fromname ORDER BY COUNT(binaries.ID) desc", Releases::PROCSTAT_READYTORELEASE));
while ($row = mysql_fetch_assoc($result)) {
$retcount++;
//
// Get the last post date and the poster name from the binary
//
$bindata = $db->queryOneRow(sprintf("select fromname, MAX(date) as date from binaries where relname = %s and procstat = %d and groupID = %d and fromname = %s group by fromname", $db->escapeString($row["relname"]), Releases::PROCSTAT_READYTORELEASE, $row["groupID"], $db->escapeString($row["fromname"])));
//
// Get all releases with the same name with a usenet posted date in a +1-1 date range.
//
$relDupes = $db->query(sprintf("select ID from releases where searchname = %s and (%s - INTERVAL 1 DAY < postdate AND %s + INTERVAL 1 DAY > postdate)", $db->escapeString($row["relname"]), $db->escapeString($bindata["date"]), $db->escapeString($bindata["date"])));
if (count($relDupes) > 0) {
$db->query(sprintf("update binaries set procstat = %d where relname = %s and procstat = %d and groupID = %d and fromname=%s ", Releases::PROCSTAT_DUPLICATE, $db->escapeString($row["relname"]), Releases::PROCSTAT_READYTORELEASE, $row["groupID"], $db->escapeString($row["fromname"])));
continue;
}
//
// Get total size of this release
// Done in a big OR statement, not an IN as the mysql binaryID index on parts table
// was not being used.
//
$totalSize = "0";
$regexAppliedCategoryID = "";
$regexIDused = "";
$reqIDused = "";
$relTotalParts = 0;
$relCompletion = 0;
$binariesForSize = $db->query(sprintf("select ID, categoryID, regexID, reqID, totalParts from binaries use index (ix_binary_relname) where relname = %s and procstat = %d and groupID = %d and fromname=%s", $db->escapeString($row["relname"]), Releases::PROCSTAT_READYTORELEASE, $row["groupID"], $db->escapeString($row["fromname"])));
if (count($binariesForSize) > 0) {
$sizeSql = "select sum(size) as totalSize, count(ID) as relParts from parts where (";
foreach ($binariesForSize as $binSizeId) {
$sizeSql .= " binaryID = " . $binSizeId["ID"] . " or ";
//
// Get categoryID if one has been allocated to this
//
if ($binSizeId["categoryID"] != "" && $regexAppliedCategoryID == "") {
$regexAppliedCategoryID = $binSizeId["categoryID"];
}
//
// Get RegexID if one has been allocated to this
//
if ($binSizeId["regexID"] != "" && $regexIDused == "") {
$regexIDused = $binSizeId["regexID"];
}
//
// Get requestID if one has been allocated to this
//
if ($binSizeId["reqID"] != "" && $reqIDused == "") {
$reqIDused = $binSizeId["reqID"];
}
//
// Get number of expected parts
//
$relTotalParts += $binSizeId["totalParts"];
}
$sizeSql .= " 1=2) ";
$temp = $db->queryOneRow($sizeSql);
$totalSize = $temp["totalSize"] + 0 . "";
$relCompletion = number_format($temp["relParts"] / $relTotalParts * 100, 1, '.', '');
}
//
// Insert the release
//
$relguid = md5(uniqid());
if ($regexAppliedCategoryID == "") {
$catId = $cat->determineCategory($row["group_name"], $row["relname"]);
} else {
$catId = $regexAppliedCategoryID;
}
if ($regexIDused == "") {
$regexID = " null ";
} else {
$regexID = $regexIDused;
}
if ($reqIDused == "") {
$reqID = " null ";
} else {
$reqID = $reqIDused;
}
//Clean release name
$cleanArr = array('#', '@', '$', '%', '^', '§', '¨', '©', 'Ö');
$cleanRelName = str_replace($cleanArr, '', $row['relname']);
$relid = $db->queryInsert(sprintf("insert into releases (name, searchname, totalpart, groupID, adddate, guid, categoryID, regexID, rageID, postdate, fromname, size, reqID, passwordstatus, completion) values (%s, %s, %d, %d, now(), %s, %d, %d, -1, %s, %s, %s, %s, %d, %f)", $db->escapeString($cleanRelName), $db->escapeString($cleanRelName), $row["parts"], $row["groupID"], $db->escapeString($relguid), $catId, $regexID, $db->escapeString($bindata["date"]), $db->escapeString($bindata["fromname"]), $totalSize, $reqID, $page->site->checkpasswordedrar == "1" ? -1 : 0, $relCompletion > 100 ? 100 : $relCompletion));
echo "Added release " . $cleanRelName . "\n";
//
// Tag every binary for this release with its parent release id
// remove the release name from the binary as its no longer required
//
$db->query(sprintf("update binaries set procstat = %d, releaseID = %d where relname = %s and procstat = %d and groupID = %d and fromname=%s", Releases::PROCSTAT_RELEASED, $relid, $db->escapeString($row["relname"]), Releases::PROCSTAT_READYTORELEASE, $row["groupID"], $db->escapeString($row["fromname"])));
//
// Find an .nfo in the release
//
$relnfo = $nfo->determineReleaseNfo($relid);
if ($relnfo !== false) {
$nfo->addReleaseNfo($relid, $relnfo['ID']);
$nfocount++;
}
//
// Write the nzb to disk
//
$nzb->writeNZBforReleaseId($relid, $relguid, $cleanRelName, $catId, $nzb->getNZBPath($relguid, $page->site->nzbpath, true));
if ($retcount % 5 == 0) {
echo "-processed " . $retcount . " releases stage three\n";
}
}
echo "Found " . $nfocount . " nfos in " . $retcount . " releases\n";
//
// Process nfo files
//
if ($page->site->lookupnfo != "1") {
echo "Site config (site.lookupnfo) prevented retrieving nfos\n";
} else {
$nfo->processNfoFiles($page->site->lookupimdb, $page->site->lookuptvrage == "1");
}
//
// Lookup imdb if enabled
//
if ($page->site->lookupimdb == 1) {
$movie = new Movie(true);
$movie->processMovieReleases();
}
//
// Lookup music if enabled
//
if ($page->site->lookupmusic == 1) {
$music = new Music(true);
$music->processMusicReleases();
}
//
// Lookup games if enabled
//
if ($page->site->lookupgames == 1) {
$console = new Console(true);
$console->processConsoleReleases();
}
//
// Check for passworded releases
//
if ($page->site->checkpasswordedrar != "1") {
echo "Site config (site.checkpasswordedrar) prevented checking releases are passworded\n";
} else {
$this->processPasswordedReleases(true);
}
//
// Process all TV related releases which will assign their series/episode/rage data
//
$tvrage = new TVRage(true);
$tvrage->processTvReleases($page->site->lookuptvrage == "1");
//
// Get the current datetime again, as using now() in the housekeeping queries prevents the index being used.
//
$currTime = $db->queryOneRow("SELECT NOW() as now");
//
// Tidy away any binaries which have been attempted to be grouped into
// a release more than x times
//
echo "Tidying away binaries which cant be grouped after " . $page->site->attemptgroupbindays . " days\n";
$db->query(sprintf("update binaries set procstat = %d where procstat = %d and dateadded < %s - interval %d day ", Releases::PROCSTAT_WRONGPARTS, Releases::PROCSTAT_NEW, $db->escapeString($currTime["now"]), $page->site->attemptgroupbindays));
//
// Delete any parts and binaries which are older than the site's retention days
//
echo "Deleting parts which are older than " . $page->site->rawretentiondays . " days\n";
$db->query(sprintf("delete from parts where dateadded < %s - interval %d day", $db->escapeString($currTime["now"]), $page->site->rawretentiondays));
echo "Deleting binaries which are older than " . $page->site->rawretentiondays . " days\n";
$db->query(sprintf("delete from binaries where dateadded < %s - interval %d day", $db->escapeString($currTime["now"]), $page->site->rawretentiondays));
//
// Delete any releases which are older than site's release retention days
//
if ($page->site->releaseretentiondays != 0) {
echo "Determining any releases past retention to be deleted.\n\n";
$result = $db->query(sprintf("select ID from releases where postdate < %s - interval %d day", $db->escapeString($currTime["now"]), $page->site->releaseretentiondays));
foreach ($result as $row) {
$this->delete($row["ID"]);
}
}
echo "Processed " . $retcount . " releases\n\n";
return $retcount;
}
<?php
require_once "config.php";
require_once WWW_DIR . "/lib/adminpage.php";
require_once WWW_DIR . "/lib/releaseregex.php";
require_once WWW_DIR . "/lib/groups.php";
require_once WWW_DIR . "/lib/category.php";
require_once WWW_DIR . "/lib/nntp.php";
$page = new AdminPage();
$reg = new ReleaseRegex();
$groups = new Groups();
$cat = new Category();
$id = 0;
// set the current action
$action = isset($_REQUEST['action']) ? $_REQUEST['action'] : 'view';
$numarticlesdefault = 20000;
$groupList = $groups->getAll();
array_unshift($groupList, array('ID' => 0, 'name' => 'All Groups'));
$gid = $gnames = array();
$groupname = isset($_REQUEST['groupname']) && !empty($_REQUEST['groupname']) ? $_REQUEST['groupname'] : '';
$groupID = isset($_REQUEST['groupID']) ? $_REQUEST['groupID'] : '0';
$regex = isset($_REQUEST['regex']) && !empty($_REQUEST['regex']) ? $_REQUEST['regex'] : '/^(?P<name>.*)$/i';
$poster = isset($_REQUEST['poster']) && !empty($_REQUEST['poster']) ? $_REQUEST['poster'] : '';
$unreleased = isset($_REQUEST['unreleased']) ? $_REQUEST['unreleased'] : '';
$matchagainstbins = isset($_REQUEST['matchagainstbins']) ? $_REQUEST['matchagainstbins'] : '';
$numarticles = isset($_REQUEST['numarticles']) && !empty($_REQUEST['numarticles']) ? $_REQUEST['numarticles'] : $numarticlesdefault;
$clearexistingbins = isset($_REQUEST['clearexistingbins']) ? true : false;
foreach ($groupList as $grp) {
$gid[$grp["ID"]] = $grp["ID"];
$gnames[$grp["ID"]] = $grp["name"];
}
