/**
* Writes out the nzb when processing releases. Performed outside of smarty due to memory issues
* of holding all parts in an array.
*/
function writeNZBforReleaseId($relid, $name, $catId, $path)
{
$db = new DB();
$cat = new Category();
$catrow = $cat->getById($catId);
$site = new Sites();
$fp = _gzopen($path, "w");
if ($fp) {
gzwrite($fp, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
gzwrite($fp, "<!DOCTYPE nzb PUBLIC \"-//newzBin//DTD NZB 1.1//EN\" \"http://www.newzbin.com/DTD/nzb/nzb-1.1.dtd\">\n");
gzwrite($fp, "<nzb xmlns=\"http://www.newzbin.com/DTD/2003/nzb\">\n\n");
gzwrite($fp, "<head>\n");
if ($catrow) {
gzwrite($fp, " <meta type=\"category\">" . htmlspecialchars($catrow["title"], ENT_QUOTES, 'utf-8') . "</meta>\n");
}
if ($name != "") {
gzwrite($fp, " <meta type=\"name\">" . htmlspecialchars($name, ENT_QUOTES, 'utf-8') . "</meta>\n");
}
gzwrite($fp, "</head>\n\n");
$result = $db->queryDirect(sprintf("SELECT binaries.*, UNIX_TIMESTAMP(date) AS unixdate, groups.name as groupname FROM binaries inner join groups on binaries.groupID = groups.ID WHERE binaries.releaseID = %d ORDER BY binaries.name", $relid));
while ($binrow = $db->getAssocArray($result)) {
$groups = array();
$groupsRaw = explode(' ', $binrow['xref']);
foreach ($groupsRaw as $grp) {
if (preg_match('/^([a-z0-9\\.\\-_]+):(\\d+)?$/i', $grp, $match) && strtolower($grp) !== 'xref') {
$groups[] = $match[1];
}
}
if (count($groups) == 0) {
$groups[] = $binrow["groupname"];
}
gzwrite($fp, "<file poster=\"" . htmlspecialchars($binrow["fromname"], ENT_QUOTES, 'utf-8') . "\" date=\"" . $binrow["unixdate"] . "\" subject=\"" . htmlspecialchars($binrow["name"], ENT_QUOTES, 'utf-8') . " (1/" . $binrow["totalParts"] . ")\">\n");
gzwrite($fp, " <groups>\n");
foreach ($groups as $group) {
gzwrite($fp, " <group>" . $group . "</group>\n");
}
gzwrite($fp, " </groups>\n");
gzwrite($fp, " <segments>\n");
$resparts = $db->queryDirect(sprintf("SELECT DISTINCT(messageID), size, partnumber FROM parts WHERE binaryID = %d ORDER BY partnumber", $binrow["ID"]));
while ($partsrow = $db->getAssocArray($resparts)) {
gzwrite($fp, " <segment bytes=\"" . $partsrow["size"] . "\" number=\"" . $partsrow["partnumber"] . "\">" . htmlspecialchars($partsrow["messageID"], ENT_QUOTES, 'utf-8') . "</segment>\n");
}
gzwrite($fp, " </segments>\n</file>\n");
}
gzwrite($fp, "<!-- generated by newznab " . $site->version() . " -->\n</nzb>");
gzclose($fp);
}
}
/**
* Process all untagged releases to see if musicinfo exists for them.
*/
public function processMusicReleases()
{
$ret = 0;
$db = new DB();
$numlookedup = 0;
$res = $db->queryDirect(sprintf("SELECT searchname, ID from releases where musicinfoID IS NULL and categoryID in ( select ID from category where parentID = %d ) ORDER BY postdate DESC LIMIT 1000", Category::CAT_PARENT_MUSIC));
if ($db->getNumRows($res) > 0) {
if ($this->echooutput) {
echo "MusicPr : Processing " . $db->getNumRows($res) . " audio releases\n";
}
while ($arr = $db->getAssocArray($res)) {
if ($numlookedup > Music::NUMTOPROCESSPERTIME) {
return;
}
$albumId = -2;
$album = $this->parseArtist($arr['searchname']);
if ($album !== false) {
if ($this->echooutput) {
echo 'MusicPr : Looking up: ' . $album["artist"] . ' - ' . $album["album"] . "\n";
}
//check for existing music entry
$albumCheck = $this->getMusicInfoByName($album["artist"], $album["album"]);
if ($albumCheck === false) {
//
// get from amazon
//
$numlookedup++;
$ret = $this->updateMusicInfo($album["artist"], $album["album"], $album['year']);
if ($ret !== false) {
$albumId = $ret;
}
} else {
$albumId = $albumCheck["ID"];
}
}
$db->exec(sprintf("update releases SET musicinfoID = %d WHERE ID = %d", $albumId, $arr["ID"]));
}
}
}
/**
* Query the indexer directly. Returns an array of the results, unless
* there was an error in which case ``false`` is returned. However, if
* Sphinx returns an "invalid query" error (1064), then an empty result
* array is returned. Note that an empty "result array" is not the same as
* an empty array and will instead look like::
*
* array({"_totalrows": 0})
*
* If ``$lookupQuery`` is an empty string, then the results returned will
* be the data from the index--this is not guaranteed to be the most recent
* data that is in the MySQL database. If you absolutely need the most
* recent data from MySQL, then ``$lookupQuery`` should be a valid SQL
* query that has contains "releases.ID IN (%s)".
*
* @param string $sphinxQuery The raw SphinxQL query.
* @param string $lookupQuery The SQL to use to lookup the results.
* @param bool/int $useCache The ttl to store the item in the cache.
* @return array|false
*/
public function searchDirect($sphinxQuery, $lookupQuery = "", $useCache = false)
{
$cache = new Cache();
if ($useCache !== false && $cache->enabled && $cache->exists($sphinxQuery)) {
$ret = $cache->fetch($sphinxQuery);
if ($ret !== false) {
return $ret;
}
}
// Connect to Sphinx
$hostport = explode(":", $this->site->sphinxserverhost);
$sdb = mysqli_connect($hostport[0], "root", "", "", $hostport[1]);
if (!$sdb) {
// Couldn't connect to Sphinx.
return false;
}
// Get the results from Sphinx.
$lev = error_reporting();
error_reporting(0);
$result = mysqli_query($sdb, $sphinxQuery);
error_reporting($lev);
$error = mysqli_error($sdb);
// A 1064 error means that the query is invalid, so we don't care
// about that.
if ($error && mysqli_errno($sdb) != 1064) {
// All other errors we will considered a failure.
return false;
}
// Get the query metadata.
$meta = array();
$mresult = mysqli_query($sdb, "SHOW META");
if (!$mresult) {
return false;
}
while ($row = mysqli_fetch_row($mresult)) {
$meta[$row[0]] = $row[1];
}
$results = array();
if ($result) {
while ($row = mysqli_fetch_assoc($result)) {
if ($lookupQuery) {
// Save the IDs for a batch lookup.
$results[] = $row["id"];
} else {
$results[] = $row;
}
}
}
if ($lookupQuery && count($results) > 0) {
$ndb = new DB();
$sql = sprintf($lookupQuery, implode(",", $results));
$result = $ndb->queryDirect($sql);
if ($result) {
$results = array();
while ($row = $ndb->getAssocArray($result)) {
$results[] = $row;
}
}
}
$count = 0;
if (count($results) > 0 && array_key_exists("total", $meta)) {
$count = (int) $meta["total_found"];
$results[0]["_totalrows"] = $count > MAX_MATCHES ? MAX_MATCHES : $count;
}
if ($useCache !== false && $cache->enabled) {
$cache->store($sphinxQuery, $results, $useCache);
}
return $results;
}
public function processTvReleases($site, $numtoProcess = 100)
{
$ret = 0;
$db = new DB();
$tvmaze = new TVMaze();
$tvdb = new TheTVDB();
$lookupTvMaze = $site->lookuptvmaze;
$lookuptheTvDB = $site->lookupthetvdb;
// get all releases without a tvinfo which are in a tv category.
$result = $db->queryDirect(sprintf("SELECT searchname, ID from releases where tvinfoID = -1 and categoryID in ( select ID from category where parentID = %d ) order by postdate desc limit %d ", Category::CAT_PARENT_TV, $numtoProcess));
if ($db->getNumRows($result) > 0) {
if ($this->echooutput) {
echo "TVInfo : Looking up " . $db->getNumRows($result) . " releases. TvMaze:" . ($lookupTvMaze ? "on" : "off") . " theTVDB:" . ($lookuptheTvDB ? "on" : "off") . "\n";
}
while ($arr = $db->getAssocArray($result)) {
$show = $this->parseNameEpSeason($arr['searchname']);
if ($show && $show['name'] != '') {
// find the ID locally
$tvinforow = $this->getTvInfoByTitle($show['cleanname']);
if ($tvinforow && $this->echooutput) {
echo sprintf("TVInfo : Found %s %s %s\n", $show['cleanname'], $show['seriesfull'], $tvinforow['localonly'] == 1 ? "Local Only" : "");
}
// if not found locally try maze
if (!$tvinforow && $lookupTvMaze) {
if ($this->echooutput) {
echo "TVInfo : Didnt find " . $show['cleanname'] . " locally, checking TvMaze\n";
}
$tvmShow = $tvmaze->singleSearch($show['cleanname']);
if ($tvmShow && $this->compare($show['cleanname'], $tvmShow->name) >= TvInfo::MATCH_PROBABILITY) {
$this->addTvInfoFromMaze($show['cleanname'], $tvmShow);
$tvinforow = $this->getByMazeID($tvmShow->id);
//get back out on maze id, not insertid, incase it failed to insert due to mazeid already existing
}
}
// if not found locally or maze try tvdb
if (!$tvinforow && $lookuptheTvDB) {
$seriesinfo = $tvdb->lookupSeriesID($show['cleanname']);
if ($seriesinfo && $seriesinfo["id"] > 0 && $this->compare($show['cleanname'], $seriesinfo["name"]) >= TvInfo::MATCH_PROBABILITY) {
$tvinforow = $this->getByTvdbID($seriesinfo["id"]);
if (!$tvinforow) {
if ($this->echooutput) {
echo "TVInfo : Didnt find " . $show['cleanname'] . " locally, checking theTVDB\n";
}
$tvdbShow = $tvdb->lookupSeries($seriesinfo["id"], $show['cleanname'], false);
$this->addTvInfoFromTVDB($show, $tvdbShow);
$tvinforow = $this->getByTvdbID($tvdbShow["tvdbID"]);
//get back out on tvdb id, not insertid, incase it failed to insert due to tvdbid already existing
}
}
}
// if tv try and get episode info
if ($tvinforow) {
$seriesfull = isset($show['seriesfull']) && !empty($show['seriesfull']) ? $db->escapeString($show['seriesfull']) : "null";
$season = isset($show['season']) && !empty($show['season']) ? $db->escapeString($show['season']) : "null";
$episode = isset($show['episode']) && !empty($show['episode']) ? $db->escapeString($show['episode']) : "null";
$tvairdate = isset($show['airdate']) && !empty($show['airdate']) ? $db->escapeString($show['airdate']) : "null";
$tvepisodetitle = "null";
$shortfullep = str_replace('S', '', $show['season']) . 'x' . str_replace('E', '', $show['episode']);
$episodeid = "null";
// only bother looking up for episode data if the match wasnt to a local tvinfo row
if ($tvinforow["localonly"] == 0) {
//check local releases to see if we already have the data (release or episodeinfo)
$epsql = sprintf("select tvtitle as title, tvairdate as airdate from releases\n where tvairdate is not null and season = %s and episode = %s and tvinfoID = %d\n union select eptitle as title, airdate from episodeinfo where tvinfoID = %d and fullep = %s", $db->escapeString($show['season']), $db->escapeString($show['episode']), $tvinforow["ID"], $tvinforow["ID"], $db->escapeString($shortfullep));
$epinfo = $db->queryOneRow($epsql);
// check maze for ep if mazeID is known
if (!$epinfo && $lookupTvMaze && $tvinforow["mazeID"]) {
if (strpos($show["seriesfull"], '/') !== false) {
$tvmEpInfo = $tvmaze->getDatedEpisode($tvinforow["mazeID"], str_replace("/", "-", $show["seriesfull"]));
} else {
$tvmEpInfo = $tvmaze->getNumberedEpisode($tvinforow["mazeID"], str_replace('S', '', $show['season']), str_replace('E', '', $show['episode']));
}
if ($tvmEpInfo) {
$episodeid = $this->addEpisode(null, $tvmEpInfo->id, $tvinforow["releasetitle"], $tvmEpInfo->airdate, $tvmEpInfo->season, $tvmEpInfo->number, $tvmEpInfo->name, null, null, $tvmEpInfo->summary, null, null, null, $tvinforow["ID"]);
$epinfo = $this->getEpisodeInfoByID($episodeid);
}
}
// check tvdb for ep if tvdb is known
if (!$epinfo && $lookuptheTvDB && $tvinforow["tvdbID"]) {
$tvdbEpInfo = $tvdb->lookupEpisode($tvinforow["tvdbID"], $show);
if ($tvdbEpInfo) {
$episodeid = $this->addEpisode($tvdbEpInfo["id"], null, $tvinforow["releasetitle"], $tvdbEpInfo["airdate"], $tvdbEpInfo["season"], $tvdbEpInfo["number"], $tvdbEpInfo["name"], $tvdbEpInfo["director"], $tvdbEpInfo["gueststars"], $tvdbEpInfo["summary"], $tvdbEpInfo["rating"], $tvdbEpInfo["writer"], $tvdbEpInfo["epabsolute"], $tvinforow["ID"]);
$epinfo = $this->getEpisodeInfoByID($episodeid);
}
}
if ($epinfo) {
if (!empty($epinfo['airdate'])) {
$tvairdate = $db->escapeString($epinfo['airdate']);
}
if (!empty($epinfo['eptitle'])) {
$tvepisodetitle = $db->escapeString($epinfo['eptitle']);
}
}
}
$db->exec(sprintf("update releases set tvinfoID=%d, seriesfull = %s, season = %s, episode = %s, tvairdate=%s, tvtitle=%s, episodeinfoID=%s where ID = %d", $tvinforow["ID"], $seriesfull, $season, $episode, $tvairdate, $tvepisodetitle, $episodeid, $arr['ID']));
} else {
$this->markAsNotFound($show, $arr["ID"]);
}
} else {
$this->markAsNotFound($show, $arr["ID"]);
}
$ret++;
}
}
return $ret;
}
function processReleases()
{
require_once WWW_DIR . "/lib/binaries.php";
$db = new DB();
$currTime_ori = $db->queryOneRow("SELECT NOW() as now");
$cat = new Category();
$nzb = new Nzb();
$s = new Sites();
$releaseRegex = new ReleaseRegex();
$page = new Page();
$groups = new Groups();
$retcount = 0;
echo $s->getLicense();
echo "\n\nStarting release update process (" . date("Y-m-d H:i:s") . ")\n";
if (!file_exists($page->site->nzbpath)) {
echo "Bad or missing nzb directory - " . $page->site->nzbpath;
return -1;
}
//
// Get all regexes for all groups which are to be applied to new binaries
// in order of how they should be applied
//
$releaseRegex->get();
echo "Stage 1 : Applying regex to binaries\n";
$activeCategories = $cat->get();
$catbasedsizes = $db->getLookupAsArray($activeCategories, "ID");
$activeGroups = $groups->getActive(false);
$groupbasedminsizes = $db->getLookupAsArray($groups->getAllNoReleases(), "ID");
foreach ($activeGroups as $groupArr) {
//check if regexes have already been applied during update binaries
if ($groupArr['regexmatchonly'] == 1) {
continue;
}
$groupRegexes = $releaseRegex->getForGroup($groupArr['name']);
echo "Stage 1 : Applying " . sizeof($groupRegexes) . " regexes to group " . $groupArr['name'] . "\n";
// Get out all binaries of STAGE0 for current group
$newUnmatchedBinaries = array();
$ressql = sprintf("SELECT binaries.ID, binaries.name, binaries.date, binaries.totalParts, binaries.procstat, binaries.fromname from binaries where groupID = %d and procstat IN (%d,%d) and regexID IS NULL order by binaries.date asc", $groupArr['ID'], Releases::PROCSTAT_NEW, Releases::PROCSTAT_TITLENOTMATCHED);
$resbin = $db->queryDirect($ressql);
$matchedbins = 0;
while ($rowbin = $db->getAssocArray($resbin)) {
$regexMatches = array();
foreach ($groupRegexes as $groupRegex) {
$regexCheck = $releaseRegex->performMatch($groupRegex, $rowbin['name']);
if ($regexCheck !== false) {
$regexMatches = $regexCheck;
break;
}
}
if (!empty($regexMatches)) {
$matchedbins++;
$relparts = explode("/", $regexMatches['parts']);
$db->exec(sprintf("update binaries set relname = replace(%s, '_', ' '), relpart = %d, reltotalpart = %d, procstat=%d, categoryID=%s, regexID=%d, reqID=%s where ID = %d", $db->escapeString($regexMatches['name']), $relparts[0], $relparts[1], Releases::PROCSTAT_TITLEMATCHED, $regexMatches['regcatid'], $regexMatches['regexID'], $db->escapeString($regexMatches['reqID']), $rowbin["ID"]));
} else {
if ($rowbin['procstat'] == Releases::PROCSTAT_NEW) {
$newUnmatchedBinaries[] = $rowbin['ID'];
}
}
}
//mark as not matched
if (!empty($newUnmatchedBinaries)) {
$db->exec(sprintf("update binaries set procstat=%d where ID IN (%s)", Releases::PROCSTAT_TITLENOTMATCHED, implode(',', $newUnmatchedBinaries)));
}
}
//
// Move all binaries from releases which have the correct number of files on to the next stage.
//
echo "Stage 2 : Marking binaries where all parts are available";
$result = $db->queryDirect(sprintf("SELECT relname, date, SUM(reltotalpart) AS reltotalpart, groupID, reqID, fromname, SUM(num) AS num, coalesce(g.minfilestoformrelease, s.minfilestoformrelease) as minfilestoformrelease FROM ( SELECT relname, reltotalpart, groupID, reqID, fromname, max(date) as date, COUNT(ID) AS num FROM binaries WHERE procstat = %s GROUP BY relname, reltotalpart, groupID, reqID, fromname ORDER BY NULL ) x left outer join groups g on g.ID = x.groupID inner join ( select value as minfilestoformrelease from site where setting = 'minfilestoformrelease' ) s GROUP BY relname, groupID, reqID, fromname, minfilestoformrelease ORDER BY NULL", Releases::PROCSTAT_TITLEMATCHED));
while ($row = $db->getAssocArray($result)) {
$retcount++;
//
// Less than the site permitted number of files in a release. Dont discard it, as it may
// be part of a set being uploaded.
//
if ($row["num"] < $row["minfilestoformrelease"]) {
//echo "Number of files in release ".$row["relname"]." less than site/group setting (".$row['num']."/".$row["minfilestoformrelease"].")\n";
//$db->exec(sprintf("update binaries set procattempts = procattempts + 1 where relname = %s and procstat = %d and groupID = %d and fromname = %s", $db->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $db->escapeString($row["fromname"]) ));
} elseif ($row["num"] >= $row["reltotalpart"]) {
$incomplete = false;
if ($row['reltotalpart'] == 0 && strtotime($currTime_ori['now']) - strtotime($row['date']) < 14400) {
$incomplete = true;
} else {
// Check that the binary is complete
$binlist = $db->query(sprintf("SELECT binaries.ID, totalParts, date, COUNT(DISTINCT parts.messageID) AS num FROM binaries, parts WHERE binaries.ID=parts.binaryID AND binaries.relname = %s AND binaries.procstat = %d AND binaries.groupID = %d AND binaries.fromname = %s GROUP BY binaries.ID ORDER BY NULL", $db->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $db->escapeString($row["fromname"])));
foreach ($binlist as $rowbin) {
if ($rowbin['num'] < $rowbin['totalParts']) {
// Allow to binary to release if posted to usenet longer than four hours ago and we still don't have all the parts
if (!(strtotime($currTime_ori['now']) - strtotime($rowbin['date']) > 14400)) {
$incomplete = true;
break;
}
}
}
}
if (!$incomplete) {
//
// Right number of files, but see if the binary is a allfilled/reqid post, in which case it needs its name looked up
//
if ($row['reqID'] != '' && $page->site->reqidurl != "") {
//
// Try and get the name using the group
//
$binGroup = $db->queryOneRow(sprintf("SELECT name FROM groups WHERE ID = %d", $row["groupID"]));
$newtitle = $this->getReleaseNameForReqId($page->site->reqidurl, $page->site->newznabID, $binGroup["name"], $row["reqID"]);
//
// if the feed/group wasnt supported by the scraper, then just use the release name as the title.
//
if ($newtitle == "no feed") {
$newtitle = $row["relname"];
}
//
// Valid release with right number of files and title now, so move it on
//
if ($newtitle != "") {
$db->exec(sprintf("update binaries set relname = %s, procstat=%d where relname = %s and procstat = %d and groupID = %d and fromname=%s", $db->escapeString($newtitle), Releases::PROCSTAT_READYTORELEASE, $db->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $db->escapeString($row["fromname"])));
} else {
//
// Item not found, if the binary was added to the index yages ago, then give up.
//
$maxaddeddate = $db->queryOneRow(sprintf("SELECT NOW() as now, MAX(dateadded) as dateadded FROM binaries WHERE relname = %s and procstat = %d and groupID = %d and fromname=%s", $db->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $db->escapeString($row["fromname"])));
//
// If added to the index over 48 hours ago, give up trying to determine the title
//
if (strtotime($maxaddeddate['now']) - strtotime($maxaddeddate['dateadded']) > 60 * 60 * 48) {
$db->exec(sprintf("update binaries set procstat=%d where relname = %s and procstat = %d and groupID = %d and fromname=%s", Releases::PROCSTAT_NOREQIDNAMELOOKUPFOUND, $db->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $db->escapeString($row["fromname"])));
}
}
} else {
$db->exec(sprintf("update binaries set procstat=%d where relname = %s and procstat = %d and groupID = %d and fromname=%s", Releases::PROCSTAT_READYTORELEASE, $db->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $db->escapeString($row["fromname"])));
}
}
}
if ($retcount % 100 == 0) {
echo ".";
}
}
$retcount = 0;
echo "\nStage 3 : Creating releases from complete binaries\n";
//
// Get out all distinct relname, group from binaries of STAGE2
//
$result = $db->queryDirect(sprintf("SELECT relname, groupID, g.name as group_name, fromname, max(categoryID) as categoryID, max(regexID) as regexID, max(reqID) as reqID, MAX(date) as date, count(binaries.ID) as parts, coalesce(sum(binaries.size),0) as size from binaries inner join groups g on g.ID = binaries.groupID where procstat = %d and relname is not null group by relname, g.name, groupID, fromname ORDER BY COUNT(binaries.ID) desc", Releases::PROCSTAT_READYTORELEASE));
while ($row = $db->getAssocArray($result)) {
$relguid = md5(uniqid());
//
// Get categoryID if one has been allocated to this
//
if ($row["categoryID"] != "") {
$catId = $row["categoryID"];
} else {
$catId = $cat->determineCategory($row["group_name"], $row["relname"]);
}
//
// Determine if size matches permitted boundaries and discard here if not.
//
$gsize = $groupbasedminsizes[$row["groupID"]][0]["minsizetoformrelease"];
if ($gsize == "" || $gsize == 0) {
$gsize = $row["size"];
}
$ssize = $page->site->minsizetoformrelease;
if ($ssize == "" || $ssize == 0) {
$ssize = $row["size"];
}
$csize = $catbasedsizes[$catId][0]["minsizetoformrelease"];
if ($csize == "" || $csize == 0) {
$csize = $row["size"];
}
$cpsize = $catbasedsizes[$catId][0]["parentminsizetoformrelease"];
if ($cpsize == "" || $cpsize == 0) {
$cpsize = $row["size"];
}
$cmaxsize = $catbasedsizes[$catId][0]["maxsizetoformrelease"];
if ($cmaxsize == "" || $cmaxsize == 0) {
$cmaxsize = $row["size"];
}
$cpmaxsize = $catbasedsizes[$catId][0]["parentmaxsizetoformrelease"];
if ($cpmaxsize == "" || $cpmaxsize == 0) {
$cpmaxsize = $row["size"];
}
$overallminsize = max($gsize, $ssize, $csize, $cpsize);
$overallmaxsize = min($cmaxsize, $cpmaxsize);
if ($row["size"] < $overallminsize || $row["size"] > $overallmaxsize) {
echo sprintf("Stage 3 : Discarding - %s (Size %s outside permitted range of %s%s)\n", $row["relname"], formatBytes($row["size"]), $overallminsize != $row["size"] ? formatBytes($overallminsize) : "", $overallmaxsize != $row["size"] ? formatBytes($overallmaxsize) : "");
$db->exec(sprintf("DELETE parts, binaries FROM parts JOIN binaries ON binaries.ID = parts.binaryID WHERE relname=%s and groupID=%d and fromname=%s and procstat=%d", $db->escapeString($row["relname"]), $row["groupID"], $db->escapeString($row["fromname"]), Releases::PROCSTAT_READYTORELEASE));
} else {
// Clean release name
$cleanRelName = $this->cleanReleaseName($row['relname']);
$relid = $this->insertRelease($cleanRelName, $row["parts"], $row["groupID"], $relguid, $catId, $row["regexID"], $row["date"], $row["fromname"], $row["reqID"], $page->site);
//
// Tag every binary for this release with its parent release id
//
$db->exec(sprintf("update binaries set procstat = %d, releaseID = %d where relname = %s and procstat = %d and groupID = %d and fromname=%s", Releases::PROCSTAT_RELEASED, $relid, $db->escapeString($row["relname"]), Releases::PROCSTAT_READYTORELEASE, $row["groupID"], $db->escapeString($row["fromname"])));
//
// Write the nzb to disk
//
$nzbfile = $nzb->getNZBPath($relguid, $page->site->nzbpath, true);
$nzb->writeNZBforReleaseId($relid, $cleanRelName, $catId, $nzbfile);
//
// Remove used binaries
//
$db->exec(sprintf("DELETE parts, binaries FROM parts JOIN binaries ON binaries.ID = parts.binaryID WHERE releaseID = %d ", $relid));
//
// If nzb successfully written, then load it and get size completion from it
//
$nzbInfo = new nzbInfo();
if (!$nzbInfo->loadFromFile($nzbfile)) {
echo "Stage 3 : Failed to write nzb file (bad perms?) " . $nzbfile . "\n";
//copy($nzbfile, "./ERRORNZB_".$relguid);
$this->delete($relid);
} else {
// Check if gid already exists
$dupes = $db->queryOneRow(sprintf("SELECT EXISTS(SELECT 1 FROM releases WHERE gid = %s) as total", $db->escapeString($nzbInfo->gid)));
if ($dupes['total'] > 0) {
echo "Stage 3 : Duplicate - " . $cleanRelName . " -" . $nzbInfo->gid . "-\n";
$this->delete($relid);
} else {
$db->exec(sprintf("update releases set totalpart = %d, size = %s, completion = %d, GID=%s where ID = %d", $nzbInfo->filecount, $nzbInfo->filesize, $nzbInfo->completion, $db->escapeString($nzbInfo->gid), $relid));
echo "Stage 3 : Added release " . $cleanRelName . "\n";
//Increment new release count
$retcount++;
}
}
}
}
//
// Delete any releases under the minimum completion percent.
//
if ($page->site->completionpercent != 0) {
echo "Stage 4 : Deleting releases less than " . $page->site->completionpercent . " complete\n";
$result = $db->query(sprintf("select ID from releases where completion > 0 and completion < %d", $page->site->completionpercent));
foreach ($result as $row) {
$this->delete($row["ID"]);
}
}
/*
*Potentially leave this in to mop up release when the cat sizes change.
//
// Delete releases whos minsize is less than the site or group minimum
//
$result = $db->query("select releases.ID from releases left outer join (SELECT g.ID, coalesce(g.minsizetoformrelease, s.minsizetoformrelease) as minsizetoformrelease FROM groups g inner join ( select value as minsizetoformrelease from site where setting = 'minsizetoformrelease' ) s ) x on x.ID = releases.groupID where minsizetoformrelease != 0 and releases.size < minsizetoformrelease");
if (count($result) > 0)
{
echo "Stage 4 : Deleting ".count($result)." release(s) where size is smaller than minsize for site/group\n";
foreach ($result as $row)
$this->delete($row["ID"]);
}
$result = $db->query("select releases.ID, name, categoryID, size FROM releases JOIN (
select
catc.ID,
case when catc.minsizetoformrelease = 0 then catp.minsizetoformrelease else catc.minsizetoformrelease end as minsizetoformrelease,
case when catc.maxsizetoformrelease = 0 then catp.maxsizetoformrelease else catc.maxsizetoformrelease end as maxsizetoformrelease
from category catp join category catc on catc.parentID = catp.ID
where (catc.minsizetoformrelease != 0 or catc.maxsizetoformrelease != 0) or (catp.minsizetoformrelease != 0 or catp.maxsizetoformrelease != 0)
) x on x.ID = releases.categoryID
where
(size < minsizetoformrelease and minsizetoformrelease != 0) or
(size > maxsizetoformrelease and maxsizetoformrelease != 0)");
if(count($result) > 0)
{
echo "Stage 4 : Deleting release(s) not matching category min/max size ...\n";
foreach ($result as $r){
$this->delete($r['ID']);
}
}
*/
echo "Stage 5 : Post processing started\n";
$postprocess = new PostProcess(true);
$postprocess->processAll();
//
// aggregate the releasefiles upto the releases.
//
echo "Stage 6 : Aggregating Files\n";
$db->exec("update releases INNER JOIN (SELECT releaseID, COUNT(ID) AS num FROM releasefiles GROUP BY releaseID) b ON b.releaseID = releases.ID and releases.rarinnerfilecount = 0 SET rarinnerfilecount = b.num");
// Remove the binaries and parts used to form releases, or that are duplicates.
//
if ($page->site->partsdeletechunks > 0) {
echo "Stage 7 : Chunk deleting unused binaries and parts";
$query = sprintf("SELECT parts.ID as partsID,binaries.ID as binariesID FROM parts\r\n\t\t\t\t\t\tLEFT JOIN binaries ON binaries.ID = parts.binaryID\r\n\t\t\t\t\t\tWHERE binaries.dateadded < %s - INTERVAL %d HOUR LIMIT 0,%d", $db->escapeString($currTime_ori["now"]), ceil($page->site->rawretentiondays * 24), $page->site->partsdeletechunks);
$cc = 0;
$done = false;
while (!$done) {
$dd = $cc;
$result = $db->query($query);
if (count($result) > 0) {
$pID = array();
$bID = array();
foreach ($result as $row) {
$pID[] = $row['partsID'];
$bID[] = $row['binariesID'];
}
$pID = '(' . implode(',', $pID) . ')';
$bID = '(' . implode(',', $bID) . ')';
$fr = $db->exec("DELETE FROM parts WHERE ID IN {$pID}");
if ($fr > 0) {
$cc += $fr;
$cc += $db->exec("DELETE FROM binaries WHERE ID IN {$bID}");
}
unset($pID);
unset($bID);
if ($cc == $dd) {
$done = true;
}
echo $cc % 10000 ? '.' : '';
} else {
$done = true;
}
}
echo "\nStage 7 : Complete - " . $cc . " rows affected\n";
} else {
echo "Stage 7 : Deleting unused binaries and parts\n";
$db->exec(sprintf("DELETE parts, binaries FROM parts JOIN binaries ON binaries.ID = parts.binaryID\r\n\t\t\tWHERE binaries.dateadded < %s - INTERVAL %d HOUR", $db->escapeString($currTime_ori["now"]), ceil($page->site->rawretentiondays * 24)));
}
//
// User/Request housekeeping, should ideally move this to its own section, but it needs to be done automatically.
//
$users = new Users();
$users->pruneRequestHistory($page->site->userdownloadpurgedays);
echo "Done : Added " . $retcount . " releases\n\n";
return $retcount;
}
<?php
//
// Script will dump out all nfos in the system into a folder based on the date posted to usenet ./YYYYMMDD/release.nfo
//
define('FS_ROOT', realpath(dirname(__FILE__)));
require_once FS_ROOT . "/../../www/config.php";
require_once FS_ROOT . "/../../www/lib/util.php";
$db = new DB();
$res = $db->queryDirect("select releases.searchname, releases.postdate, uncompress(releasenfo.nfo) as nfo from releases inner join releasenfo on releases.ID = releasenfo.releaseID and releasenfo.nfo is not null order by postdate");
while ($row = $db->getAssocArray($res)) {
$dir = date("Ymd", strtotime($row["postdate"]));
if (!file_exists($dir)) {
mkdir($dir);
}
$filename = $dir . "/" . safeFilename($row["searchname"]) . ".nfo";
if (!file_exists($filename)) {
$fh = fopen($filename, 'w');
fwrite($fh, cp437toUTF($row["nfo"]));
fclose($fh);
}
}
$i["isdir"] = is_dir($f) ? 1 : 0;
$i["fullname"] = $f;
$i["webpath"] = $webpath . "/" . $i["name"] . ($i["isdir"] == 1 ? "/" : "");
$i["mtime"] = filemtime($f);
$i["pathinfo"] = pathinfo($f);
if ($i["isdir"] == 1) {
$i["pathinfo"]["extension"] = "";
$dirs[] = $i["name"];
} else {
$i["size"] = filesize($f);
}
$items[$i["name"]] = $i;
}
if (!$listmode) {
$relres = $r->getByNames($dirs);
while ($rel = $db->getAssocArray($relres)) {
if (isset($items[$rel["searchname"]])) {
$items[$rel["searchname"]]["release"] = $rel;
}
}
}
uasort($items, 'sortbymodified');
$page->smarty->assign('results', $items);
$page->smarty->assign('lm', $listmode);
if ($subpath != "") {
$page->smarty->assign('subpath', $subpath . "/");
}
$parentpath = "";
if ($subpath != "") {
$pos = strrpos($subpath, "/");
if ($pos !== false) {
/**
* Process all untagged book releases for additional metadata.
*/
public function processBookReleases()
{
$ret = 0;
$db = new DB();
$numlookedup = 0;
$res = $db->queryDirect(sprintf("SELECT searchname, ID from releases where bookinfoID IS NULL and categoryID = %d ORDER BY postdate DESC LIMIT 100", Category::CAT_BOOK_EBOOK));
if ($db->getNumRows($res) > 0) {
if ($this->echooutput) {
echo "BookPrc : Processing " . $db->getNumRows($res) . " book releases\n";
}
while ($arr = $db->getAssocArray($res)) {
if ($numlookedup > Book::NUMTOPROCESSPERTIME) {
return;
}
$bookId = -2;
$book = $this->parseAuthor($arr['searchname']);
if ($book !== false) {
if ($this->echooutput) {
echo 'BookPrc : ' . $book["author"] . ' - ' . $book["title"] . "\n";
}
//check for existing book entry
$bookCheck = $this->getBookInfoByName($book["author"], $book["title"]);
if ($bookCheck === false) {
//
// get from amazon
//
$numlookedup++;
$ret = $this->updateBookInfo($book["author"], $book["title"]);
if ($ret !== false) {
$bookId = $ret;
}
} else {
$bookId = $bookCheck["ID"];
}
}
$db->exec(sprintf("update releases SET bookinfoID = %d WHERE ID = %d", $bookId, $arr["ID"]));
}
}
}
require_once "config.php";
require_once WWW_DIR . "/lib/adminpage.php";
require_once WWW_DIR . "/lib/site.php";
require_once WWW_DIR . "/lib/framework/db.php";
$page = new AdminPage();
$page->title = "Admin Hangout";
$statusmsgs = array();
//
// status messages
//
//
// mysql config settings
//
$db = new DB();
$result = $db->queryDirect("SELECT @@group_concat_max_len, @@max_allowed_packet");
$data = $db->getAssocArray($result);
if ($data['@@group_concat_max_len'] < 8192) {
$statusmsgs[] = addmsg("MySql my.cnf setting group_concat_max_len is too low, should be >= 8192.", "http://dev.mysql.com/doc/refman/5.1/en/server-system-variables.html#sysvar_group_concat_max_len");
}
if ($data['@@max_allowed_packet'] < 12582912) {
$statusmsgs[] = addmsg("MySql my.cnf setting max_allowed_packet is too low, should be >= 12582912.", "http://dev.mysql.com/doc/refman/5.1/en/server-system-variables.html#sysvar_max_allowed_packet");
}
//
// default keys not changed
//
if ($page->site->amazonpubkey == "AKIAIPDNG5EU7LB4AD3Q" && $page->site->lookupmusic + $page->site->lookupgames + $page->site->lookupbooks != 0) {
$statusmsgs[] = addmsg("Amazon shared key in use. Not using your own Amazon API key will result in failed amazon lookups.", "http://newznab.readthedocs.org/en/latest/faq/");
}
if ($page->site->rawretentiondays > 10) {
$statusmsgs[] = addmsg("Binary header retention is set at " . $page->site->rawretentiondays . ". Having this value any higher than 2 can cause the database to grow very large.", "site-edit.php");
}
$group = $_POST["group"];
}
if (isset($_POST["categoryID"])) {
$category = $_POST["categoryID"];
}
}
if ($path != "") {
if (substr($path, strlen($path) - 1) != '/') {
$path = $path . "/";
}
$releases = $rel->getForExport($postfrom, $postto, $group, $category);
$s = new Sites();
$nzb = new NZB();
$site = $s->get();
$nzbCount = 0;
while ($release = $db->getAssocArray($releases)) {
$catname = safeFilename($release["catName"]);
if (!file_exists($path . $catname)) {
mkdir($path . $catname);
}
ob_start();
@readgzfile($nzb->getNZBPath($release["guid"], $site->nzbpath));
$nzbfile = ob_get_contents();
ob_end_clean();
$filename = getFilename($path . $catname . "/", $release["searchname"]);
echo $filename . "\n";
$fh = fopen($filename, 'w');
fwrite($fh, $nzbfile);
fclose($fh);
$nzbCount++;
if ($nzbCount % 10 == 0 && !empty($argc)) {
/**
* Process all untagged releases to see if they are found in predb.
*/
public function processReleases($daysback = 3)
{
$db = new DB();
if ($this->echooutput) {
echo "Predb : Updating releases with pre data\n";
}
$matched = 0;
$releasesQuery = $db->queryDirect(sprintf('SELECT ID, searchname FROM releases WHERE preID IS NULL AND adddate > DATE_SUB(NOW(), INTERVAL %d DAY)', $daysback));
while ($arr = $db->getAssocArray($releasesQuery)) {
$arr['searchname'] = str_replace(' ', '_', $arr['searchname']);
$sql = sprintf("SELECT ID FROM predb WHERE dirname = %s LIMIT 1", $db->escapeString($arr['searchname']));
$predbQuery = $db->queryOneRow($sql);
if ($predbQuery) {
$db->exec(sprintf('UPDATE releases SET preID = %d WHERE ID = %d', $predbQuery['ID'], $arr['ID']));
$matched++;
}
}
if ($this->echooutput) {
echo "Predb : Matched pre data to " . $matched . " releases\n";
}
}
/**
* Process all untagged movies to link them to a movieinfo row.
*/
public function processMovieReleases()
{
$ret = 0;
$db = new DB();
$nfo = new Nfo();
$res = $db->queryDirect(sprintf("SELECT searchname, ID from releases where imdbID IS NULL and categoryID in ( select ID from category where parentID = %d ) ORDER BY postdate DESC LIMIT 100", Category::CAT_PARENT_MOVIE));
if ($db->getNumRows($res) > 0) {
if ($this->echooutput) {
echo "MovProc : Processing " . $db->getNumRows($res) . " movie releases\n";
}
while ($arr = $db->getAssocArray($res)) {
$imdbID = false;
/* Preliminary IMDB ID Detection from NFO file */
$rawnfo = '';
if ($nfo->getNfo($arr['ID'], $rawnfo)) {
$imdbID = $this->parseImdbFromNfo($rawnfo);
}
if ($imdbID !== false) {
// Set IMDB (if found in nfo) and move along
$db->exec(sprintf("update releases set imdbID = %s where ID = %d", $db->escapeString($imdbID), $arr["ID"]));
//check for existing movie entry
$movCheck = $this->getMovieInfo($imdbID);
if ($movCheck === false || isset($movCheck['updateddate']) && time() - strtotime($movCheck['updateddate']) > 2592000) {
$movieId = $this->updateMovieInfo($imdbID);
}
continue;
}
$moviename = $this->parseMovieName($arr['searchname']);
if ($moviename !== false) {
if ($this->echooutput) {
echo 'MovProc : ' . $moviename . ' [' . $arr['searchname'] . ']' . "\n";
}
//$buffer = getUrl("https://www.google.com/search?source=ig&hl=en&rlz=&btnG=Google+Search&aq=f&oq=&q=".urlencode($moviename.' site:imdb.com'));
$buffer = getUrl("http://www.bing.com/search?&q=" . urlencode($moviename . ' site:imdb.com'));
// make sure we got some data
if ($buffer !== false && strlen($buffer)) {
$imdbId = $this->parseImdbFromNfo($buffer);
if ($imdbId !== false) {
//update release with imdb id
$db->exec(sprintf("update releases SET imdbID = %s WHERE ID = %d", $db->escapeString($imdbId), $arr["ID"]));
//check for existing movie entry
$movCheck = $this->getMovieInfo($imdbId);
if ($movCheck === false || isset($movCheck['updateddate']) && time() - strtotime($movCheck['updateddate']) > 2592000) {
$movieId = $this->updateMovieInfo($imdbId);
}
} else {
//no imdb id found, set to all zeros so we dont process again
$db->exec(sprintf("update releases SET imdbID = %d WHERE ID = %d", 0, $arr["ID"]));
}
} else {
//url fetch failed, will try next run
}
} else {
//no valid movie name found, set to all zeros so we dont process again
$db->exec(sprintf("update releases SET imdbID = %d WHERE ID = %d", 0, $arr["ID"]));
}
}
}
}
/**
* Check all untagged console releases for their extended metadata.
*/
public function processConsoleReleases()
{
$ret = 0;
$db = new DB();
$numlookedup = 0;
$res = $db->queryDirect(sprintf("SELECT searchname, ID from releases where consoleinfoID IS NULL and categoryID in ( select ID from category where parentID = %d ) ORDER BY postdate DESC LIMIT 100", Category::CAT_PARENT_GAME));
if ($db->getNumRows($res) > 0) {
if ($this->echooutput) {
echo "ConsPrc : Processing " . $db->getNumRows($res) . " console releases\n";
}
while ($arr = $db->getAssocArray($res)) {
if ($numlookedup > Console::NUMTOPROCESSPERTIME) {
return;
}
$gameInfo = $this->parseTitle($arr['searchname']);
if ($gameInfo !== false) {
if ($this->echooutput) {
echo 'ConsPrc : ' . $gameInfo["title"] . ' (' . $gameInfo["platform"] . ')' . "\n";
}
//check for existing console entry
$gameCheck = $this->getConsoleInfoByName($gameInfo["title"], $gameInfo["platform"]);
if ($gameCheck === false) {
$numlookedup++;
$gameId = $this->updateConsoleInfo($gameInfo);
if ($gameId === false) {
$gameId = -2;
}
} else {
$gameId = $gameCheck["ID"];
}
//update release
$db->exec(sprintf("update releases SET consoleinfoID = %d WHERE ID = %d", $gameId, $arr["ID"]));
} else {
//could not parse release title
$db->exec(sprintf("update releases SET consoleinfoID = %d WHERE ID = %d", -2, $arr["ID"]));
}
}
}
}
