function processReleases() { require_once WWW_DIR . "/lib/binaries.php"; $db = new DB(); $currTime_ori = $db->queryOneRow("SELECT NOW() as now"); $cat = new Category(); $nzb = new Nzb(); $s = new Sites(); $releaseRegex = new ReleaseRegex(); $page = new Page(); $groups = new Groups(); $retcount = 0; echo $s->getLicense(); echo "\n\nStarting release update process (" . date("Y-m-d H:i:s") . ")\n"; if (!file_exists($page->site->nzbpath)) { echo "Bad or missing nzb directory - " . $page->site->nzbpath; return -1; } // // Get all regexes for all groups which are to be applied to new binaries // in order of how they should be applied // $releaseRegex->get(); echo "Stage 1 : Applying regex to binaries\n"; $activeCategories = $cat->get(); $catbasedsizes = $db->getLookupAsArray($activeCategories, "ID"); $activeGroups = $groups->getActive(false); $groupbasedminsizes = $db->getLookupAsArray($groups->getAllNoReleases(), "ID"); foreach ($activeGroups as $groupArr) { //check if regexes have already been applied during update binaries if ($groupArr['regexmatchonly'] == 1) { continue; } $groupRegexes = $releaseRegex->getForGroup($groupArr['name']); echo "Stage 1 : Applying " . sizeof($groupRegexes) . " regexes to group " . $groupArr['name'] . "\n"; // Get out all binaries of STAGE0 for current group $newUnmatchedBinaries = array(); $ressql = sprintf("SELECT binaries.ID, binaries.name, binaries.date, binaries.totalParts, binaries.procstat, binaries.fromname from binaries where groupID = %d and procstat IN (%d,%d) and regexID IS NULL order by binaries.date asc", $groupArr['ID'], Releases::PROCSTAT_NEW, Releases::PROCSTAT_TITLENOTMATCHED); $resbin = $db->queryDirect($ressql); $matchedbins = 0; while ($rowbin = $db->getAssocArray($resbin)) { $regexMatches = array(); foreach ($groupRegexes as $groupRegex) { $regexCheck = $releaseRegex->performMatch($groupRegex, $rowbin['name']); if ($regexCheck !== false) { $regexMatches = $regexCheck; break; } } if (!empty($regexMatches)) { $matchedbins++; $relparts = explode("/", $regexMatches['parts']); $db->exec(sprintf("update binaries set relname = replace(%s, '_', ' '), relpart = %d, reltotalpart = %d, procstat=%d, categoryID=%s, regexID=%d, reqID=%s where ID = %d", $db->escapeString($regexMatches['name']), $relparts[0], $relparts[1], Releases::PROCSTAT_TITLEMATCHED, $regexMatches['regcatid'], $regexMatches['regexID'], $db->escapeString($regexMatches['reqID']), $rowbin["ID"])); } else { if ($rowbin['procstat'] == Releases::PROCSTAT_NEW) { $newUnmatchedBinaries[] = $rowbin['ID']; } } } //mark as not matched if (!empty($newUnmatchedBinaries)) { $db->exec(sprintf("update binaries set procstat=%d where ID IN (%s)", Releases::PROCSTAT_TITLENOTMATCHED, implode(',', $newUnmatchedBinaries))); } } // // Move all binaries from releases which have the correct number of files on to the next stage. // echo "Stage 2 : Marking binaries where all parts are available"; $result = $db->queryDirect(sprintf("SELECT relname, date, SUM(reltotalpart) AS reltotalpart, groupID, reqID, fromname, SUM(num) AS num, coalesce(g.minfilestoformrelease, s.minfilestoformrelease) as minfilestoformrelease FROM ( SELECT relname, reltotalpart, groupID, reqID, fromname, max(date) as date, COUNT(ID) AS num FROM binaries WHERE procstat = %s GROUP BY relname, reltotalpart, groupID, reqID, fromname ORDER BY NULL ) x left outer join groups g on g.ID = x.groupID inner join ( select value as minfilestoformrelease from site where setting = 'minfilestoformrelease' ) s GROUP BY relname, groupID, reqID, fromname, minfilestoformrelease ORDER BY NULL", Releases::PROCSTAT_TITLEMATCHED)); while ($row = $db->getAssocArray($result)) { $retcount++; // // Less than the site permitted number of files in a release. Dont discard it, as it may // be part of a set being uploaded. // if ($row["num"] < $row["minfilestoformrelease"]) { //echo "Number of files in release ".$row["relname"]." less than site/group setting (".$row['num']."/".$row["minfilestoformrelease"].")\n"; //$db->exec(sprintf("update binaries set procattempts = procattempts + 1 where relname = %s and procstat = %d and groupID = %d and fromname = %s", $db->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $db->escapeString($row["fromname"]) )); } elseif ($row["num"] >= $row["reltotalpart"]) { $incomplete = false; if ($row['reltotalpart'] == 0 && strtotime($currTime_ori['now']) - strtotime($row['date']) < 14400) { $incomplete = true; } else { // Check that the binary is complete $binlist = $db->query(sprintf("SELECT binaries.ID, totalParts, date, COUNT(DISTINCT parts.messageID) AS num FROM binaries, parts WHERE binaries.ID=parts.binaryID AND binaries.relname = %s AND binaries.procstat = %d AND binaries.groupID = %d AND binaries.fromname = %s GROUP BY binaries.ID ORDER BY NULL", $db->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $db->escapeString($row["fromname"]))); foreach ($binlist as $rowbin) { if ($rowbin['num'] < $rowbin['totalParts']) { // Allow to binary to release if posted to usenet longer than four hours ago and we still don't have all the parts if (!(strtotime($currTime_ori['now']) - strtotime($rowbin['date']) > 14400)) { $incomplete = true; break; } } } } if (!$incomplete) { // // Right number of files, but see if the binary is a allfilled/reqid post, in which case it needs its name looked up // if ($row['reqID'] != '' && $page->site->reqidurl != "") { // // Try and get the name using the group // $binGroup = $db->queryOneRow(sprintf("SELECT name FROM groups WHERE ID = %d", $row["groupID"])); $newtitle = $this->getReleaseNameForReqId($page->site->reqidurl, $page->site->newznabID, $binGroup["name"], $row["reqID"]); // // if the feed/group wasnt supported by the scraper, then just use the release name as the title. // if ($newtitle == "no feed") { $newtitle = $row["relname"]; } // // Valid release with right number of files and title now, so move it on // if ($newtitle != "") { $db->exec(sprintf("update binaries set relname = %s, procstat=%d where relname = %s and procstat = %d and groupID = %d and fromname=%s", $db->escapeString($newtitle), Releases::PROCSTAT_READYTORELEASE, $db->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $db->escapeString($row["fromname"]))); } else { // // Item not found, if the binary was added to the index yages ago, then give up. // $maxaddeddate = $db->queryOneRow(sprintf("SELECT NOW() as now, MAX(dateadded) as dateadded FROM binaries WHERE relname = %s and procstat = %d and groupID = %d and fromname=%s", $db->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $db->escapeString($row["fromname"]))); // // If added to the index over 48 hours ago, give up trying to determine the title // if (strtotime($maxaddeddate['now']) - strtotime($maxaddeddate['dateadded']) > 60 * 60 * 48) { $db->exec(sprintf("update binaries set procstat=%d where relname = %s and procstat = %d and groupID = %d and fromname=%s", Releases::PROCSTAT_NOREQIDNAMELOOKUPFOUND, $db->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $db->escapeString($row["fromname"]))); } } } else { $db->exec(sprintf("update binaries set procstat=%d where relname = %s and procstat = %d and groupID = %d and fromname=%s", Releases::PROCSTAT_READYTORELEASE, $db->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $db->escapeString($row["fromname"]))); } } } if ($retcount % 100 == 0) { echo "."; } } $retcount = 0; echo "\nStage 3 : Creating releases from complete binaries\n"; // // Get out all distinct relname, group from binaries of STAGE2 // $result = $db->queryDirect(sprintf("SELECT relname, groupID, g.name as group_name, fromname, max(categoryID) as categoryID, max(regexID) as regexID, max(reqID) as reqID, MAX(date) as date, count(binaries.ID) as parts, coalesce(sum(binaries.size),0) as size from binaries inner join groups g on g.ID = binaries.groupID where procstat = %d and relname is not null group by relname, g.name, groupID, fromname ORDER BY COUNT(binaries.ID) desc", Releases::PROCSTAT_READYTORELEASE)); while ($row = $db->getAssocArray($result)) { $relguid = md5(uniqid()); // // Get categoryID if one has been allocated to this // if ($row["categoryID"] != "") { $catId = $row["categoryID"]; } else { $catId = $cat->determineCategory($row["group_name"], $row["relname"]); } // // Determine if size matches permitted boundaries and discard here if not. // $gsize = $groupbasedminsizes[$row["groupID"]][0]["minsizetoformrelease"]; if ($gsize == "" || $gsize == 0) { $gsize = $row["size"]; } $ssize = $page->site->minsizetoformrelease; if ($ssize == "" || $ssize == 0) { $ssize = $row["size"]; } $csize = $catbasedsizes[$catId][0]["minsizetoformrelease"]; if ($csize == "" || $csize == 0) { $csize = $row["size"]; } $cpsize = $catbasedsizes[$catId][0]["parentminsizetoformrelease"]; if ($cpsize == "" || $cpsize == 0) { $cpsize = $row["size"]; } $cmaxsize = $catbasedsizes[$catId][0]["maxsizetoformrelease"]; if ($cmaxsize == "" || $cmaxsize == 0) { $cmaxsize = $row["size"]; } $cpmaxsize = $catbasedsizes[$catId][0]["parentmaxsizetoformrelease"]; if ($cpmaxsize == "" || $cpmaxsize == 0) { $cpmaxsize = $row["size"]; } $overallminsize = max($gsize, $ssize, $csize, $cpsize); $overallmaxsize = min($cmaxsize, $cpmaxsize); if ($row["size"] < $overallminsize || $row["size"] > $overallmaxsize) { echo sprintf("Stage 3 : Discarding - %s (Size %s outside permitted range of %s%s)\n", $row["relname"], formatBytes($row["size"]), $overallminsize != $row["size"] ? formatBytes($overallminsize) : "", $overallmaxsize != $row["size"] ? formatBytes($overallmaxsize) : ""); $db->exec(sprintf("DELETE parts, binaries FROM parts JOIN binaries ON binaries.ID = parts.binaryID WHERE relname=%s and groupID=%d and fromname=%s and procstat=%d", $db->escapeString($row["relname"]), $row["groupID"], $db->escapeString($row["fromname"]), Releases::PROCSTAT_READYTORELEASE)); } else { // Clean release name $cleanRelName = $this->cleanReleaseName($row['relname']); $relid = $this->insertRelease($cleanRelName, $row["parts"], $row["groupID"], $relguid, $catId, $row["regexID"], $row["date"], $row["fromname"], $row["reqID"], $page->site); // // Tag every binary for this release with its parent release id // $db->exec(sprintf("update binaries set procstat = %d, releaseID = %d where relname = %s and procstat = %d and groupID = %d and fromname=%s", Releases::PROCSTAT_RELEASED, $relid, $db->escapeString($row["relname"]), Releases::PROCSTAT_READYTORELEASE, $row["groupID"], $db->escapeString($row["fromname"]))); // // Write the nzb to disk // $nzbfile = $nzb->getNZBPath($relguid, $page->site->nzbpath, true); $nzb->writeNZBforReleaseId($relid, $cleanRelName, $catId, $nzbfile); // // Remove used binaries // $db->exec(sprintf("DELETE parts, binaries FROM parts JOIN binaries ON binaries.ID = parts.binaryID WHERE releaseID = %d ", $relid)); // // If nzb successfully written, then load it and get size completion from it // $nzbInfo = new nzbInfo(); if (!$nzbInfo->loadFromFile($nzbfile)) { echo "Stage 3 : Failed to write nzb file (bad perms?) " . $nzbfile . "\n"; //copy($nzbfile, "./ERRORNZB_".$relguid); $this->delete($relid); } else { // Check if gid already exists $dupes = $db->queryOneRow(sprintf("SELECT EXISTS(SELECT 1 FROM releases WHERE gid = %s) as total", $db->escapeString($nzbInfo->gid))); if ($dupes['total'] > 0) { echo "Stage 3 : Duplicate - " . $cleanRelName . " -" . $nzbInfo->gid . "-\n"; $this->delete($relid); } else { $db->exec(sprintf("update releases set totalpart = %d, size = %s, completion = %d, GID=%s where ID = %d", $nzbInfo->filecount, $nzbInfo->filesize, $nzbInfo->completion, $db->escapeString($nzbInfo->gid), $relid)); echo "Stage 3 : Added release " . $cleanRelName . "\n"; //Increment new release count $retcount++; } } } } // // Delete any releases under the minimum completion percent. // if ($page->site->completionpercent != 0) { echo "Stage 4 : Deleting releases less than " . $page->site->completionpercent . " complete\n"; $result = $db->query(sprintf("select ID from releases where completion > 0 and completion < %d", $page->site->completionpercent)); foreach ($result as $row) { $this->delete($row["ID"]); } } /* *Potentially leave this in to mop up release when the cat sizes change. // // Delete releases whos minsize is less than the site or group minimum // $result = $db->query("select releases.ID from releases left outer join (SELECT g.ID, coalesce(g.minsizetoformrelease, s.minsizetoformrelease) as minsizetoformrelease FROM groups g inner join ( select value as minsizetoformrelease from site where setting = 'minsizetoformrelease' ) s ) x on x.ID = releases.groupID where minsizetoformrelease != 0 and releases.size < minsizetoformrelease"); if (count($result) > 0) { echo "Stage 4 : Deleting ".count($result)." release(s) where size is smaller than minsize for site/group\n"; foreach ($result as $row) $this->delete($row["ID"]); } $result = $db->query("select releases.ID, name, categoryID, size FROM releases JOIN ( select catc.ID, case when catc.minsizetoformrelease = 0 then catp.minsizetoformrelease else catc.minsizetoformrelease end as minsizetoformrelease, case when catc.maxsizetoformrelease = 0 then catp.maxsizetoformrelease else catc.maxsizetoformrelease end as maxsizetoformrelease from category catp join category catc on catc.parentID = catp.ID where (catc.minsizetoformrelease != 0 or catc.maxsizetoformrelease != 0) or (catp.minsizetoformrelease != 0 or catp.maxsizetoformrelease != 0) ) x on x.ID = releases.categoryID where (size < minsizetoformrelease and minsizetoformrelease != 0) or (size > maxsizetoformrelease and maxsizetoformrelease != 0)"); if(count($result) > 0) { echo "Stage 4 : Deleting release(s) not matching category min/max size ...\n"; foreach ($result as $r){ $this->delete($r['ID']); } } */ echo "Stage 5 : Post processing started\n"; $postprocess = new PostProcess(true); $postprocess->processAll(); // // aggregate the releasefiles upto the releases. // echo "Stage 6 : Aggregating Files\n"; $db->exec("update releases INNER JOIN (SELECT releaseID, COUNT(ID) AS num FROM releasefiles GROUP BY releaseID) b ON b.releaseID = releases.ID and releases.rarinnerfilecount = 0 SET rarinnerfilecount = b.num"); // Remove the binaries and parts used to form releases, or that are duplicates. // if ($page->site->partsdeletechunks > 0) { echo "Stage 7 : Chunk deleting unused binaries and parts"; $query = sprintf("SELECT parts.ID as partsID,binaries.ID as binariesID FROM parts\r\n\t\t\t\t\t\tLEFT JOIN binaries ON binaries.ID = parts.binaryID\r\n\t\t\t\t\t\tWHERE binaries.dateadded < %s - INTERVAL %d HOUR LIMIT 0,%d", $db->escapeString($currTime_ori["now"]), ceil($page->site->rawretentiondays * 24), $page->site->partsdeletechunks); $cc = 0; $done = false; while (!$done) { $dd = $cc; $result = $db->query($query); if (count($result) > 0) { $pID = array(); $bID = array(); foreach ($result as $row) { $pID[] = $row['partsID']; $bID[] = $row['binariesID']; } $pID = '(' . implode(',', $pID) . ')'; $bID = '(' . implode(',', $bID) . ')'; $fr = $db->exec("DELETE FROM parts WHERE ID IN {$pID}"); if ($fr > 0) { $cc += $fr; $cc += $db->exec("DELETE FROM binaries WHERE ID IN {$bID}"); } unset($pID); unset($bID); if ($cc == $dd) { $done = true; } echo $cc % 10000 ? '.' : ''; } else { $done = true; } } echo "\nStage 7 : Complete - " . $cc . " rows affected\n"; } else { echo "Stage 7 : Deleting unused binaries and parts\n"; $db->exec(sprintf("DELETE parts, binaries FROM parts JOIN binaries ON binaries.ID = parts.binaryID\r\n\t\t\tWHERE binaries.dateadded < %s - INTERVAL %d HOUR", $db->escapeString($currTime_ori["now"]), ceil($page->site->rawretentiondays * 24))); } // // User/Request housekeeping, should ideally move this to its own section, but it needs to be done automatically. // $users = new Users(); $users->pruneRequestHistory($page->site->userdownloadpurgedays); echo "Done : Added " . $retcount . " releases\n\n"; return $retcount; }
<?php require_once WWW_DIR . "/lib/movie.php"; if (!$users->isLoggedIn()) { $page->show403(); } $m = new Movie(); $um = new UserMovies(); $db = new DB(); if (!isset($_GET["id"])) { $_GET["id"] = 1; } $lists = $db->getLookupAsArray($m->getUpcomingAll(), "source"); $data = $m->getUpcoming($_GET["id"]); if ($data["info"] == "") { $page->smarty->assign("nodata", "No data."); } else { $rtdata = json_decode($data["info"])->movies; $imdbids = array(); $ourmovieimdbs = array(); if (count($rtdata) > 0) { foreach ($rtdata as &$movie) { if (isset($movie->alternate_ids) && isset($movie->alternate_ids->imdb)) { $imdbids[] = $movie->alternate_ids->imdb; } $movie->source = $data["source"]; } } if (count($imdbids) > 0) { $allmovies = $m->getMovieInfoMultiImdb($imdbids); foreach ($allmovies as $ourmovie) {