/**
* Process headers and store in database for all active groups.
*/
function updateAllGroups()
{
$n = $this->n;
$groups = new Groups();
$res = $groups->getActive();
$s = new Sites();
echo $s->getLicense();
if ($res) {
shuffle($res);
$alltime = microtime(true);
echo 'Updating: ' . sizeof($res) . ' groups - Using compression? ' . ($this->compressedHeaders ? 'Yes' : 'No') . $n;
$nntp = new Nntp();
if ($nntp->doConnect()) {
$pos = 0;
foreach ($res as $groupArr) {
$pos++;
echo 'Group ' . $pos . ' of ' . sizeof($res) . $n;
$this->message = array();
$this->updateGroup($nntp, $groupArr);
}
$nntp->doQuit();
echo 'Updating completed in ' . number_format(microtime(true) - $alltime, 2) . ' seconds' . $n;
} else {
echo "Failed to get NNTP connection.{$n}";
}
} else {
echo "No groups specified. Ensure groups are added to newznab's database and activated before updating.{$n}";
}
}
/**
* @param int|string $groupID (optional)
*/
public function applyRegex($groupID)
{
//
// Get all regexes for all groups which are to be applied to new binaries
// in order of how they should be applied
//
$group = $this->groups->getCBPTableNames($this->tablePerGroup, $groupID);
$activeGroups = $this->groups->getActive();
$this->releaseRegex->get();
$this->pdo->log->doEcho($this->pdo->log->primary('Applying regex to binaries'), true);
foreach ($activeGroups as $groupArr) {
//check if regexes have already been applied during update binaries
if ($groupArr['regexmatchonly'] == 1) {
continue;
}
$groupRegexes = $this->releaseRegex->getForGroup($groupArr['name']);
$this->pdo->log->doEcho($this->pdo->log->primary('Applying ' . sizeof($groupRegexes) . ' regexes to group ' . $groupArr['name']), true);
// Get out all binaries of STAGE0 for current group
$newUnmatchedBinaries = array();
$ressql = sprintf('SELECT id, name, date, totalparts, procstat, fromname FROM %s b
WHERE groupid = %d AND procstat IN (%d, %d) AND regexid IS NULL ORDER BY b.date ASC', $group['bname'], $groupArr['id'], Releases::PROCSTAT_NEW, Releases::PROCSTAT_TITLENOTMATCHED);
$resbin = $this->pdo->queryDirect($ressql);
$matchedbins = 0;
while ($rowbin = $this->pdo->getAssocArray($resbin)) {
$regexMatches = array();
foreach ($groupRegexes as $groupRegex) {
$regexCheck = $this->releaseRegex->performMatch($groupRegex, $rowbin['name']);
if ($regexCheck !== false) {
$regexMatches = $regexCheck;
break;
}
}
if (!empty($regexMatches)) {
$matchedbins++;
$relparts = explode("/", $regexMatches['parts']);
$this->pdo->queryExec(sprintf("UPDATE %s SET relname = replace(%s, '_', ' '), relpart = %d, reltotalpart = %d, procstat=%d, categoryid=%s, regexid=%d, reqid=%s WHERE id = %d", $group['bname'], $this->pdo->escapeString($regexMatches['name']), $relparts[0], $relparts[1], Releases::PROCSTAT_TITLEMATCHED, $regexMatches['regcatid'], $regexMatches['regexid'], $this->pdo->escapeString($regexMatches['reqid']), $rowbin["id"]));
} else {
if ($rowbin['procstat'] == Releases::PROCSTAT_NEW) {
$newUnmatchedBinaries[] = $rowbin['id'];
}
}
}
//mark as not matched
if (!empty($newUnmatchedBinaries)) {
$this->pdo->queryExec(sprintf("UPDATE %s SET procstat=%d WHERE id IN (%s)", $group['bname'], Releases::PROCSTAT_TITLENOTMATCHED, implode(',', $newUnmatchedBinaries)));
}
}
}
/**
* Download new headers for all active groups.
*
* @param int $maxHeaders (Optional) How many headers to download max.
*
* @return void
*/
public function updateAllGroups($maxHeaders = 0)
{
$groups = $this->_groups->getActive();
$groupCount = count($groups);
if ($groupCount > 0) {
$counter = 1;
$allTime = microtime(true);
$this->log('Updating: ' . $groupCount . ' group(s) - Using compression? ' . ($this->_compressedHeaders ? 'Yes' : 'No'), 'updateAllGroups', \Logger::LOG_INFO, 'header');
// Loop through groups.
foreach ($groups as $group) {
$this->log('Starting group ' . $counter . ' of ' . $groupCount, 'updateAllGroups', \Logger::LOG_INFO, 'header');
$this->updateGroup($group, $maxHeaders);
$counter++;
}
$this->log('Updating completed in ' . number_format(microtime(true) - $allTime, 2) . ' seconds.', 'updateAllGroups', \Logger::LOG_INFO, 'primary');
} else {
$this->log('No groups specified. Ensure groups are added to newznab\'s database for updating.', 'updateAllGroups', \Logger::LOG_NOTICE, 'warning');
}
}
/**
* Update all active groups categories and descriptions.
*/
function backfillAllGroups($groupName = '', $backfillDate = null, $regexOnly = false)
{
$n = $this->n;
$groups = new Groups();
$res = false;
if ($groupName != '') {
$grp = $groups->getByName($groupName);
if ($grp) {
$res = array($grp);
}
} else {
$res = $groups->getActive();
}
if ($res) {
foreach ($res as $groupArr) {
$this->backfillGroup($groupArr, $backfillDate, $regexOnly);
}
} else {
echo "No groups specified. Ensure groups are added to newznab's database for updating.{$n}";
}
}
function backfillAllGroups($groupName = '', $backfillDate = null)
{
$n = $this->n;
$groups = new Groups();
if ($groupName != '') {
$grp = $groups->getByName($groupName);
if ($grp) {
$res = array($grp);
}
} else {
$res = $groups->getActive();
}
if ($res) {
$nntp = new Nntp();
$nntp->doConnect();
foreach ($res as $groupArr) {
$this->backfillGroup($nntp, $groupArr, $backfillDate);
}
$nntp->doQuit();
} else {
echo "No groups specified. Ensure groups are added to newznab's database for updating.{$n}";
}
}
echo $content;
} else {
header('Content-type: application/json');
echo json_encode(responseXmlToObject($content));
}
break;
//
// capabilities request
//
//
// capabilities request
//
case "c":
$parentcatlist = $category->getForMenu();
$page->smarty->assign('parentcatlist', $parentcatlist);
$grps = $grp->getActive();
$page->smarty->assign('grps', $grps);
$genres = $gen->getGenres('', true);
$page->smarty->assign('genres', $genres);
header("Content-type: text/xml");
$output = $page->smarty->fetch('apicaps.tpl');
if ($outputtype == "xml") {
header("Content-type: text/xml");
echo $output;
} else {
header('Content-type: application/json');
echo json_encode(responseXmlToObject($output));
}
break;
//
// register request
<?php
require "config.php";
require_once WWW_DIR . "/lib/groups.php";
require_once WWW_DIR . "/lib/binaries.php";
require_once WWW_DIR . '/lib/powerprocess.php';
$groups = new Groups();
$groupList = $groups->getActive();
unset($groups);
$ps = new PowerProcess();
$ps->RegisterCallback('psUpdateComplete');
$ps->maxThreads = 10;
$ps->tickCount = 10000;
// value in usecs. change this to 1000000 (one second) to reduce cpu use
$ps->threadTimeLimit = 0;
// Disable child timeout
echo "Starting threaded backfill process\n";
while ($ps->RunControlCode()) {
// Start the parent loop
if (count($groupList)) {
// We still have groups to process
if ($ps->SpawnReady()) {
// Spawn another thread
$ps->threadData = array_pop($groupList);
echo "[Thread-MASTER] Spawning new thread. Still have " . count($groupList) . " group(s) to update after this\n";
$ps->spawnThread();
} else {
// There are no more slots available to run
//$ps->tick();
//echo ".\n";
}
<?php
require_once dirname(__FILE__) . '/../../../../www/config.php';
if (!isset($argv[1])) {
exit('You must start the script like this (# of articles) : php test-backfillcleansubject.php 20000' . "\n");
} else {
$pdo = new nzedb\db\Settings();
$nntp = new NNTP(['Settings' => $pdo]);
if ($nntp->doConnect() !== true) {
exit($pdo->log->error("Unable to connect to usenet."));
}
$backfill = new Backfill(['NNTP' => $nntp, 'Settings' => $pdo]);
$groups = new Groups(['Settings' => $pdo]);
$grouplist = $groups->getActive();
foreach ($grouplist as $name) {
dogroup($name["name"], $argv[1]);
}
}
function dogroup($name, $articles)
{
global $backfill, $pdo;
$backfill->backfillAllGroups($name, $articles);
echo $pdo->log->primaryOver("Type y and press enter to continue, n to quit.\n");
if (trim(fgets(fopen("php://stdin", "r"))) == 'y') {
return true;
} else {
exit($pdo->log->primary("Done"));
}
}
function processReleases()
{
require_once WWW_DIR . "/lib/binaries.php";
$db = new DB();
$currTime_ori = $db->queryOneRow("SELECT NOW() as now");
$cat = new Category();
$nzb = new Nzb();
$s = new Sites();
$releaseRegex = new ReleaseRegex();
$page = new Page();
$groups = new Groups();
$retcount = 0;
echo $s->getLicense();
echo "\n\nStarting release update process (" . date("Y-m-d H:i:s") . ")\n";
if (!file_exists($page->site->nzbpath)) {
echo "Bad or missing nzb directory - " . $page->site->nzbpath;
return -1;
}
//
// Get all regexes for all groups which are to be applied to new binaries
// in order of how they should be applied
//
$releaseRegex->get();
echo "Stage 1 : Applying regex to binaries\n";
$activeCategories = $cat->get();
$catbasedsizes = $db->getLookupAsArray($activeCategories, "ID");
$activeGroups = $groups->getActive(false);
$groupbasedminsizes = $db->getLookupAsArray($groups->getAllNoReleases(), "ID");
foreach ($activeGroups as $groupArr) {
//check if regexes have already been applied during update binaries
if ($groupArr['regexmatchonly'] == 1) {
continue;
}
$groupRegexes = $releaseRegex->getForGroup($groupArr['name']);
echo "Stage 1 : Applying " . sizeof($groupRegexes) . " regexes to group " . $groupArr['name'] . "\n";
// Get out all binaries of STAGE0 for current group
$newUnmatchedBinaries = array();
$ressql = sprintf("SELECT binaries.ID, binaries.name, binaries.date, binaries.totalParts, binaries.procstat, binaries.fromname from binaries where groupID = %d and procstat IN (%d,%d) and regexID IS NULL order by binaries.date asc", $groupArr['ID'], Releases::PROCSTAT_NEW, Releases::PROCSTAT_TITLENOTMATCHED);
$resbin = $db->queryDirect($ressql);
$matchedbins = 0;
while ($rowbin = $db->getAssocArray($resbin)) {
$regexMatches = array();
foreach ($groupRegexes as $groupRegex) {
$regexCheck = $releaseRegex->performMatch($groupRegex, $rowbin['name']);
if ($regexCheck !== false) {
$regexMatches = $regexCheck;
break;
}
}
if (!empty($regexMatches)) {
$matchedbins++;
$relparts = explode("/", $regexMatches['parts']);
$db->exec(sprintf("update binaries set relname = replace(%s, '_', ' '), relpart = %d, reltotalpart = %d, procstat=%d, categoryID=%s, regexID=%d, reqID=%s where ID = %d", $db->escapeString($regexMatches['name']), $relparts[0], $relparts[1], Releases::PROCSTAT_TITLEMATCHED, $regexMatches['regcatid'], $regexMatches['regexID'], $db->escapeString($regexMatches['reqID']), $rowbin["ID"]));
} else {
if ($rowbin['procstat'] == Releases::PROCSTAT_NEW) {
$newUnmatchedBinaries[] = $rowbin['ID'];
}
}
}
//mark as not matched
if (!empty($newUnmatchedBinaries)) {
$db->exec(sprintf("update binaries set procstat=%d where ID IN (%s)", Releases::PROCSTAT_TITLENOTMATCHED, implode(',', $newUnmatchedBinaries)));
}
}
//
// Move all binaries from releases which have the correct number of files on to the next stage.
//
echo "Stage 2 : Marking binaries where all parts are available";
$result = $db->queryDirect(sprintf("SELECT relname, date, SUM(reltotalpart) AS reltotalpart, groupID, reqID, fromname, SUM(num) AS num, coalesce(g.minfilestoformrelease, s.minfilestoformrelease) as minfilestoformrelease FROM ( SELECT relname, reltotalpart, groupID, reqID, fromname, max(date) as date, COUNT(ID) AS num FROM binaries WHERE procstat = %s GROUP BY relname, reltotalpart, groupID, reqID, fromname ORDER BY NULL ) x left outer join groups g on g.ID = x.groupID inner join ( select value as minfilestoformrelease from site where setting = 'minfilestoformrelease' ) s GROUP BY relname, groupID, reqID, fromname, minfilestoformrelease ORDER BY NULL", Releases::PROCSTAT_TITLEMATCHED));
while ($row = $db->getAssocArray($result)) {
$retcount++;
//
// Less than the site permitted number of files in a release. Dont discard it, as it may
// be part of a set being uploaded.
//
if ($row["num"] < $row["minfilestoformrelease"]) {
//echo "Number of files in release ".$row["relname"]." less than site/group setting (".$row['num']."/".$row["minfilestoformrelease"].")\n";
//$db->exec(sprintf("update binaries set procattempts = procattempts + 1 where relname = %s and procstat = %d and groupID = %d and fromname = %s", $db->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $db->escapeString($row["fromname"]) ));
} elseif ($row["num"] >= $row["reltotalpart"]) {
$incomplete = false;
if ($row['reltotalpart'] == 0 && strtotime($currTime_ori['now']) - strtotime($row['date']) < 14400) {
$incomplete = true;
} else {
// Check that the binary is complete
$binlist = $db->query(sprintf("SELECT binaries.ID, totalParts, date, COUNT(DISTINCT parts.messageID) AS num FROM binaries, parts WHERE binaries.ID=parts.binaryID AND binaries.relname = %s AND binaries.procstat = %d AND binaries.groupID = %d AND binaries.fromname = %s GROUP BY binaries.ID ORDER BY NULL", $db->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $db->escapeString($row["fromname"])));
foreach ($binlist as $rowbin) {
if ($rowbin['num'] < $rowbin['totalParts']) {
// Allow to binary to release if posted to usenet longer than four hours ago and we still don't have all the parts
if (!(strtotime($currTime_ori['now']) - strtotime($rowbin['date']) > 14400)) {
$incomplete = true;
break;
}
}
}
}
if (!$incomplete) {
//
// Right number of files, but see if the binary is a allfilled/reqid post, in which case it needs its name looked up
//
if ($row['reqID'] != '' && $page->site->reqidurl != "") {
//
// Try and get the name using the group
//
$binGroup = $db->queryOneRow(sprintf("SELECT name FROM groups WHERE ID = %d", $row["groupID"]));
$newtitle = $this->getReleaseNameForReqId($page->site->reqidurl, $page->site->newznabID, $binGroup["name"], $row["reqID"]);
//
// if the feed/group wasnt supported by the scraper, then just use the release name as the title.
//
if ($newtitle == "no feed") {
$newtitle = $row["relname"];
}
//
// Valid release with right number of files and title now, so move it on
//
if ($newtitle != "") {
$db->exec(sprintf("update binaries set relname = %s, procstat=%d where relname = %s and procstat = %d and groupID = %d and fromname=%s", $db->escapeString($newtitle), Releases::PROCSTAT_READYTORELEASE, $db->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $db->escapeString($row["fromname"])));
} else {
//
// Item not found, if the binary was added to the index yages ago, then give up.
//
$maxaddeddate = $db->queryOneRow(sprintf("SELECT NOW() as now, MAX(dateadded) as dateadded FROM binaries WHERE relname = %s and procstat = %d and groupID = %d and fromname=%s", $db->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $db->escapeString($row["fromname"])));
//
// If added to the index over 48 hours ago, give up trying to determine the title
//
if (strtotime($maxaddeddate['now']) - strtotime($maxaddeddate['dateadded']) > 60 * 60 * 48) {
$db->exec(sprintf("update binaries set procstat=%d where relname = %s and procstat = %d and groupID = %d and fromname=%s", Releases::PROCSTAT_NOREQIDNAMELOOKUPFOUND, $db->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $db->escapeString($row["fromname"])));
}
}
} else {
$db->exec(sprintf("update binaries set procstat=%d where relname = %s and procstat = %d and groupID = %d and fromname=%s", Releases::PROCSTAT_READYTORELEASE, $db->escapeString($row["relname"]), Releases::PROCSTAT_TITLEMATCHED, $row["groupID"], $db->escapeString($row["fromname"])));
}
}
}
if ($retcount % 100 == 0) {
echo ".";
}
}
$retcount = 0;
echo "\nStage 3 : Creating releases from complete binaries\n";
//
// Get out all distinct relname, group from binaries of STAGE2
//
$result = $db->queryDirect(sprintf("SELECT relname, groupID, g.name as group_name, fromname, max(categoryID) as categoryID, max(regexID) as regexID, max(reqID) as reqID, MAX(date) as date, count(binaries.ID) as parts, coalesce(sum(binaries.size),0) as size from binaries inner join groups g on g.ID = binaries.groupID where procstat = %d and relname is not null group by relname, g.name, groupID, fromname ORDER BY COUNT(binaries.ID) desc", Releases::PROCSTAT_READYTORELEASE));
while ($row = $db->getAssocArray($result)) {
$relguid = md5(uniqid());
//
// Get categoryID if one has been allocated to this
//
if ($row["categoryID"] != "") {
$catId = $row["categoryID"];
} else {
$catId = $cat->determineCategory($row["group_name"], $row["relname"]);
}
//
// Determine if size matches permitted boundaries and discard here if not.
//
$gsize = $groupbasedminsizes[$row["groupID"]][0]["minsizetoformrelease"];
if ($gsize == "" || $gsize == 0) {
$gsize = $row["size"];
}
$ssize = $page->site->minsizetoformrelease;
if ($ssize == "" || $ssize == 0) {
$ssize = $row["size"];
}
$csize = $catbasedsizes[$catId][0]["minsizetoformrelease"];
if ($csize == "" || $csize == 0) {
$csize = $row["size"];
}
$cpsize = $catbasedsizes[$catId][0]["parentminsizetoformrelease"];
if ($cpsize == "" || $cpsize == 0) {
$cpsize = $row["size"];
}
$cmaxsize = $catbasedsizes[$catId][0]["maxsizetoformrelease"];
if ($cmaxsize == "" || $cmaxsize == 0) {
$cmaxsize = $row["size"];
}
$cpmaxsize = $catbasedsizes[$catId][0]["parentmaxsizetoformrelease"];
if ($cpmaxsize == "" || $cpmaxsize == 0) {
$cpmaxsize = $row["size"];
}
$overallminsize = max($gsize, $ssize, $csize, $cpsize);
$overallmaxsize = min($cmaxsize, $cpmaxsize);
if ($row["size"] < $overallminsize || $row["size"] > $overallmaxsize) {
echo sprintf("Stage 3 : Discarding - %s (Size %s outside permitted range of %s%s)\n", $row["relname"], formatBytes($row["size"]), $overallminsize != $row["size"] ? formatBytes($overallminsize) : "", $overallmaxsize != $row["size"] ? formatBytes($overallmaxsize) : "");
$db->exec(sprintf("DELETE parts, binaries FROM parts JOIN binaries ON binaries.ID = parts.binaryID WHERE relname=%s and groupID=%d and fromname=%s and procstat=%d", $db->escapeString($row["relname"]), $row["groupID"], $db->escapeString($row["fromname"]), Releases::PROCSTAT_READYTORELEASE));
} else {
// Clean release name
$cleanRelName = $this->cleanReleaseName($row['relname']);
$relid = $this->insertRelease($cleanRelName, $row["parts"], $row["groupID"], $relguid, $catId, $row["regexID"], $row["date"], $row["fromname"], $row["reqID"], $page->site);
//
// Tag every binary for this release with its parent release id
//
$db->exec(sprintf("update binaries set procstat = %d, releaseID = %d where relname = %s and procstat = %d and groupID = %d and fromname=%s", Releases::PROCSTAT_RELEASED, $relid, $db->escapeString($row["relname"]), Releases::PROCSTAT_READYTORELEASE, $row["groupID"], $db->escapeString($row["fromname"])));
//
// Write the nzb to disk
//
$nzbfile = $nzb->getNZBPath($relguid, $page->site->nzbpath, true);
$nzb->writeNZBforReleaseId($relid, $cleanRelName, $catId, $nzbfile);
//
// Remove used binaries
//
$db->exec(sprintf("DELETE parts, binaries FROM parts JOIN binaries ON binaries.ID = parts.binaryID WHERE releaseID = %d ", $relid));
//
// If nzb successfully written, then load it and get size completion from it
//
$nzbInfo = new nzbInfo();
if (!$nzbInfo->loadFromFile($nzbfile)) {
echo "Stage 3 : Failed to write nzb file (bad perms?) " . $nzbfile . "\n";
//copy($nzbfile, "./ERRORNZB_".$relguid);
$this->delete($relid);
} else {
// Check if gid already exists
$dupes = $db->queryOneRow(sprintf("SELECT EXISTS(SELECT 1 FROM releases WHERE gid = %s) as total", $db->escapeString($nzbInfo->gid)));
if ($dupes['total'] > 0) {
echo "Stage 3 : Duplicate - " . $cleanRelName . " -" . $nzbInfo->gid . "-\n";
$this->delete($relid);
} else {
$db->exec(sprintf("update releases set totalpart = %d, size = %s, completion = %d, GID=%s where ID = %d", $nzbInfo->filecount, $nzbInfo->filesize, $nzbInfo->completion, $db->escapeString($nzbInfo->gid), $relid));
echo "Stage 3 : Added release " . $cleanRelName . "\n";
//Increment new release count
$retcount++;
}
}
}
}
//
// Delete any releases under the minimum completion percent.
//
if ($page->site->completionpercent != 0) {
echo "Stage 4 : Deleting releases less than " . $page->site->completionpercent . " complete\n";
$result = $db->query(sprintf("select ID from releases where completion > 0 and completion < %d", $page->site->completionpercent));
foreach ($result as $row) {
$this->delete($row["ID"]);
}
}
/*
*Potentially leave this in to mop up release when the cat sizes change.
//
// Delete releases whos minsize is less than the site or group minimum
//
$result = $db->query("select releases.ID from releases left outer join (SELECT g.ID, coalesce(g.minsizetoformrelease, s.minsizetoformrelease) as minsizetoformrelease FROM groups g inner join ( select value as minsizetoformrelease from site where setting = 'minsizetoformrelease' ) s ) x on x.ID = releases.groupID where minsizetoformrelease != 0 and releases.size < minsizetoformrelease");
if (count($result) > 0)
{
echo "Stage 4 : Deleting ".count($result)." release(s) where size is smaller than minsize for site/group\n";
foreach ($result as $row)
$this->delete($row["ID"]);
}
$result = $db->query("select releases.ID, name, categoryID, size FROM releases JOIN (
select
catc.ID,
case when catc.minsizetoformrelease = 0 then catp.minsizetoformrelease else catc.minsizetoformrelease end as minsizetoformrelease,
case when catc.maxsizetoformrelease = 0 then catp.maxsizetoformrelease else catc.maxsizetoformrelease end as maxsizetoformrelease
from category catp join category catc on catc.parentID = catp.ID
where (catc.minsizetoformrelease != 0 or catc.maxsizetoformrelease != 0) or (catp.minsizetoformrelease != 0 or catp.maxsizetoformrelease != 0)
) x on x.ID = releases.categoryID
where
(size < minsizetoformrelease and minsizetoformrelease != 0) or
(size > maxsizetoformrelease and maxsizetoformrelease != 0)");
if(count($result) > 0)
{
echo "Stage 4 : Deleting release(s) not matching category min/max size ...\n";
foreach ($result as $r){
$this->delete($r['ID']);
}
}
*/
echo "Stage 5 : Post processing started\n";
$postprocess = new PostProcess(true);
$postprocess->processAll();
//
// aggregate the releasefiles upto the releases.
//
echo "Stage 6 : Aggregating Files\n";
$db->exec("update releases INNER JOIN (SELECT releaseID, COUNT(ID) AS num FROM releasefiles GROUP BY releaseID) b ON b.releaseID = releases.ID and releases.rarinnerfilecount = 0 SET rarinnerfilecount = b.num");
// Remove the binaries and parts used to form releases, or that are duplicates.
//
if ($page->site->partsdeletechunks > 0) {
echo "Stage 7 : Chunk deleting unused binaries and parts";
$query = sprintf("SELECT parts.ID as partsID,binaries.ID as binariesID FROM parts\r\n\t\t\t\t\t\tLEFT JOIN binaries ON binaries.ID = parts.binaryID\r\n\t\t\t\t\t\tWHERE binaries.dateadded < %s - INTERVAL %d HOUR LIMIT 0,%d", $db->escapeString($currTime_ori["now"]), ceil($page->site->rawretentiondays * 24), $page->site->partsdeletechunks);
$cc = 0;
$done = false;
while (!$done) {
$dd = $cc;
$result = $db->query($query);
if (count($result) > 0) {
$pID = array();
$bID = array();
foreach ($result as $row) {
$pID[] = $row['partsID'];
$bID[] = $row['binariesID'];
}
$pID = '(' . implode(',', $pID) . ')';
$bID = '(' . implode(',', $bID) . ')';
$fr = $db->exec("DELETE FROM parts WHERE ID IN {$pID}");
if ($fr > 0) {
$cc += $fr;
$cc += $db->exec("DELETE FROM binaries WHERE ID IN {$bID}");
}
unset($pID);
unset($bID);
if ($cc == $dd) {
$done = true;
}
echo $cc % 10000 ? '.' : '';
} else {
$done = true;
}
}
echo "\nStage 7 : Complete - " . $cc . " rows affected\n";
} else {
echo "Stage 7 : Deleting unused binaries and parts\n";
$db->exec(sprintf("DELETE parts, binaries FROM parts JOIN binaries ON binaries.ID = parts.binaryID\r\n\t\t\tWHERE binaries.dateadded < %s - INTERVAL %d HOUR", $db->escapeString($currTime_ori["now"]), ceil($page->site->rawretentiondays * 24)));
}
//
// User/Request housekeeping, should ideally move this to its own section, but it needs to be done automatically.
//
$users = new Users();
$users->pruneRequestHistory($page->site->userdownloadpurgedays);
echo "Done : Added " . $retcount . " releases\n\n";
return $retcount;
}
php convert_to_tpg.php true Convert c/b/p to tpg leaving current binaries/parts tables in-tact.
php convert_to_tgp.php true delete Convert c/b/p to tpg and TRUNCATE current binaries/parts tables.
*/
$debug = false;
$pdo = new Settings();
$groups = new \Groups(['Settings' => $pdo]);
$consoletools = new \ConsoleTools(['ColorCLI' => $pdo->log]);
$DoPartRepair = $pdo->getSetting('partrepair') == '0' ? false : true;
if (!isset($argv[1]) || $argv[1] != 'true') {
exit($pdo->log->error("\nMandatory argument missing\n\n" . "This script will allow you to move from single binaries/parts tables to TPG without having to run reset_truncate.\n" . "Please STOP all update scripts before running this script.\n\n" . "Use the following options to run:\n" . "php {$argv['0']} true ...: Convert b/p to tpg leaving current binaries/parts tables in-tact.\n" . "php {$argv['0']} true delete ...: Convert b/p to tpg and TRUNCATE current binaries/parts tables.\n"));
}
$blen = $pdo->queryOneRow('SELECT COUNT(*) AS total FROM binaries;');
$bdone = 0;
$bcount = 1;
$gdone = 1;
$actgroups = $groups->getActive();
$glen = count($actgroups);
$newtables = $glen * 3;
$begintime = time();
echo "Creating new binaries, and parts tables for each active group...\n";
foreach ($actgroups as $group) {
if ($groups->createNewTPGTables($group['id']) === false) {
exit($pdo->log->error("There is a problem creating new parts/files tables for group {$group['name']}."));
}
$consoletools->overWrite("Tables Created: " . $consoletools->percentString($gdone * 3, $newtables));
$gdone++;
}
$endtime = time();
echo "\nTable creation took " . $consoletools->convertTime($endtime - $begintime) . ".\n";
$starttime = time();
echo "\nNew tables created, moving data from old tables to new tables.\nThis will take awhile....\n\n";
