/** * Insert the NZB details into the database. * * @param $nzbDetails * * @return bool * * @access protected */ protected function insertNZB($nzbDetails) { // Make up a GUID for the release. $this->relGuid = $this->releases->createGUID(); // Remove part count from subject. $partLess = preg_replace('/(\\(\\d+\\/\\d+\\))*$/', 'yEnc', $nzbDetails['subject']); // Remove added yEnc from above and anything after. $subject = utf8_encode(trim(preg_replace('/yEnc.*$/i', 'yEnc', $partLess))); $renamed = 0; if ($nzbDetails['useFName']) { // If the user wants to use the file name.. use it. $cleanName = $nzbDetails['useFName']; $renamed = 1; } else { // Pass the subject through release cleaner to get a nicer name. $cleanName = $this->releaseCleaner->releaseCleaner($subject, $nzbDetails['from'], $nzbDetails['totalSize'], $nzbDetails['groupName']); if (isset($cleanName['properlynamed'])) { $cleanName = $cleanName['cleansubject']; $renamed = isset($cleanName['properlynamed']) && $cleanName['properlynamed'] === true ? 1 : 0; } } $escapedSubject = $this->pdo->escapeString($subject); $escapedFromName = $this->pdo->escapeString($nzbDetails['from']); // Look for a duplicate on name, poster and size. $dupeCheck = $this->pdo->queryOneRow(sprintf('SELECT id FROM releases WHERE name = %s AND fromname = %s AND size BETWEEN %s AND %s', $escapedSubject, $escapedFromName, $this->pdo->escapeString($nzbDetails['totalSize'] * 0.99), $this->pdo->escapeString($nzbDetails['totalSize'] * 1.01))); if ($dupeCheck === false) { $escapedSearchName = $this->pdo->escapeString($cleanName); // Insert the release into the DB. $relID = $this->releases->insertRelease(['name' => $escapedSubject, 'searchname' => $escapedSearchName, 'totalpart' => $nzbDetails['totalFiles'], 'groupid' => $nzbDetails['groupid'], 'guid' => $this->pdo->escapeString($this->relGuid), 'postdate' => $this->pdo->escapeString($nzbDetails['postDate']), 'fromname' => $escapedFromName, 'size' => $this->pdo->escapeString($nzbDetails['totalSize']), 'categoryid' => $this->category->determineCategory($nzbDetails['groupid'], $cleanName), 'isrenamed' => $renamed, 'reqidstatus' => 0, 'prehashid' => 0, 'nzbstatus' => NZB::NZB_ADDED]); } else { //$this->echoOut('This release is already in our DB so skipping: ' . $subject); return false; } if (isset($relID) && $relID === false) { $this->echoOut('ERROR: Problem inserting: ' . $subject); return false; } return true; }
/** * Create releases from complete binaries. * * @param int|string $groupID (optional) * * @return int * @access public */ public function createReleases($groupID) { $startTime = time(); $group = $this->groups->getCBPTableNames($this->tablePerGroup, $groupID); $page = new Page(); $this->pdo->log->doEcho($this->pdo->log->primary('Creating releases from complete binaries')); $this->pdo->ping(true); // // Get out all distinct relname, group from binaries // $categorize = new \Categorize(['Settings' => $this->pdo]); $returnCount = $duplicate = 0; $result = $this->pdo->queryDirect(sprintf("SELECT %s.*, g.name AS group_name, count(%s.id) AS parts FROM %s INNER JOIN groups g ON g.id = %s.groupid WHERE %s procstat = %d AND relname IS NOT NULL GROUP BY relname, g.name, groupid, fromname ORDER BY COUNT(%s.id) DESC LIMIT %d", $group['bname'], $group['bname'], $group['bname'], $group['bname'], !empty($groupID) ? ' groupid = ' . $groupID . ' AND ' : ' ', Releases::PROCSTAT_READYTORELEASE, $group['bname'], $this->releaseCreationLimit)); while ($row = $this->pdo->getAssocArray($result)) { $relguid = $this->createGUID(); // Clean release name $releaseCleaning = new ReleaseCleaning(); $cleanRelName = $this->cleanReleaseName($row['relname']); $cleanedName = $releaseCleaning->releaseCleaner($row['relname'], $row['fromname'], $row['group_name']); if (is_array($cleanedName)) { $properName = $cleanedName['properlynamed']; $prehashID = isset($cleanerName['predb']) ? $cleanerName['predb'] : false; $isReqID = isset($cleanerName['requestid']) ? $cleanerName['requestid'] : false; $cleanedName = $cleanedName['cleansubject']; } else { $properName = true; $isReqID = $prehashID = false; } if ($prehashID === false && $cleanedName !== '') { // try to match the cleaned searchname to predb title or filename here $preHash = new PreHash(); $preMatch = $preHash->matchPre($cleanedName); if ($preMatch !== false) { $cleanedName = $preMatch['title']; $prehashID = $preMatch['prehashid']; $properName = true; } } $relid = $this->insertRelease(['name' => $this->pdo->escapeString($cleanRelName), 'searchname' => $this->pdo->escapeString(utf8_encode($cleanedName)), 'totalpart' => $row["parts"], 'groupid' => $row["groupid"], 'guid' => $this->pdo->escapeString($relguid), 'categoryid' => $categorize->determineCategory($groupID, $cleanedName), 'regexid' => $row["regexid"], 'postdate' => $this->pdo->escapeString($row['date']), 'fromname' => $this->pdo->escapeString($row['fromname']), 'reqid' => $row["reqid"], 'passwordstatus' => $page->settings->getSetting('checkpasswordedrar') > 0 ? -1 : 0, 'nzbstatus' => \Enzebe::NZB_NONE, 'isrenamed' => $properName === true ? 1 : 0, 'reqidstatus' => $isReqID === true ? 1 : 0, 'prehashid' => $prehashID === false ? 0 : $prehashID]); // // Tag every binary for this release with its parent release id // $this->pdo->queryExec(sprintf("UPDATE %s SET procstat = %d, releaseid = %d WHERE relname = %s AND procstat = %d AND %s fromname=%s", $group['bname'], Releases::PROCSTAT_RELEASED, $relid, $this->pdo->escapeString($row["relname"]), Releases::PROCSTAT_READYTORELEASE, !empty($groupID) ? ' groupid = ' . $groupID . ' AND ' : ' ', $this->pdo->escapeString($row["fromname"]))); $cat = new \Categorize(['Settings' => $this->pdo]); // // Write the nzb to disk // $catId = $cat->determineCategory($groupID, $cleanRelName); $nzbfile = $this->nzb->getNZBPath($relguid, $page->settings->getSetting('nzbpath'), true); $this->nzb->writeNZBforreleaseID($relid, $cleanRelName, $catId, $nzbfile, $groupID); // // Remove used binaries // $this->pdo->queryDelete(sprintf("DELETE %s, %s FROM %s JOIN %s ON %s.id = %s.binaryid WHERE releaseid = %d ", $group['pname'], $group['bname'], $group['pname'], $group['bname'], $group['bname'], $group['pname'], $relid)); // // If nzb successfully written, then load it and get size completion from it // $nzbInfo = new NZBInfo(); if (!$nzbInfo->loadFromFile($nzbfile)) { $this->pdo->log->doEcho($this->pdo->log->primary('Failed to write nzb file (bad perms?) ' . $nzbfile . '')); $this->delete($relid); } else { // Check if gid already exists $dupes = $this->pdo->queryOneRow(sprintf("SELECT EXISTS(SELECT 1 FROM releases WHERE gid = %s) AS total", $this->pdo->escapeString($nzbInfo->gid))); if ($dupes['total'] > 0) { $this->pdo->log->doEcho($this->pdo->log->primary('Duplicate - ' . $cleanRelName . '')); $this->delete($relid); $duplicate++; } else { $this->pdo->queryExec(sprintf("UPDATE releases SET totalpart = %d, size = %s, COMPLETION = %d, GID=%s , nzb_guid = %s WHERE id = %d", $nzbInfo->filecount, $nzbInfo->filesize, $nzbInfo->completion, $this->pdo->escapeString($nzbInfo->gid), $this->pdo->escapeString($nzbInfo->gid), $relid)); $this->pdo->log->doEcho($this->pdo->log->primary('Added release ' . $cleanRelName . '')); $returnCount++; if ($this->echoCLI) { $this->pdo->log->doEcho($this->pdo->log->primary('Added ' . $returnCount . 'releases.')); } } } } if ($this->echoCLI) { $this->pdo->log->doEcho($this->pdo->log->primary(PHP_EOL . number_format($returnCount) . ' Releases added and ' . number_format($duplicate) . ' duplicate releases deleted in ' . $this->consoleTools->convertTime(time() - $startTime)), true); } return ['added' => $returnCount, 'dupes' => $duplicate]; }
function releaseCleaner($subject, $fromName, $size, $groupname, $usepre) { $groups = new \Groups(); $releaseCleaning = new \ReleaseCleaning($groups->pdo); $cleanerName = $releaseCleaning->releaseCleaner($subject, $fromName, $size, $groupname, $usepre); if (!is_array($cleanerName) && $cleanerName != false) { return array("cleansubject" => $cleanerName, "properlynamed" => true, "increment" => false); } else { return $cleanerName; } }
echo $pdo->log->header("Finished recreating search names / recategorizing / refixing names in " . $timetotal); } else { exit($pdo->log->info("You have no releases in the DB.")); } } else { if (isset($argv[1]) && $argv[1] == "reset") { $pdo = new Settings(); $res = $pdo->query("SELECT releases.id, releases.name, releases.fromname, releases.size, groups.name AS gname FROM releases INNER JOIN groups ON releases.group_id = groups.id"); if (count($res) > 0) { echo $pdo->log->header("Going to reset search names, this can take a while."); $done = 0; $timestart = time(); $consoletools = new ConsoleTools(['ColorCLI' => $pdo->log]); foreach ($res as $row) { $rc = new ReleaseCleaning($pdo); $newname = $rc->releaseCleaner($row['name'], $row['fromname'], $row['size'], $row['gname']); if (is_array($newname)) { $newname = $newname['cleansubject']; } $newname = $pdo->escapeString($newname); $pdo->queryExec(sprintf("UPDATE releases SET searchname = %s where id = %d", $newname, $row['id'])); $sphinx->updateReleaseSearchName($newname, $row['id']); $done++; $consoletools->overWritePrimary("Renaming:" . $consoletools->percentString($done, count($res))); } $timenc = $consoletools->convertTime(TIME() - $timestart); echo $pdo->log->header($done . " releases renamed in " . $timenc); } } else { exit($pdo->log->error("\nThis script runs the subject names through namecleaner to create a clean search name, it also recategorizes and runs the releases through namefixer.\n" . "php resetSearchname.php full ...: To run this, recategorize and refix release names on all releases.\n" . "php resetSearchname.php limited ...: To run this on releases that have not had their names fixed, then categorizing them.\n" . "php resetSearchname.php reset ...: To just reset searchnames.\n")); }
$nzbSkipped++; } $importfailed = true; break; } } if ($importfailed) { @unlink($nzbFile); } else { $relguid = $releases->createGUID(); $propername = true; $relid = false; if ($usenzbname === true) { $cleanerName = $usename; } else { $cleanerName = $releasecleaning->releaseCleaner($subject, $fromname, $totalsize, $groupName); } if (!is_array($cleanerName)) { $cleanName = $cleanerName; } else { $cleanName = $cleanerName['cleansubject']; $propername = $cleanerName['properlynamed']; } if (empty($postername[0])) { $poster = ''; } else { $poster = $postername[0]; } if (empty($postdate[0])) { $posteddate = $date = date("Y-m-d H:i:s"); } else {
/** * Create releases from complete collections. * * @param int|string $groupID (optional) * * @return int * @access public */ public function createReleases($groupID) { $startTime = time(); $group = $this->groups->getCBPTableNames($this->tablePerGroup, $groupID); $categorize = new \Categorize(['Settings' => $this->pdo]); $returnCount = $duplicate = 0; if ($this->echoCLI) { $this->pdo->log->doEcho($this->pdo->log->header("Process Releases -> Create releases from complete collections.")); } $this->pdo->ping(true); $collections = $this->pdo->queryDirect(sprintf(' SELECT %s.*, groups.name AS gname FROM %s INNER JOIN groups ON %s.group_id = groups.id WHERE %s %s.filecheck = %d AND filesize > 0 LIMIT %d', $group['cname'], $group['cname'], $group['cname'], !empty($groupID) ? ' group_id = ' . $groupID . ' AND ' : ' ', $group['cname'], self::COLLFC_SIZED, $this->releaseCreationLimit)); if ($this->echoCLI && $collections !== false) { echo $this->pdo->log->primary($collections->rowCount() . " Collections ready to be converted to releases."); } if ($collections instanceof \Traversable) { $preDB = new \PreDb(['Echo' => $this->echoCLI, 'Settings' => $this->pdo]); foreach ($collections as $collection) { $cleanRelName = $this->pdo->escapeString(utf8_encode(str_replace(['#', '@', '$', '%', '^', '§', '¨', '©', 'Ö'], '', $collection['subject']))); $fromName = $this->pdo->escapeString(utf8_encode(trim($collection['fromname'], "'"))); // Look for duplicates, duplicates match on releases.name, releases.fromname and releases.size // A 1% variance in size is considered the same size when the subject and poster are the same $dupeCheck = $this->pdo->queryOneRow(sprintf("\n\t\t\t\t\t\tSELECT id\n\t\t\t\t\t\tFROM releases\n\t\t\t\t\t\tWHERE name = %s\n\t\t\t\t\t\tAND fromname = %s\n\t\t\t\t\t\tAND size BETWEEN '%s'\n\t\t\t\t\t\tAND '%s'", $cleanRelName, $fromName, $collection['filesize'] * 0.99, $collection['filesize'] * 1.01)); if ($dupeCheck === false) { $cleanedName = $this->releaseCleaning->releaseCleaner($collection['subject'], $collection['fromname'], $collection['filesize'], $collection['gname']); if (is_array($cleanedName)) { $properName = $cleanedName['properlynamed']; $preID = isset($cleanerName['predb']) ? $cleanerName['predb'] : false; $isReqID = isset($cleanerName['requestid']) ? $cleanerName['requestid'] : false; $cleanedName = $cleanedName['cleansubject']; } else { $properName = true; $isReqID = $preID = false; } if ($preID === false && $cleanedName !== '') { // try to match the cleaned searchname to predb title or filename here $preMatch = $preDB->matchPre($cleanedName); if ($preMatch !== false) { $cleanedName = $preMatch['title']; $preID = $preMatch['preid']; $properName = true; } } $releaseID = $this->releases->insertRelease(['name' => $cleanRelName, 'searchname' => $this->pdo->escapeString(utf8_encode($cleanedName)), 'totalpart' => $collection['totalfiles'], 'group_id' => $collection['group_id'], 'guid' => $this->pdo->escapeString($this->releases->createGUID($cleanRelName)), 'postdate' => $this->pdo->escapeString($collection['date']), 'fromname' => $fromName, 'size' => $collection['filesize'], 'categoryid' => $categorize->determineCategory($cleanedName, $collection['group_id']), 'isrenamed' => $properName === true ? 1 : 0, 'reqidstatus' => $isReqID === true ? 1 : 0, 'preid' => $preID === false ? 0 : $preID, 'nzbstatus' => \NZB::NZB_NONE]); if ($releaseID !== false) { // Update collections table to say we inserted the release. $this->pdo->queryExec(sprintf(' UPDATE %s SET filecheck = %d, releaseid = %d WHERE id = %d', $group['cname'], self::COLLFC_INSERTED, $releaseID, $collection['id'])); $returnCount++; if ($this->echoCLI) { echo "Added {$returnCount} releases.\r"; } } } else { // The release was already in the DB, so delete the collection. $this->pdo->queryExec(sprintf(' DELETE FROM %s WHERE collectionhash = %s', $group['cname'], $this->pdo->escapeString($collection['collectionhash']))); $duplicate++; } } } if ($this->echoCLI) { $this->pdo->log->doEcho($this->pdo->log->primary(PHP_EOL . number_format($returnCount) . ' Releases added and ' . number_format($duplicate) . ' duplicate collections deleted in ' . $this->consoleTools->convertTime(time() - $startTime)), true); } return $returnCount; }
$rename = false; if ($argv[3] === 'true') { $rename = true; } require_once dirname(__FILE__) . '/../../../www/config.php'; $pdo = new Settings(); $group = $pdo->queryOneRow(sprintf('SELECT id FROM groups WHERE name = %s', $pdo->escapeString($argv[1]))); if ($group === false) { exit('No group with name ' . $argv[1] . ' found in the database.'); } $releases = $pdo->query(sprintf('SELECT name, searchname, fromname, size, id FROM releases WHERE group_id = %d %s ORDER BY postdate LIMIT %d', $group['id'], $category, $argv[2])); if (count($releases) === 0) { exit('No releases found in your database for group ' . $argv[1] . PHP_EOL); } $RC = new ReleaseCleaning($pdo); $sphinx = new SphinxSearch(); foreach ($releases as $release) { $newName = $RC->releaseCleaner($release['name'], $release['fromname'], $release['size'], $argv[1]); if (is_array($newName)) { $newName = $newName['cleansubject']; } if ($newName !== $release['searchname']) { echo 'Old name: ' . $release['searchname'] . PHP_EOL; echo 'New name: ' . $newName . PHP_EOL . PHP_EOL; if ($rename === true) { $newName = $pdo->escapeString($newName); $pdo->queryExec(sprintf('UPDATE releases SET searchname = %s WHERE id = %d', $newName, $release['id'])); $sphinx->updateReleaseSearchName($release['id'], $newName); } } }
<?php require_once dirname(__FILE__) . '/../../../www/config.php'; $group = $argv[1]; $cleaner = new ReleaseCleaning(); if (isset($argv[1]) && file_exists($argv[1])) { $filename = $argv[1]; $fp = fopen($filename, "r") or die("Couldn't open {$filename}"); $groups = new Groups(); $group = str_replace('.txt', '', basename($filename)); @unlink(nZEDb_RES . 'logs' . DS . 'not_yenc' . DS . $group . ".failed.regex.txt"); while (!feof($fp)) { $line = fgets($fp, 1024); if (preg_match('/^(.+yEnc).+\\(\\d+\\/\\d+\\)/', $line, $matches)) { $clean = $cleaner->releaseCleaner($matches[1], '', 0, $group); if (is_array($clean)) { file_put_contents(nZEDb_RES . 'logs' . DS . 'not_yenc' . DS . $group . '.failed.regex.txt', $line . "\n", FILE_APPEND); } /*$clean = preg_replace(array('/\d+\.mp3"/', '/\.jpg"/', '/\.nzb"/', '/\.sfv"/', '/\.srr"/', '/\.mp4"/', '/\.m4b"/', '/\.mov"/', '/\.mkv"/', '/\.avi"/', '/\.m4v"/', '/\.nfo"/', '/\.mkv"/', '/\.par2"/i', '/\.r\d+"/', '/\.rar"/i', '/\.vol\d+\+\d+"/i'), '"', $matches[1]); $clean = preg_replace(array('/\.mp3"/', '/\.sample"/', '/-sample"/', '/\.part\d+"/'), '"', $clean); $clean = preg_replace(array('/Bk \d+/', '/Book \d+/', '/\d+[.,]\d+ [MKG]B/i', '/sample-/i', '/\d+-\d+/'), '', $clean); $clean = preg_replace('/"\d\d/', '"', $clean); if (preg_match('/(.*\[)\d+(\/\d+\].*)/', $clean, $grouped)) { $clean = $grouped[1] . $grouped[2]; } if (preg_match('/(.*\()\d+(\/\d+\).*)/', $clean, $grouped)) { $clean = $grouped[1] . $grouped[2]; } if (preg_match('/(.+File )\d+( of \d+.+)/i', $clean, $grouped)) { $clean = $grouped[1] . $grouped[2]; }