/**
* Insert the NZB details into the database.
*
* @param $nzbDetails
*
* @return bool
*
* @access protected
*/
protected function insertNZB($nzbDetails)
{
// Make up a GUID for the release.
$this->relGuid = $this->releases->createGUID();
// Remove part count from subject.
$partLess = preg_replace('/(\\(\\d+\\/\\d+\\))*$/', 'yEnc', $nzbDetails['subject']);
// Remove added yEnc from above and anything after.
$subject = utf8_encode(trim(preg_replace('/yEnc.*$/i', 'yEnc', $partLess)));
$renamed = 0;
if ($nzbDetails['useFName']) {
// If the user wants to use the file name.. use it.
$cleanName = $nzbDetails['useFName'];
$renamed = 1;
} else {
// Pass the subject through release cleaner to get a nicer name.
$cleanName = $this->releaseCleaner->releaseCleaner($subject, $nzbDetails['from'], $nzbDetails['totalSize'], $nzbDetails['groupName']);
if (isset($cleanName['properlynamed'])) {
$cleanName = $cleanName['cleansubject'];
$renamed = isset($cleanName['properlynamed']) && $cleanName['properlynamed'] === true ? 1 : 0;
}
}
$escapedSubject = $this->pdo->escapeString($subject);
$escapedFromName = $this->pdo->escapeString($nzbDetails['from']);
// Look for a duplicate on name, poster and size.
$dupeCheck = $this->pdo->queryOneRow(sprintf('SELECT id FROM releases WHERE name = %s AND fromname = %s AND size BETWEEN %s AND %s', $escapedSubject, $escapedFromName, $this->pdo->escapeString($nzbDetails['totalSize'] * 0.99), $this->pdo->escapeString($nzbDetails['totalSize'] * 1.01)));
if ($dupeCheck === false) {
$escapedSearchName = $this->pdo->escapeString($cleanName);
// Insert the release into the DB.
$relID = $this->releases->insertRelease(['name' => $escapedSubject, 'searchname' => $escapedSearchName, 'totalpart' => $nzbDetails['totalFiles'], 'group_id' => $nzbDetails['group_id'], 'guid' => $this->pdo->escapeString($this->relGuid), 'postdate' => $this->pdo->escapeString($nzbDetails['postDate']), 'fromname' => $escapedFromName, 'size' => $this->pdo->escapeString($nzbDetails['totalSize']), 'categoryid' => $this->category->determineCategory($nzbDetails['group_id'], $cleanName), 'isrenamed' => $renamed, 'reqidstatus' => 0, 'preid' => 0, 'nzbstatus' => NZB::NZB_ADDED]);
} else {
//$this->echoOut('This release is already in our DB so skipping: ' . $subject);
return false;
}
if (isset($relID) && $relID === false) {
$this->echoOut('ERROR: Problem inserting: ' . $subject);
return false;
}
return true;
}
/**
* Update the release with the new information.
*
* @param array $release
* @param string $name
* @param string $method
* @param boolean $echo
* @param string $type
* @param int $nameStatus
* @param int $show
* @param int $preId
*/
public function updateRelease($release, $name, $method, $echo, $type, $nameStatus, $show, $preId = 0)
{
if ($this->relid !== $release['releaseid']) {
$releaseCleaning = new ReleaseCleaning($this->pdo);
$newName = $releaseCleaning->fixerCleaner($name);
if (strtolower($newName) != strtolower($release["searchname"])) {
$this->matched = true;
$this->relid = $release["releaseid"];
$determinedCategory = $this->category->determineCategory($release['group_id'], $newName);
if ($type === "PAR2, ") {
$newName = ucwords($newName);
if (preg_match('/(.+?)\\.[a-z0-9]{2,3}(PAR2)?$/i', $name, $match)) {
$newName = $match[1];
}
}
$this->fixed++;
$newName = explode("\\", $newName);
$newName = preg_replace(['/^[-=_\\.:\\s]+/', '/[-=_\\.:\\s]+$/'], '', $newName[0]);
if ($this->echooutput === true && $show === 1) {
$groupName = $this->_groups->getByNameByID($release['group_id']);
$oldCatName = $this->category->getNameByID($release['categoryid']);
$newCatName = $this->category->getNameByID($determinedCategory);
if ($type === "PAR2, ") {
echo PHP_EOL;
}
echo $this->pdo->log->headerOver("\nNew name: ") . $this->pdo->log->primary(substr($newName, 0, 255)) . $this->pdo->log->headerOver("Old name: ") . $this->pdo->log->primary($release["searchname"]) . $this->pdo->log->headerOver("Use name: ") . $this->pdo->log->primary($release["name"]) . $this->pdo->log->headerOver("New cat: ") . $this->pdo->log->primary($newCatName) . $this->pdo->log->headerOver("Old cat: ") . $this->pdo->log->primary($oldCatName) . $this->pdo->log->headerOver("Group: ") . $this->pdo->log->primary($groupName) . $this->pdo->log->headerOver("Method: ") . $this->pdo->log->primary($type . $method) . $this->pdo->log->headerOver("ReleaseID: ") . $this->pdo->log->primary($release["releaseid"]);
if (isset($release['filename']) && $release['filename'] != "") {
echo $this->pdo->log->headerOver("Filename: ") . $this->pdo->log->primary($release["filename"]);
}
if ($type !== "PAR2, ") {
echo "\n";
}
}
$newTitle = $this->pdo->escapeString(substr($newName, 0, 255));
if ($echo == true) {
if ($nameStatus == 1) {
$status = '';
switch ($type) {
case "NFO, ":
$status = "isrenamed = 1, iscategorized = 1, proc_nfo = 1,";
break;
case "PAR2, ":
$status = "isrenamed = 1, iscategorized = 1, proc_par2 = 1,";
break;
case "Filenames, ":
case "file matched source: ":
$status = "isrenamed = 1, iscategorized = 1, proc_files = 1,";
break;
case "SHA1, ":
case "MD5, ":
$status = "isrenamed = 1, iscategorized = 1, dehashstatus = 1,";
break;
case "PreDB FT Exact, ":
$status = "isrenamed = 1, iscategorized = 1,";
break;
case "sorter ":
$status = "isrenamed = 1, iscategorized = 1, proc_sorter = 1,";
break;
}
$this->pdo->queryExec(sprintf('
UPDATE releases
SET videos_id = 0, tv_episodes_id = 0, imdbid = NULL, musicinfoid = NULL,
consoleinfoid = NULL, bookinfoid = NULL, anidbid = NULL, preid = %d,
searchname = %s, %s categoryid = %d
WHERE id = %d', $preId, $newTitle, $status, $determinedCategory, $release['releaseid']));
$this->sphinx->updateRelease($release['releaseid'], $this->pdo);
} else {
$newTitle = $this->pdo->escapeString(substr($newName, 0, 255));
$this->pdo->queryExec(sprintf('
UPDATE releases
SET videos_id = 0, tv_episodes_id = 0, imdbid = NULL, musicinfoid = NULL,
consoleinfoid = NULL, bookinfoid = NULL, anidbid = NULL, preid = %d,
searchname = %s, iscategorized = 1, categoryid = %d
WHERE id = %d', $preId, $newTitle, $determinedCategory, $release['releaseid']));
$this->sphinx->updateRelease($release['releaseid'], $this->pdo);
}
}
}
}
$this->done = true;
}
}
$rename = false;
if ($argv[3] === 'true') {
$rename = true;
}
require_once dirname(__FILE__) . '/../../../www/config.php';
$pdo = new Settings();
$group = $pdo->queryOneRow(sprintf('SELECT id FROM groups WHERE name = %s', $pdo->escapeString($argv[1])));
if ($group === false) {
exit('No group with name ' . $argv[1] . ' found in the database.');
}
$releases = $pdo->query(sprintf('SELECT name, searchname, fromname, size, id FROM releases WHERE group_id = %d %s ORDER BY postdate LIMIT %d', $group['id'], $category, $argv[2]));
if (count($releases) === 0) {
exit('No releases found in your database for group ' . $argv[1] . PHP_EOL);
}
$RC = new ReleaseCleaning($pdo);
$sphinx = new SphinxSearch();
foreach ($releases as $release) {
$newName = $RC->releaseCleaner($release['name'], $release['fromname'], $release['size'], $argv[1]);
if (is_array($newName)) {
$newName = $newName['cleansubject'];
}
if ($newName !== $release['searchname']) {
echo 'Old name: ' . $release['searchname'] . PHP_EOL;
echo 'New name: ' . $newName . PHP_EOL . PHP_EOL;
if ($rename === true) {
$newName = $pdo->escapeString($newName);
$pdo->queryExec(sprintf('UPDATE releases SET searchname = %s WHERE id = %d', $newName, $release['id']));
$sphinx->updateReleaseSearchName($release['id'], $newName);
}
}
$timetotal = $consoletools->convertTime(time() - $timestart);
echo $pdo->log->header("Finished recreating search names / recategorizing / refixing names in " . $timetotal);
} else {
exit($pdo->log->info("You have no releases in the DB."));
}
} else {
if (isset($argv[1]) && $argv[1] == "reset") {
$pdo = new Settings();
$res = $pdo->query("SELECT releases.id, releases.name, releases.fromname, releases.size, groups.name AS gname FROM releases INNER JOIN groups ON releases.group_id = groups.id");
if (count($res) > 0) {
echo $pdo->log->header("Going to reset search names, this can take a while.");
$done = 0;
$timestart = time();
$consoletools = new ConsoleTools(['ColorCLI' => $pdo->log]);
foreach ($res as $row) {
$rc = new ReleaseCleaning($pdo);
$newname = $rc->releaseCleaner($row['name'], $row['fromname'], $row['size'], $row['gname']);
if (is_array($newname)) {
$newname = $newname['cleansubject'];
}
$newname = $pdo->escapeString($newname);
$pdo->queryExec(sprintf("UPDATE releases SET searchname = %s where id = %d", $newname, $row['id']));
$sphinx->updateRelease($row['id']);
$done++;
$consoletools->overWritePrimary("Renaming:" . $consoletools->percentString($done, count($res)));
}
$timenc = $consoletools->convertTime(TIME() - $timestart);
echo $pdo->log->header($done . " releases renamed in " . $timenc);
}
} else {
exit($pdo->log->error("\nThis script runs the subject names through namecleaner to create a clean search name, it also recategorizes and runs the releases through namefixer.\n" . "php resetSearchname.php full ...: To run this, recategorize and refix release names on all releases.\n" . "php resetSearchname.php limited ...: To run this on releases that have not had their names fixed, then categorizing them.\n" . "php resetSearchname.php reset ...: To just reset searchnames.\n"));
function releaseCleaner($subject, $fromName, $size, $groupname, $usepre)
{
$groups = new Groups();
$releaseCleaning = new ReleaseCleaning($groups->pdo);
$cleanerName = $releaseCleaning->releaseCleaner($subject, $fromName, $size, $groupname, $usepre);
if (!is_array($cleanerName) && $cleanerName != false) {
return ["cleansubject" => $cleanerName, "properlynamed" => true, "increment" => false];
} else {
return $cleanerName;
}
}
use nzedb\Binaries;
use nzedb\Categorize;
use nzedb\ConsoleTools;
use nzedb\NZB;
use nzedb\ReleaseCleaning;
use nzedb\Releases;
use nzedb\db\Settings;
$pdo = new Settings();
if (!isset($argv[1])) {
exit($pdo->log->error("\nYou must supply a path as the first argument. Two additional, optional arguments can also be used.\n\n" . "php {$argv['0']} /path/to/import true 1000 ...: To import using the filename as release searchname, limited to 1000\n" . "php {$argv['0']} /path/to/import false ...: To import using the subject as release searchname\n"));
}
$consoleTools = new ConsoleTools(['ColorCLI' => $pdo->log]);
$binaries = new Binaries(['Settings' => $pdo]);
$crosspostt = $pdo->getSetting('crossposttime');
$crosspostt = !empty($crosspostt) ? $crosspostt : 2;
$releasecleaning = new ReleaseCleaning($pdo);
$categorize = new Categorize(['Settings' => $pdo]);
$nzb = new NZB($pdo);
$releases = new Releases(['Settings' => $pdo]);
$nzbsperhour = $nzbSkipped = $maxtoprocess = 0;
if (isset($argv[2]) && is_numeric($argv[2])) {
exit($pdo->log->error("\nTo use a max number to process, it must be the third argument. \nTo run:\nphp nzb-import.php /path [true, false] 1000\n"));
}
if (!isset($argv[2])) {
$pieces = explode(" ", $argv[1]);
$usenzbname = isset($pieces[1]) && $pieces[1] == 'true' ? true : false;
$path = $pieces[0];
} else {
$path = $argv[1];
$usenzbname = isset($argv[2]) && $argv[2] == 'true' ? true : false;
}
/**
* Create releases from complete collections.
*
* @param int|string $groupID (optional)
*
* @return int
* @access public
*/
public function createReleases($groupID)
{
$startTime = time();
$group = $this->groups->getCBPTableNames($this->tablePerGroup, $groupID);
$categorize = new Categorize(['Settings' => $this->pdo]);
$returnCount = $duplicate = 0;
if ($this->echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->header("Process Releases -> Create releases from complete collections."));
}
$this->pdo->ping(true);
$collections = $this->pdo->queryDirect(sprintf('
SELECT SQL_NO_CACHE %s.*, groups.name AS gname
FROM %s
INNER JOIN groups ON %s.group_id = groups.id
WHERE %s %s.filecheck = %d
AND filesize > 0 LIMIT %d', $group['cname'], $group['cname'], $group['cname'], !empty($groupID) ? ' group_id = ' . $groupID . ' AND ' : ' ', $group['cname'], self::COLLFC_SIZED, $this->releaseCreationLimit));
if ($this->echoCLI && $collections !== false) {
echo $this->pdo->log->primary($collections->rowCount() . " Collections ready to be converted to releases.");
}
if ($collections instanceof \Traversable) {
$preDB = new PreDb(['Echo' => $this->echoCLI, 'Settings' => $this->pdo]);
foreach ($collections as $collection) {
$cleanRelName = $this->pdo->escapeString(utf8_encode(str_replace(['#', '@', '$', '%', '^', '§', '¨', '©', 'Ö'], '', $collection['subject'])));
$fromName = $this->pdo->escapeString(utf8_encode(trim($collection['fromname'], "'")));
// Look for duplicates, duplicates match on releases.name, releases.fromname and releases.size
// A 1% variance in size is considered the same size when the subject and poster are the same
$dupeCheck = $this->pdo->queryOneRow(sprintf("\n\t\t\t\t\t\tSELECT SQL_NO_CACHE id\n\t\t\t\t\t\tFROM releases\n\t\t\t\t\t\tWHERE name = %s\n\t\t\t\t\t\tAND fromname = %s\n\t\t\t\t\t\tAND size BETWEEN '%s'\n\t\t\t\t\t\tAND '%s'", $cleanRelName, $fromName, $collection['filesize'] * 0.99, $collection['filesize'] * 1.01));
if ($dupeCheck === false) {
$cleanedName = $this->releaseCleaning->releaseCleaner($collection['subject'], $collection['fromname'], $collection['filesize'], $collection['gname']);
if (is_array($cleanedName)) {
$properName = $cleanedName['properlynamed'];
$preID = isset($cleanerName['predb']) ? $cleanerName['predb'] : false;
$isReqID = isset($cleanerName['requestid']) ? $cleanerName['requestid'] : false;
$cleanedName = $cleanedName['cleansubject'];
} else {
$properName = true;
$isReqID = $preID = false;
}
if ($preID === false && $cleanedName !== '') {
// try to match the cleaned searchname to predb title or filename here
$preMatch = $preDB->matchPre($cleanedName);
if ($preMatch !== false) {
$cleanedName = $preMatch['title'];
$preID = $preMatch['preid'];
$properName = true;
}
}
$releaseID = $this->releases->insertRelease(['name' => $cleanRelName, 'searchname' => $this->pdo->escapeString(utf8_encode($cleanedName)), 'totalpart' => $collection['totalfiles'], 'group_id' => $collection['group_id'], 'guid' => $this->pdo->escapeString($this->releases->createGUID($cleanRelName)), 'postdate' => $this->pdo->escapeString($collection['date']), 'fromname' => $fromName, 'size' => $collection['filesize'], 'categoryid' => $categorize->determineCategory($collection['group_id'], $cleanedName), 'isrenamed' => $properName === true ? 1 : 0, 'reqidstatus' => $isReqID === true ? 1 : 0, 'preid' => $preID === false ? 0 : $preID, 'nzbstatus' => NZB::NZB_NONE]);
if ($releaseID !== false) {
// Update collections table to say we inserted the release.
$this->pdo->queryExec(sprintf('
UPDATE %s
SET filecheck = %d, releaseid = %d
WHERE id = %d', $group['cname'], self::COLLFC_INSERTED, $releaseID, $collection['id']));
$returnCount++;
if ($this->echoCLI) {
echo "Added {$returnCount} releases.\r";
}
}
} else {
// The release was already in the DB, so delete the collection.
$this->pdo->queryExec(sprintf('
DELETE c, b, p FROM %s c
INNER JOIN %s b ON(c.id=b.collection_id)
STRAIGHT_JOIN %s p ON(b.id=p.binaryid)
WHERE c.collectionhash = %s', $group['cname'], $group['bname'], $group['pname'], $this->pdo->escapeString($collection['collectionhash'])));
$duplicate++;
}
}
}
if ($this->echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->primary(PHP_EOL . number_format($returnCount) . ' Releases added and ' . number_format($duplicate) . ' duplicate collections deleted in ' . $this->consoleTools->convertTime(time() - $startTime)), true);
}
return ['added' => $returnCount, 'dupes' => $duplicate];
}
