function updateAllGroups()
{
$n = $this->n;
$groups = new Groups();
$res = $groups->getActive();
$s = new Sites();
echo $s->getLicense();
if ($res) {
$alltime = microtime(true);
echo 'Updating: ' . sizeof($res) . ' groups - Using compression? ' . ($this->compressedHeaders ? 'Yes' : 'No') . $n;
$nntp = new Nntp();
$nntp->doConnect();
foreach ($res as $groupArr) {
$this->message = array();
$this->updateGroup($nntp, $groupArr);
}
$nntp->doQuit();
echo 'Updating completed in ' . number_format(microtime(true) - $alltime, 2) . ' seconds' . $n;
} else {
echo "No groups specified. Ensure groups are added to newznab's database for updating.{$n}";
}
}
function backfillAllGroups($groupName = '', $backfillDate = null)
{
$n = $this->n;
$groups = new Groups();
if ($groupName != '') {
$grp = $groups->getByName($groupName);
if ($grp) {
$res = array($grp);
}
} else {
$res = $groups->getActive();
}
if ($res) {
$nntp = new Nntp();
$nntp->doConnect();
foreach ($res as $groupArr) {
$this->backfillGroup($nntp, $groupArr, $backfillDate);
}
$nntp->doQuit();
} else {
echo "No groups specified. Ensure groups are added to newznab's database for updating.{$n}";
}
}
/**
* Update the list of newsgroups from nntp provider matching a regex and return an array of messages.
*/
function addBulk($groupList, $active = 1)
{
require_once WWW_DIR . "/lib/binaries.php";
require_once WWW_DIR . "/lib/nntp.php";
$ret = array();
if ($groupList == "") {
$ret[] = "No group list provided.";
} else {
$db = new DB();
$nntp = new Nntp();
if (!$nntp->doConnect()) {
$ret[] = "Failed to get NNTP connection";
return $ret;
}
$groups = $nntp->getGroups();
$nntp->doQuit();
$regfilter = "/(" . str_replace(array('.', '*'), array('\\.', '.*?'), $groupList) . ")\$/";
foreach ($groups as $group) {
if (preg_match($regfilter, $group['group']) > 0) {
$res = $db->queryOneRow(sprintf("SELECT ID FROM groups WHERE name = %s ", $db->escapeString($group['group'])));
if ($res) {
$db->exec(sprintf("update groups SET active = %d where ID = %d", $active, $res["ID"]));
$ret[] = array('group' => $group['group'], 'msg' => 'Updated');
} else {
$desc = "";
$db->queryInsert(sprintf("INSERT INTO groups (name, description, active) VALUES (%s, %s, %d)", $db->escapeString($group['group']), $db->escapeString($desc), $active));
$ret[] = array('group' => $group['group'], 'msg' => 'Created');
}
}
}
}
return $ret;
}
private function _nfo_grab($nfometa, &$blobhash)
{
// nfometa should be an array() of segments from nzb file
// it will then populate the blobhash which uses the segments
// as hash entries for the blob data.
// nfometa is an array of arrays simiar to the following
// structure:
//
// The list is structured in such a way that the most ideal
// matches are at the front, while less likely ones at the
// back of the array
//
// $nfometa = array(
// [<releaseID>] = array(
// [groups] = array(
// "alt.binaries.mygroupa",
// "alt.binaries.mygroupb",
// "alt.binaries.mygroupc",
// ...
// )
// [segment] = array(<segment id>),
// [groups] = array(
// "alt.binaries.mygroupa",
// "alt.binaries.mygroupb",
// ...
// )
// ),
// [<releaseID>] = array(
// [groups] = array(
// "alt.binaries.mygroupa",
// ...
// )
// [segment] = array(<segment id>),
// ),
// ...
// )
$nntp = new Nntp();
// Connect to server (we throw an exception if we fail) which
// is caught upstairs with the nfo_grab() function
// no error handling is needed here
$nntp->doConnect(1, true);
foreach ($nfometa as $uid => $matches) {
$blobhash[$uid] = array();
foreach ($matches as $idx => $match) {
$fetched = false;
foreach ($match["groups"] as $group) {
// Don not try other groups if we already got it
if ($fetched) {
break;
}
// Select the group and then attempt to fetch the article
$blob = $nntp->getMessages($group, $match["segment"], false);
if ($blob === false) {
if ($this->verbose) {
echo '*';
}
continue;
}
// Mark that we fetched it to prevent fetching more
// of the same thing
$fetched = true;
if ($this->verbose) {
echo '.';
}
// Update blob with decrypted version and store
if ($this->is_binary($blob)) {
// Binary data is not acceptable, we only
// work with text from here on out.
continue;
}
// Read-able ascii at this point... store it
$blobhash[$uid][$idx] = $blob;
}
if (!$fetched) {
// handle empty/failed segments
$blobhash[$uid][$idx] = Null;
}
}
}
$nntp->doQuit();
}
define('FS_ROOT', realpath(dirname(__FILE__)));
require_once FS_ROOT . "/../../www/config.php";
require_once FS_ROOT . "/../../www/lib/framework/db.php";
require_once FS_ROOT . "/../../www/lib/releases.php";
require_once FS_ROOT . "/../../www/lib/nzbinfo.php";
require_once FS_ROOT . "/../../www/lib/nzb.php";
require_once FS_ROOT . "/../../www/lib/site.php";
require_once FS_ROOT . "/../../www/lib/nntp.php";
require_once FS_ROOT . "/../../www/lib/rarinfo/par2info.php";
$s = new Sites();
$site = $s->get();
$releases = new Releases();
$db = new Db();
$nzb = new Nzb();
$nntp = new Nntp();
// read pars for a release GUID, echo out any that look like a rar
$relguid = "249f9ec1f0d68d33b5fa85594ba1a47d";
$nzbfile = $nzb->getNZBPath($relguid, $site->nzbpath, true);
$nzbInfo = new nzbInfo();
$nzbInfo->loadFromFile($nzbfile);
$nntp->doConnect();
echo $nzbInfo->summarize();
foreach ($nzbInfo->parfiles as $parfile) {
echo "Fetching " . $parfile['subject'] . "\n";
$parBinary = $nntp->getMessages($parfile['groups'][0], $parfile['segments']);
if ($parBinary) {
$par2 = new Par2info();
$par2->setData($parBinary);
if (!$par2->error) {
$parFiles = $par2->getFileList();
public function fetchTestBinaries($groupname, $numarticles, $clearexistingbins)
{
$db = new DB();
$nntp = new Nntp();
$binaries = new Binaries();
$groups = new Groups();
$ret = array();
if ($clearexistingbins == true) {
$db->exec('truncate releaseregextesting');
}
$nntp->doConnect();
$groupsToFetch = array();
if (preg_match('/^[a-z]{2,3}(\\.[a-z0-9\\-]+)+$/', $groupname)) {
$groupsToFetch[] = array('name' => $groupname);
} elseif ($groupname === 0) {
$groupsToFetch = $groups->getAll();
} else {
$newsgroups = $nntp->getGroups();
foreach ($newsgroups as $ngroup) {
if (preg_match('/' . $groupname . '/', $ngroup['group'])) {
$groupsToFetch[] = array('name' => $ngroup['group']);
}
}
}
foreach ($groupsToFetch as $groupArr) {
$group = $groupArr['name'];
$data = $nntp->selectGroup($group);
if ($nntp->isError($data)) {
$ret[] = "Could not select group (doesnt exist on USP): {$group}";
continue;
} else {
$rangeStart = $data['last'] - $numarticles;
$rangeEnd = $groupEnd = $data['last'];
$rangeTotal = $rangeEnd - $rangeStart;
$done = false;
while ($done === false) {
if ($rangeTotal > $binaries->messagebuffer) {
if ($rangeStart + $binaries->messagebuffer > $groupEnd) {
$rangeEnd = $groupEnd;
} else {
$rangeEnd = $rangeStart + $binaries->messagebuffer;
}
}
if ($binaries->compressedHeaders) {
$msgs = $nntp->getXOverview($rangeStart . "-" . $rangeEnd, true, false);
} else {
$msgs = $nntp->getOverview($rangeStart . "-" . $rangeEnd, true, false);
}
if ($nntp->isError($msgs)) {
$ret[] = "Error {$msgs->code}: {$msgs->message} on " . $group;
continue 2;
}
$headers = array();
if (is_array($msgs)) {
//loop headers, figure out parts
foreach ($msgs as $msg) {
if (!isset($msg['Number'])) {
continue;
}
$msgPart = $msgTotalParts = 0;
$pattern = '|\\((\\d+)[\\/](\\d+)\\)|i';
preg_match_all($pattern, $msg['Subject'], $matches, PREG_PATTERN_ORDER);
$matchcnt = sizeof($matches[0]);
for ($i = 0; $i < $matchcnt; $i++) {
//not (int)'d here because of the preg_replace later on
$msgPart = $matches[1][$i];
$msgTotalParts = $matches[2][$i];
}
if (!isset($msg['Subject']) || $matchcnt == 0) {
// not a binary post most likely.. continue
continue;
}
if ((int) $msgPart > 0 && (int) $msgTotalParts > 0) {
$subject = utf8_encode(trim(preg_replace('|\\(' . $msgPart . '[\\/]' . $msgTotalParts . '\\)|i', '', $msg['Subject'])));
if (!isset($headers[$subject])) {
$headers[$subject]['Subject'] = $subject;
$headers[$subject]['From'] = $msg['From'];
$headers[$subject]['Date'] = strtotime($msg['Date']);
$headers[$subject]['Message-ID'] = $msg['Message-ID'];
$headers[$subject]['Size'] = $msg['Bytes'];
} else {
$headers[$subject]['Size'] += $msg['Bytes'];
}
}
}
unset($msgs);
if (isset($headers) && count($headers)) {
$groupRegexes = $this->getForGroup($group);
$binSetData = array();
foreach ($headers as $subject => $data) {
$binData = array('name' => $subject, 'fromname' => $data['From'], 'date' => $data['Date'], 'binaryhash' => md5($subject . $data['From'] . $group), 'groupname' => $group, 'regexID' => "null", 'categoryID' => "null", 'reqID' => "null", 'blacklistID' => 0, 'size' => $data['Size'], 'relname' => "null", 'relpart' => "null", 'reltotalpart' => "null");
//Filter binaries based on black/white list
if ($binaries->isBlackListed($data, $group)) {
//binary is blacklisted
$binData['blacklistID'] = 1;
}
//Apply Regexes
$regexMatches = array();
foreach ($groupRegexes as $groupRegex) {
$regexCheck = $this->performMatch($groupRegex, $subject, $data['From']);
if ($regexCheck !== false) {
$regexMatches = $regexCheck;
$binData['regexID'] = $regexCheck['regexID'];
$binData['categoryID'] = $regexCheck['regcatid'];
$binData['reqID'] = empty($regexCheck['reqID']) ? "null" : $regexCheck['reqID'];
$binData['relname'] = $regexCheck['name'];
break;
}
}
$binSetData[] = $binData;
}
//insert 500 bins at a time
$binChunks = array_chunk($binSetData, 500);
foreach ($binChunks as $binChunk) {
foreach ($binChunk as $chunk) {
$binParams[] = sprintf("(%s, %s, FROM_UNIXTIME(%s), %s, %s, %s, %s, %s, %d, %d, now())", $db->escapeString($chunk['name']), $db->escapeString($chunk['fromname']), $db->escapeString($chunk['date']), $db->escapeString($chunk['binaryhash']), $db->escapeString($chunk['groupname']), $chunk['regexID'], $chunk['categoryID'], $chunk['reqID'], $chunk['blacklistID'], $chunk['size']);
}
$binSql = "INSERT IGNORE INTO releaseregextesting (name, fromname, date, binaryhash, groupname, regexID, categoryID, reqID, blacklistID, size, dateadded) VALUES " . implode(', ', $binParams);
//echo $binSql;
$db->exec($binSql);
}
$ret[] = "Fetched " . number_format($numarticles) . " articles from " . $group;
} else {
$ret[] = "No headers found on " . $group;
continue;
}
} else {
$ret[] = "Can't get parts from server (msgs not array) on " . $group;
continue;
}
if ($rangeEnd == $groupEnd) {
$done = true;
}
$rangeStart = $rangeEnd + 1;
}
}
}
$nntp->doQuit();
return $ret;
}
/**
* Performing parsing.
*/
public function process()
{
$db = new DB();
// Default query for both full db and last 4 hours.
$sql = "SELECT r.searchname, r.name, r.fromname, r.ID as RID, r.categoryID, r.guid, r.postdate,\n\t\t\t rn.ID as nfoID,\n\t\t\t g.name as groupname,\n\t\t\t GROUP_CONCAT(rf.name) as filenames\n\t\tFROM releases r\n\t\tLEFT JOIN releasenfo rn ON (rn.releaseID = r.ID)\n\t\tLEFT JOIN groups g ON (g.ID = r.groupID)\n\t\tLEFT JOIN releasefiles rf ON (rf.releaseID = r.ID)\n\t\tWHERE r.categoryID in (" . Category::CAT_TV_OTHER . "," . Category::CAT_MOVIE_OTHER . "," . Category::CAT_MISC_OTHER . "," . Category::CAT_XXX_OTHER . ")\n\t\t%s\n\t\tGROUP BY r.ID";
$res = $db->query(sprintf($sql, $this->limited ? "AND r.adddate BETWEEN NOW() - INTERVAL 4 HOUR AND NOW()" : ""));
$this->releasestocheck = sizeof($res);
if ($res) {
echo "PostPrc : Parsing last " . $this->releasestocheck . " releases in the Other-Misc categories\n";
foreach ($res as $rel) {
$tempname = $foundName = $methodused = '';
//Knoc.One
if (preg_match("/KNOC.ONE/i", $rel['name'])) {
$title = '';
$items = preg_split("/(\\.| )/", $rel['name']);
foreach ($items as $value) {
if (preg_match("/^[a-z]+\$/i", $value)) {
$len = strlen($value);
if ($len > 2) {
$title .= substr($value, -2) . substr($value, 0, -2) . " ";
} elseif ($len = 2) {
$title .= substr($value, -1) . substr($value, 0, -1) . " ";
} else {
$title .= $value . " ";
}
} else {
$title .= $value . " ";
}
}
$foundName = $title;
$methodused = "Knoc.One";
$this->determineCategory($rel, $foundName, $methodused);
}
///
///Use the Nfo to try to get the proper Releasename.
///
$nfo = $db->queryOneRow(sprintf("select uncompress(nfo) as nfo from releasenfo where releaseID = %d", $rel['RID']));
if ($nfo && $foundName == "") {
$this->nfosprocessed++;
$nfo = $nfo['nfo'];
//LOUNGE releases
if (preg_match('/([a-z0-9.]+\\.MBLURAY)/i', $nfo, $matches)) {
$foundName = $matches[1];
$methodused = "LOUNGE";
$this->determineCategory($rel, $foundName, $methodused);
}
//AsianDVDClub releases
if (preg_match('/adc-[a-z0-9]{1,10}/', $rel['name'])) {
if (preg_match('/.*\\(\\d{4}\\).*/i', $nfo, $matches)) {
$foundName = $matches[0];
$methodused = "AsianDVDClub";
$this->determineCategory($rel, $foundName, $methodused);
}
}
//ACOUSTiC releases
if (preg_match('/ACOUSTiC presents \\.\\.\\..*?([a-z0-9].*?\\(.*?\\))/is', $nfo, $matches)) {
$foundName = $matches[1] . ".MBLURAY";
$methodused = "ACOUSTiC ";
$this->determineCategory($rel, $foundName, $methodused);
}
//Japhson releases
if (preg_match('/Japhson/i', $nfo, $matches)) {
$movie = new Movie();
$imdbID = null;
if (preg_match('/tt(\\d{7})/i', $nfo, $matches)) {
$imdbId = $matches[1];
$movCheck = $movie->fetchImdbProperties($imdbId);
$foundName = $movCheck['title'];
if (!preg_match('/(19|20)\\d{2}/i', $foundName)) {
$foundName = $foundName . "." . $movCheck['year'];
}
if (preg_match('/language.*?\\b([a-z0-9]+)\\b/i', $nfo, $matches)) {
if (!preg_match('/English/i', $matches[1])) {
$foundName = $foundName . "." . $matches[1];
}
}
if (preg_match('/audio.*?\\b(\\w+)\\b/i', $nfo, $matches)) {
if (preg_match('/(Chinese|German|Dutch|Spanish|Hebrew|Finnish|Norwegian)/i', $matches[1])) {
$foundName = $foundName . "." . $matches[1];
}
}
if (preg_match('/(video|resolution|video res).*?(1080|720|816|820|272|1280 @|528|1920)/i', $nfo, $matches)) {
if ($matches[2] == '1280 @') {
$matches[2] = '720';
}
if ($matches[2] == '1920') {
$matches[2] = '1080';
}
$foundName = $foundName . "." . $matches[2];
}
if (preg_match('/source.*?\\b(DVD9|DVD5|BDRIP|DVD\\-?RIP|BLURAY)\\b/i', $nfo, $matches)) {
$foundName = $foundName . "." . $matches[1];
}
if (preg_match('/(video|resolution|video res).*?(XVID|X264|WMV)/i', $nfo, $matches)) {
$foundName = $foundName . "." . $matches[2];
}
if (preg_match('/audio.*?\\b(DTS|AC3)\\b/i', $nfo, $matches)) {
$foundName = $foundName . "." . $matches[1];
}
$foundName = $foundName . "-Japhson";
$methodused = "Japhson";
$this->determineCategory($rel, $foundName, $methodused);
}
}
//AIHD releases
if (preg_match('/ALWAYS iN HiGH/i', $nfo, $matches)) {
$movie = new Movie();
$imdbID = null;
if (preg_match('/tt(\\d{7})/i', $nfo, $matches)) {
$imdbId = $matches[1];
$movCheck = $movie->fetchImdbProperties($imdbId);
$foundName = $movCheck['title'];
if (!preg_match('/(19|20)\\d{2}/i', $foundName)) {
$foundName = $foundName . "." . $movCheck['year'];
}
if (preg_match('/L\\.([a-z0-9]+)\\b/i', $nfo, $matches)) {
if (!preg_match('/En/i', $matches[1])) {
$foundName = $foundName . "." . $matches[1];
}
}
if (preg_match('/(V).*?(1080|720|816|820|272|1280 @|528|1920)/i', $nfo, $matches)) {
if ($matches[2] == '1280 @') {
$matches[2] = '720';
}
if ($matches[2] == '1920') {
$matches[2] = '1080';
}
$foundName = $foundName . "." . $matches[2];
}
if (preg_match('/V.*?\\b(DVD9|DVD5|BDRIP|DVD\\-?RIP|BLURAY)\\b/i', $nfo, $matches)) {
$foundName = $foundName . "." . $matches[1];
}
if (preg_match('/(V).*?(XVID|X264|WMV)/i', $nfo, $matches)) {
$foundName = $foundName . "." . $matches[2];
}
if (preg_match('/A.*?\\b(DTS|AC3)\\b/i', $nfo, $matches)) {
$foundName = $foundName . "." . $matches[1];
}
$foundName = $foundName . "-AIHD";
$methodused = "AIHD";
$this->determineCategory($rel, $foundName, $methodused);
}
}
//IMAGiNE releases
if (preg_match('/\\*\\s+([a-z0-9]+(?:\\.|_| )[a-z0-9\\.\\_\\- ]+ \\- imagine)\\s+\\*/i', $nfo, $matches)) {
$foundName = $matches[1];
$methodused = "imagine";
$this->determineCategory($rel, $foundName, $methodused);
}
//LEGION releases
if (preg_match('/([a-z0-9 \\.\\-]+LEGi0N)/is', $nfo, $matches) && $foundName == "") {
$foundName = $matches[1];
$methodused = "Legion";
$this->determineCategory($rel, $foundName, $methodused);
}
//SWAGGER releases
if (preg_match('/(S W A G G E R|swg.*?nfo)/i', $nfo) && $foundName == "") {
if (preg_match('/presents.*?([a-z0-9].*?\\((19|20)\\d{2}\\))/is', $nfo, $matches)) {
$foundName = $matches[1];
}
if (preg_match('/language.*?\\b([a-z0-9]+)\\b/i', $nfo, $matches)) {
if ($matches[1] != "english") {
$foundName = $foundName . "." . $matches[1];
}
}
if (preg_match('/resolution.*?(1080|720)/i', $nfo, $matches)) {
$foundName = $foundName . ".BluRay." . $matches[1];
}
if (preg_match('/video.*?\\b([a-z0-9]+)\\b/i', $nfo, $matches)) {
$foundName = $foundName . "." . $matches[1];
}
if (preg_match('/audio.*?\\b([a-z0-9]+)\\b/i', $nfo, $matches)) {
$foundName = $foundName . "." . $matches[1];
}
$foundName = $foundName . "-SWAGGER";
$methodused = "SWAGGER";
$this->determineCategory($rel, $foundName, $methodused);
}
//cm8 releases
if (preg_match('/([a-z0-9]+(?:\\.|_| )[a-z0-9\\.\\_\\- \'\\)\\(]+\\-(futv|crimson|qcf|runner|clue|ftp|episode|momentum|PFA|topaz|vision|tdp|haggis|nogrp|shirk|imagine|santi|sys|deimos|ltu|ficodvdr|cm8|dvdr|Nodlabs|aaf|sprinter|exvid|flawl3ss|rx|magicbox|done|unveil))\\b/i', $nfo, $matches) && $foundName == "") {
//echo "this: ".$matches[1]."\n";
$foundName = $matches[1];
$methodused = "cm8";
$this->determineCategory($rel, $foundName, $methodused);
}
//river
if (preg_match('/([a-z0-9\\.\\_\\-]+\\-(webios|river|w4f|sometv|ngchd|C4|gf|bov|26k|ftw))\\b/i', $nfo, $matches) && $foundName == "") {
$foundName = $matches[1];
$methodused = "river-1";
$this->determineCategory($rel, $foundName, $methodused);
}
if (preg_match('/([a-z0-9]+(?:\\.|_| )[a-z0-9\\.\\_\\- \'\\)\\(]+\\-(CiA|Anarchy|RemixHD|FTW|Revott|WAF|CtrlHD|Telly|Nif|Line|NPW|Rude|EbP|CRisC|SHK|AssAss1ns|Leverage|BBW|NPW))\\b/i', $nfo, $matches) && $foundName == "") {
$foundName = $matches[1];
$methodused = "river-2";
$this->determineCategory($rel, $foundName, $methodused);
}
if (preg_match('/([a-z0-9]+(?:\\.|_| )[a-z0-9\\.\\_\\- \'\\)\\(]+\\-(XPD|RHyTM))\\b/i', $nfo, $matches) && $foundName == "") {
$foundName = $matches[1];
$methodused = "river-3";
$this->determineCategory($rel, $foundName, $methodused);
}
if (preg_match('/(-PROD$|-BOV$|-NMR$|$-HAGGiS|-JUST$|CRNTV$|-MCA$|int$|-DEiTY$|-VoMiT$|-iNCiTE$|-BRUTUS$|-DCN$|-saints$|-sfm$|-lol$|-fov$|-logies$|-c4tv$|-fqm$|-jetset$|-ils$|-miragetv$|-gfvid$|-btl$|-terra$)/i', $rel['searchname']) && $foundName == "") {
$foundName = $rel['searchname'];
$methodused = "river-4";
$this->determineCategory($rel, $foundName, $methodused);
}
//SANTi releases
if (preg_match('/\\b([a-z0-9]+(?:\\.|_| )[a-z0-9\\.\\_\\- \']+\\-santi)\\b/i', $nfo, $matches) && $foundName == "") {
$foundName = $matches[1];
$methodused = "SANTi";
$this->determineCategory($rel, $foundName, $methodused);
}
//INSPiRAL releases
if (preg_match('/^([a-z0-9]+(?:\\.|_| )[a-z0-9\\.\\_\\- ]+ \\- INSPiRAL)\\s+/im', $nfo, $matches) && $foundName == "") {
$foundName = $matches[1];
$methodused = "INSPiRAL";
$this->determineCategory($rel, $foundName, $methodused);
}
//CIA releases
if (preg_match('/Release NAME.*?\\:.*?([a-z0-9][a-z0-9\\.\\ ]+)\\b.*?([a-z0-9][a-z0-9\\.\\ ]+\\-CIA)\\b/is', $nfo, $matches) && $foundName == "") {
$foundName = $matches[1] . $matches[2];
$methodused = "CIA";
$this->determineCategory($rel, $foundName, $methodused);
}
//HDChina releases
if (preg_match('/HDChina/', $nfo) && $foundName == "") {
if (preg_match('/Disc Title\\:.*?\\b([a-z0-9\\ \\.\\-\\_()]+\\-HDChina)/i', $nfo, $matches)) {
$foundName = $matches[1];
$methodused = "HDChina";
$this->determineCategory($rel, $foundName, $methodused);
}
}
//Pringles
if (preg_match('/PRiNGLES/', $nfo) && $foundName == "") {
if (preg_match('/is giving you.*?\\b([a-z0-9 ]+)\\s/i', $nfo, $matches)) {
$foundName = $matches[1];
$foundName = rtrim($foundName);
$foundName = ltrim($foundName);
}
if (preg_match('/this release.*?((19|20)\\d{2})/i', $nfo, $matches)) {
$foundName = $foundName . "." . $matches[1];
$foundName = rtrim($foundName);
}
if (preg_match('/\\[x\\] (Danish|Norwegian|Swedish|Finish|Other)/i', $nfo, $matches)) {
$foundName = $foundName . "." . $matches[1];
}
if (preg_match('/\\[x\\] (DVD9|DVD5)/i', $nfo, $matches)) {
$foundName = $foundName . "." . $matches[1];
}
$foundName = $foundName . "-PRiNGLES";
$methodused = "Pringles";
$this->determineCategory($rel, $foundName, $methodused);
}
//Fairlight releases
if (preg_match('/\\/Team FairLight/', $nfo) && $foundName == "") {
$title = null;
$os = null;
$method = null;
if (preg_match('/\\b([a-z0-9\\ \\- \\_()\\.]+) \\(c\\)/i', $nfo, $matches)) {
$title = $matches['1'];
$foundName = $title;
}
$foundName = $foundName . "-FLT";
$methodused = "FairLight";
$this->determineCategory($rel, $foundName, $methodused);
}
//CORE releases
if (preg_match('/Supplied.*?\\:.*?(CORE)/', $nfo) || preg_match('/Packaged.*?\\:.*?(CORE)/', $nfo) && $foundName == "") {
$title = null;
$os = null;
$method = null;
if (preg_match('/\\b([a-z0-9\\.\\-\\_\\+\\ ]+) \\*[a-z0-9]+\\*/i', $nfo, $matches)) {
$title = $matches['1'];
$foundName = $title;
}
if (preg_match('/Crack\\/.*?\\:.*?([a-z]+)/i', $nfo, $matches)) {
$method = $matches['1'];
$foundName = $foundName . " " . $method;
}
if (preg_match('/OS.*?\\:.*?([a-z]+)/i', $nfo, $matches)) {
$os = $matches['1'];
$foundName = $foundName . " " . $os;
}
$foundName = $foundName . "-CORE";
$methodused = "CORE";
$this->determineCategory($rel, $foundName, $methodused);
}
//CompleteRelease
if (preg_match('/Complete name.*?([a-z0-9].*?\\-[a-z0-9]+)\\b/i', $nfo, $matches) && $foundName == "") {
$foundName = $matches[1];
$methodused = "CompleteRelease";
$this->determineCategory($rel, $foundName, $methodused);
}
//Livesets
if (preg_match('/\\nLivesets.*?\\n.*?\\n.*?\\n.*?\\n.*?\\n(?P<name>\\w.*?)\\n(?P<album>\\w.*?)\\n/im', $nfo, $matches) && $foundName == "") {
$artist = $matches['name'];
$title = $matches['album'];
$source = null;
$year = null;
if (preg_match('/Year.*?\\:{1,2} ?(?P<year>(19|20)\\d{2})/i', $nfo, $matches)) {
$year = $matches[1];
} elseif (preg_match('/date.*?\\:.*?(?P<year>(19|20)\\d{2})/i', $nfo, $matches)) {
$year = $matches[1];
}
if (preg_match('/(web|cable|sat)/i', $title)) {
$source = "";
} elseif (preg_match('/Source.*?\\:{1,2} ?(?P<source>.*?)(\\s{2,}|\\s{1,})/i', $nfo, $matches)) {
$source = $matches[1];
if ($source == "Satellite") {
$source = "Sat";
}
}
if ($artist) {
$tempname = $artist;
if ($title) {
$tempname = $tempname . "-" . $title;
}
if ($source) {
$tempname = $tempname . "-" . $source;
}
if ($year) {
$tempname = $tempname . "-" . $year;
}
$tempname = preg_replace("/[^a-zA-Z,0-9,\\-,\\s]/", "", $tempname);
$foundName = $tempname;
$methodused = "Live Sets";
$this->determineCategory($rel, $foundName, $methodused);
}
}
//Typical scene regex
if (preg_match('/(?P<source>Source[\\s\\.]*?:|fix for nuke)?(?:\\s|\\]|\\[)?(?P<name>[a-z0-9\'\\-]+(?:\\.|_)[a-z0-9\\.\\-_\'&]+\\-[a-z0-9&]+)(?:\\s|\\[|\\])/i', $nfo, $matches) && $foundName == "") {
if (empty($matches['source'])) {
if (!preg_match('/usenet\\-space/i', $matches['name'])) {
$foundName = $matches['name'];
$methodused = "Scene";
$this->determineCategory($rel, $foundName, $methodused);
}
}
}
}
//The Big One
if (preg_match_all('/([a-z0-9\\ ]+)\\.{1,}(\\:|\\[)(?P<name>.*)(\\s{2}|\\s{1})/i', $nfo, $matches) && $foundName == "") {
$lut = array();
foreach ($matches[1] as $key => $k) {
$lut[str_replace(' ', '', strtolower(trim($k)))] = trim($matches[3][$key]);
}
$year = null;
$vidsource = null;
$series = null;
$season = null;
$episode = null;
$language = null;
$artist = null;
$source = null;
foreach ($lut as $k => $v) {
$v = rtrim($v);
if (!$year && preg_match('/((19|20)\\d{2})/', $v, $matches)) {
$year = $matches[1];
}
if (!$vidsource && preg_match('/(xvid|x264|h264|wmv|divx)/i', $v, $matches)) {
$vidsource = $matches[1];
}
if (!$season && preg_match('/(season|seizon).*?(\\d{1,3})/i', $v, $matches)) {
$season = $matches[2];
}
if (!$episode && preg_match('/(Episode|ep).*?(\\d{1,3})/i', $v, $matches)) {
$episode = $matches[2];
}
}
if (isset($lut['artist'])) {
$del = "-";
if (isset($lut['artist'])) {
$lut['artist'] = trim($lut['artist'], " ");
$tempname = $lut['artist'];
}
if (isset($lut['title'])) {
$tempname = $tempname . $del . $lut['title'];
}
if (isset($lut['album']) && !isset($lut['title'])) {
$tempname = $tempname . $del . $lut['album'];
}
if (isset($lut['track']) && !isset($lut['title']) && !isset($lut['album'])) {
$tempname = $tempname . $del . $lut['track'];
}
if (!isset($lut['source'])) {
$lut['source'] = 'WEB';
}
if (isset($lut['source']) && !preg_match('/SAT/i', $tempname)) {
$tempname = $tempname . $del . $lut['source'];
}
if (!preg_match('/(19|20)\\d{2}/', $tempname) && $year) {
$tempname = $tempname . $del . $year;
}
if (isset($lut['ripper'])) {
$tempname = $tempname . $del . $lut['ripper'];
}
$tempname = preg_replace("/[^a-zA-Z,0-9,\\-,\\&,\\s]/", "", $tempname);
$tempname = preg_replace("/[ ]{2,}/", "", $tempname);
$methodused = "The Big One Music";
$foundName = $tempname;
$this->determineCategory($rel, $foundName, $methodused);
} else {
if (isset($lut['title'])) {
$del = " ";
if (isset($lut['series'])) {
$tempname = $lut['series'];
}
$tempname = $tempname . $del . $lut['title'];
if ($season && $episode) {
$tempname = $tempname . $del . "S" . str_pad($season, 2, '0', STR_PAD_LEFT) . 'E' . str_pad($episode, 2, '0', STR_PAD_LEFT);
} else {
if ($season) {
$tempname = $tempname . $del . "S" . $season;
}
if ($episode) {
$tempname = $tempname . $del . "Ep" . $episode;
}
}
if (isset($lut['source']) && !preg_match('/SAT/i', $lut['title'])) {
$tempname = $tempname . $del . $lut['source'];
}
if (!preg_match('/(19|20)\\d{2}/', $tempname) && $year) {
$tempname = $tempname . $del . $year;
}
if (isset($lut['language'])) {
$tempname = $tempname . $del . $lut['language'];
}
if ($vidsource) {
$tempname = $tempname . $del . $vidsource;
}
$tempname = preg_replace("/ /", " ", $tempname);
$tempname = preg_replace("/[^a-zA-Z,0-9,\\-,\\&,\\s]/", " ", $tempname);
$tempname = preg_replace("/[ ]+/", " ", $tempname);
$methodused = "The Big One Other";
$foundName = $tempname;
$this->determineCategory($rel, $foundName, $methodused);
}
}
}
///
///unable to extract releasename from nfo, try the rar file
///
if ($rel['filenames'] && $foundName == '') {
$this->releasefilesprocessed++;
$files = explode(',', $rel['filenames']);
if (!array($files)) {
$files = array($files);
}
// Scene regex
$sceneRegex = '/([a-z0-9\'\\-\\.\\_\\(\\)\\+\\ ]+\\-[a-z0-9\'\\-\\.\\_\\(\\)\\ ]+)(.*?\\\\.*?|)\\.(?:\\w{3,4})$/i';
foreach ($files as $file) {
// Petje Releases
if (preg_match('/Petje \\<petje\\@pietamientje\\.com\\>/', $rel['fromname'], $matches3) && $foundName == '') {
if (preg_match('/.*\\.(mkv|avi|mp4|wmv|divx)/', $file, $matches4)) {
$array_new = explode('\\', $matches4[0]);
foreach ($array_new as $item) {
if (preg_match('/.*\\((19|20\\d{2})\\)$/', $item, $matched)) {
//echo $matched[0].".720p.x264-Petje";
//print_r($matched);
$foundName = $matched[0] . ".720p.x264-Petje";
$methodused = "Petje";
$this->determineCategory($rel, $foundName, $methodused);
break 2;
}
}
}
}
//3D Remux
if (preg_match('/.*Remux\\.mkv/', $file, $matches4)) {
$foundName = str_replace(".mkv", "", $matches4[0]);
$methodused = "3D Remux";
$this->determineCategory($rel, $foundName, $methodused);
}
//QoQ Extended
if (preg_match('/Q\\-sbuSLN.*/i', $file, $matches4)) {
$new1 = preg_match('/( )?(\\.wmv|\\.divx|\\.avi|\\.mkv)/i', $matches4[0], $matched);
$new2 = str_replace($matched[0], "", $matches4[0]);
$foundName = strrev($new2);
$methodused = "QoQ Extended";
$this->determineCategory($rel, $foundName, $methodused);
}
// Directory\Title.Year.Format.Group.mkv
if (preg_match('/(?<=\\\\).*?BLURAY.(1080|720)P.*?KNORLOADING(?=\\.MKV)/i', $file, $matches3) && $foundName == '') {
$foundName = $matches3['0'];
$methodused = "a.b.hdtv.x264";
$this->determineCategory($rel, $foundName, $methodused);
}
// ReleaseGroup.Title.Format.mkv
if (preg_match('/(?<=swg_|swghd\\-|lost\\-|veto\\-|kaka\\-|abd\\-|airline\\-|daa\\-|data\\-|japhson\\-|ika\\-|lng\\-|nrdhd\\-|saimorny\\-|sparks\\-|ulshd\\-|nscrns\\-|ifpd\\-|invan\\-|an0\\-|besthd\\-|muxhd\\-|s7\\-).*?((1080|720)|P)(?=\\.MKV)/i', $file, $matches3) && $foundName == '') {
$foundName = str_replace("_", ".", $matches3['0']);
$methodused = "a.b.hdtv.x264";
$this->determineCategory($rel, $foundName, $methodused);
}
// Title.Format.ReleaseGroup.mkv
if (preg_match('/.*?(1080|720)(|P).(SON)/i', $file, $matches3) && $foundName == '') {
$foundName = str_replace("_", ".", $matches3['0']);
$methodused = "a.b.hdtv.x264";
$this->determineCategory($rel, $foundName, $methodused);
}
//epubmobi
if (preg_match('/.*\\.(epub|mobi|azw3|pdf|prc|lit|rtf|azw|cbr|doc)/', $file, $matches4)) {
$foundName = str_replace(".doc", "", str_replace(".cbr", "", str_replace(".prc", "", str_replace(".pdf", "", str_replace(".azw3", "", str_replace(".mobi", "", str_replace(".epub", "", str_replace(".rtf", "", str_replace(".azw", "", str_replace(".lit", "", $matches4[0]))))))))));
$methodused = "EpubMobi";
$this->determineCategory($rel, $foundName, $methodused);
}
//Check rarfile contents for a scene name
if (preg_match($sceneRegex, $file, $matches) && $foundName == '') {
//Simply Releases Toppers
if (preg_match('/(\\\\)(?P<name>.*?ReleaseS Toppers)/', $file, $matches1) && $foundName == '') {
$foundName = $matches1['name'];
$methodused = "Release Files-1";
$this->determineCategory($rel, $foundName, $methodused);
}
//Scene format no folder.
if (preg_match('/^([a-z0-9\\.\\_\\- ]+\\-[a-z0-9\\_]+)(\\\\|)$/i', $matches[1]) && $foundName == '') {
if (strlen($matches['1']) >= 15) {
$foundName = $matches['1'];
$methodused = "Scene format no folder.";
$this->determineCategory($rel, $foundName, $methodused);
}
}
//Check to see if file is inside of a folder. Use folder name if it is
if (preg_match('/^(.*?\\\\)(.*?\\\\|)(.*?)$/i', $file, $matches1) && $foundName == '') {
if (preg_match('/^([a-z0-9\\.\\_\\- ]+\\-[a-z0-9\\_]+)(\\\\|)$/i', $matches1['1'], $res)) {
$foundName = $res['1'];
$methodused = "Release Files-1";
$this->determineCategory($rel, $foundName, $methodused);
}
if (preg_match('/(?!UTC)([a-z0-9]+[a-z0-9\\.\\_\\- \'\\)\\(]+(\\d{4}|HDTV).*?\\-[a-z0-9]+)/i', $matches1['1'], $res) && $foundName == '') {
$foundName = $res['1'];
$methodused = "Release Files-2";
$this->determineCategory($rel, $foundName, $methodused);
}
if (preg_match('/^([a-z0-9\\.\\_\\- ]+\\-[a-z0-9\\_]+)(\\\\|)$/i', $matches1['2'], $res) && $foundName == '') {
$foundName = $res['1'];
$methodused = "Release Files-3";
$this->determineCategory($rel, $foundName, $methodused);
}
if (preg_match('/^([a-z0-9\\.\\_\\- ]+\\-(?:.+)\\(html\\))\\\\/i', $matches1['1'], $res) && $foundName == '') {
$foundName = $res['1'];
$methodused = "Release Files-4";
$this->determineCategory($rel, $foundName, $methodused);
}
}
if (preg_match('/(?!UTC)([a-z0-9]+[a-z0-9\\.\\_\\- \'\\)\\(]+(\\d{4}|HDTV).*?\\-[a-z0-9]+)/i', $file, $matches2) && $foundName == '') {
$foundName = $matches2['1'];
$methodused = "Release Files-4";
$this->determineCategory($rel, $foundName, $methodused);
}
}
}
//RAR file contents release name matching
/*
if (sizeof($files) > 0 && $foundName == '')
{
echo "RAR checking\n";
//Loop through releaseFiles to find a match
foreach($releaseFiles as $rarFile)
{
//echo "-{$rarFile}\n";
if ($foundName == '')
{
//Lookup name via reqid (filename)
if (preg_match('/\.(avi|mkv|mp4|mov|wmv|iso|img|gcm|ps3|wad|ac3|nds|bin|cue|mdf)/i', $rarFile))
{
$this->site->reqidurl
$lookupUrl = 'http://allfilled/query.php?t=alt.binaries.srrdb&reqid='.urlencode(basename($rarFile));
echo '-lookup: '.$lookupUrl."\n";
$xml = getUrl($lookupUrl);
//$xml = false;
if ($xml !== false)
{
$xmlObj = @simplexml_load_string($xml);
$arrXml = objectsIntoArray($xmlObj);
if (isset($arrXml["item"]) && is_array($arrXml["item"]) && isset($arrXml["item"]["@attributes"]) && is_array($arrXml["item"]["@attributes"]))
{
$foundName = $arrXml["item"]["@attributes"]["title"];
}
}
}
}
}
}
*/
}
// do par check if user has elected for downloading extra stuff
if ($this->site->unrarpath != '' && $foundName == "") {
$nzb = new NZB();
$nzbfile = $nzb->getNZBPath($rel['guid'], $this->site->nzbpath, true);
$nzbInfo = new nzbInfo();
$nzbInfo->loadFromFile($nzbfile);
if (!empty($nzbInfo->parfiles) && empty($nzbInfo->rarfiles) && empty($nzbInfo->audiofiles)) {
$nntp = new Nntp();
$nntp->doConnect();
if ($this->verbose) {
echo "Checking Par\n";
}
foreach ($nzbInfo->parfiles as $parfile) {
$this->parsprocessed++;
$parBinary = $nntp->getMessages($parfile['groups'][0], $parfile['segments'], $this->verbose);
if ($parBinary) {
$par2 = new Par2info();
$par2->setData($parBinary);
if (!$par2->error) {
$parFiles = $par2->getFileList();
foreach ($parFiles as $file) {
if (isset($file['name']) && (preg_match('/.*part0*1\\.rar$/iS', $file['name'], $matches) || preg_match('/(?!part0*1)\\.rar$/iS', $file['name'], $matches) || preg_match('/\\.001$/iS', $file['name'], $matches))) {
$foundName = preg_replace('/^(.*)(\\.part0*1\\.rar|\\.rar|\\.001)$/i', '\\1', $file['name']);
$methodused = "Par file";
$this->determineCategory($rel, $foundName, $methodused);
break;
}
}
}
}
unset($parBinary);
if ($foundName != "") {
break;
}
}
$nntp->doQuit();
}
}
///
/// This is a last ditch effort, build a ReleaseName from the Nfo
///
if ($nfo && ($foundName == "" || $methodused == 'Scene format no folder.')) {
//LastNfoAttempt
if (preg_match('/tt(\\d{7})/i', $nfo, $matches) && $foundName == "") {
$movie = new Movie();
$imdbId = $matches[1];
$movCheck = $movie->fetchImdbProperties($imdbId);
$buffer = getUrl('http://akas.imdb.com/title/tt' . $imdbId . '/');
if (!preg_match('/content\\=\\"video\\.tv\\_show\\"/i', $buffer)) {
if (isset($movCheck['title'])) {
$foundName = $movCheck['title'];
if (!preg_match('/(19|20)\\d{2}/i', $foundName)) {
$foundName = $foundName . "." . (isset($movCheck['year']) ? $movCheck['year'] : "");
}
if (preg_match('/language.*?\\b([a-z0-9]+)\\b/i', $nfo, $matches)) {
if (!preg_match('/English/i', $matches[1])) {
$foundName = $foundName . "." . $matches[1];
}
}
if (preg_match('/audio.*?\\b(\\w+)\\b/i', $nfo, $matches)) {
if (preg_match('/(Chinese|German|Dutch|Spanish|Hebrew|Finnish|Norwegian)/i', $matches[1])) {
$foundName = $foundName . "." . $matches[1];
}
}
if (preg_match('/(video|resolution|video res).*?(1080|720|816|820|272|1280 @|528|1920)/i', $nfo, $matches)) {
if ($matches[2] == '1280 @') {
$matches[2] = '720';
}
if ($matches[2] == '1920') {
$matches[2] = '1080';
}
$foundName = $foundName . "." . $matches[2];
}
if (preg_match('/source.*?\\b(DVD9|DVD5|BDRIP|DVD\\-?RIP|BLURAY|BD)\\b/i', $nfo, $matches)) {
if ($matches[1] == 'BD') {
$matches[1] = 'Bluray.x264';
}
$foundName = $foundName . "." . $matches[1];
}
if (preg_match('/(video|resolution|video res).*?(XVID|X264|WMV)/i', $nfo, $matches)) {
$foundName = $foundName . "." . $matches[2];
}
if (preg_match('/audio.*?\\b(DTS|AC3)\\b/i', $nfo, $matches)) {
$foundName = $foundName . "." . $matches[1];
}
$foundName = $foundName . "-NoGroup";
$methodused = "LastNfoAttempt";
$this->determineCategory($rel, $foundName, $methodused);
}
}
}
}
if ($foundName == '' && $this->verbose) {
echo "ReleaseID: \t\t" . $rel["RID"] . "\n" . " Group: \t\t" . $rel["groupname"] . "\n" . " Old Name: \t\t" . $rel["name"] . "\n" . " Old SearchName: \t" . $rel["searchname"] . "\n" . " Status: \t\tNo new name found.\n\n";
}
}
}
if ($this->verbose) {
echo $this->releasestocheck . " releases checked\n" . $this->nfosprocessed . " of " . $this->releasestocheck . " releases had nfo's processed\n" . $this->parsprocessed . " of " . $this->releasestocheck . " releases had par's processed\n" . $this->releasefilesprocessed . " of " . $this->releasestocheck . " releases had releasefiles processed\n" . $this->numupdated . " of " . $this->releasestocheck . " releases " . ($this->releasestocheck > 0 ? floor($this->numupdated / $this->releasestocheck * 100) . "%" : "") . " changed\n";
}
}
/**
* Update a group back to a specified date.
*/
function backfillGroup($groupArr, $backfillDate = null, $regexOnly = false)
{
$db = new DB();
$binaries = new Binaries();
$n = $this->n;
if ($regexOnly === true) {
echo "Only inserting binaries which match regex{$n}";
$binaries->onlyProcessRegexBinaries = true;
}
$this->startGroup = microtime(true);
$nntp = new Nntp();
$nntpc = new Nntp();
//Make sure we actually have a connection going before doing anything.
if ($nntp->doConnect(5, false, true)) {
echo 'Processing ' . $groupArr['name'] . $n;
$data = $nntp->selectGroup($groupArr['name']);
if ($nntp->isError($data)) {
echo "Could not select group (bad name?): {$groupArr['name']}{$n}";
return;
}
if ($backfillDate) {
$targetpost = $this->daytopost($nntp, $groupArr['name'], $this->dateToDays($backfillDate), TRUE);
} else {
$targetpost = $this->daytopost($nntp, $groupArr['name'], $groupArr['backfill_target'], TRUE);
}
//get targetpost based on days target
if ($groupArr['first_record'] == 0 || $groupArr['backfill_target'] == 0 && !$backfillDate) {
echo "Group " . $groupArr['name'] . " has invalid numbers. Have you run update on it? Have you set the backfill days amount?{$n}";
return;
}
echo "Group " . $data["group"] . ": server has " . $data['first'] . " - " . $data['last'] . ", or ~";
echo (int) (($this->postdate($nntp, $data['last'], FALSE) - $this->postdate($nntp, $data['first'], FALSE)) / 86400);
echo " days." . $n . "Local first = " . $groupArr['first_record'] . " (";
echo (int) ((date('U') - $this->postdate($nntp, $groupArr['first_record'], FALSE)) / 86400);
echo " days). Backfill target of " . ($backfillDate ? date('Y-m-d', $backfillDate) : $groupArr['backfill_target'] . " days") . " is post {$targetpost}.{$n}";
if ($targetpost >= $groupArr['first_record']) {
echo "Nothing to do, we already have the target post.{$n} {$n}";
return "";
}
//get first and last part numbers from newsgroup
if ($targetpost < $data['first']) {
echo "WARNING: Backfill came back as before server's first. Setting targetpost to server first.{$n}";
echo "Skipping Group {$n}";
return "";
}
echo $binaries->onlyProcessRegexBinaries === true ? "Note: Discarding parts that do not match a regex" . $n : "";
//Done with $nntp for now, close it to avoid timeouts.
$nntp->doQuit();
$nntpc->doConnect();
$datac = $nntpc->selectGroup($groupArr['name']);
if ($nntpc->isError($datac)) {
echo "Could not select group (bad name?): {$groupArr['name']}{$n}";
return;
}
//calculate total number of parts
$total = $groupArr['first_record'] - $targetpost;
$done = false;
//set first and last, moving the window by maxxMssgs
$last = $groupArr['first_record'] - 1;
$first = $last - $binaries->messagebuffer + 1;
//set initial "chunk"
if ($targetpost > $first) {
//just in case this is the last chunk we needed
$first = $targetpost;
}
while ($done === false) {
$binaries->startLoop = microtime(true);
echo "Getting " . ($last - $first + 1) . " parts (" . number_format($first - $targetpost) . " in queue)" . $n;
flush();
$success = $binaries->scan($nntpc, $groupArr, $first, $last, 'backfill');
if (!$success) {
return "";
}
$db->exec(sprintf("update groups SET first_record = %s, last_updated = now() WHERE ID = %d", $db->escapeString($first), $groupArr['ID']));
if ($first == $targetpost) {
$done = true;
} else {
//Keep going: set new last, new first, check for last chunk.
$last = $first - 1;
$first = $last - $binaries->messagebuffer + 1;
if ($targetpost > $first) {
$first = $targetpost;
}
}
}
//Done with $nntpc
$nntpc->doQuit();
//Just need $nntp for a quick check on the first_record_postdate
$nntp->doConnect();
//$nntp->selectGroup($groupArr['name']); // some users report having this in keeps backfill working
$first_record_postdate = $this->postdate($nntp, $first, false);
//All done with NNTP.
$nntp->doQuit();
if ($first_record_postdate != "") {
$db->exec(sprintf("update groups SET first_record_postdate = FROM_UNIXTIME(" . $first_record_postdate . "), last_updated = now() WHERE ID = %d", $groupArr['ID']));
}
//Set group's first postdate
$timeGroup = number_format(microtime(true) - $this->startGroup, 2);
echo "Group processed in {$timeGroup} seconds {$n}";
} else {
echo "Failed to get NNTP connection.{$n}";
}
}
/**
* Check for passworded releases, RAR contents and Sample/Media info
*/
public function processAdditional($numToProcess = 0)
{
require_once WWW_DIR . "/lib/nntp.php";
$maxattemptstocheckpassworded = 5;
$postProcPercentage = 10;
// process max(percentage,100) unless numtoprocess provided
$processVideoSample = $this->site->ffmpegpath != '' ? true : false;
$processMediainfo = $this->site->mediainfopath != '' ? true : false;
$processPasswords = $this->site->unrarpath != '' ? true : false;
$processAudioSample = $this->site->saveaudiopreview == 1 ? true : false;
$tmpPath = $this->site->tmpunrarpath;
if (substr($tmpPath, -strlen('/')) != '/') {
$tmpPath = $tmpPath . '/';
}
if (!file_exists($tmpPath)) {
mkdir($tmpPath, 0766, true);
}
$db = new DB();
$nntp = new Nntp();
$nzb = new Nzb();
//
// Get out all releases which have not been checked more than max attempts for password.
//
$sql = sprintf("select r.ID, r.guid, r.name, c.disablepreview from releases r\n\t\t\tleft join category c on c.ID = r.categoryID\n\t\t\twhere (r.passwordstatus between %d and -1)\n\t\t\tor (r.haspreview = -1 and c.disablepreview = 0) order by r.postdate desc %s", ($maxattemptstocheckpassworded + 1) * -1, $numToProcess > 0 ? "limit " . $numToProcess : "");
$result = $db->query($sql);
// chop off right amount if a specific number wasnt provided
if ($numToProcess == 0) {
$result = array_slice($result, 0, round(max(count($result) / $postProcPercentage, min(count($result), 100))));
}
$iteration = $rescount = sizeof($result);
if ($rescount > 0) {
echo "PostPrc : Performing additional post processing on last " . $rescount . " releases ...";
$nntpconnected = false;
foreach ($result as $rel) {
echo $iteration-- . ".";
// Per release defaults
$passStatus = array(Releases::PASSWD_NONE);
$blnTookMediainfo = false;
$blnTookSample = $rel['disablepreview'] == 1 ? true : false;
//only attempt sample if not disabled
if ($blnTookSample) {
$db->exec(sprintf("update releases set haspreview = 0 where id = %d", $rel['ID']));
}
//
// Go through the binaries for this release looking for a rar, a sample, and a mediafile
//
$nzbInfo = new nzbInfo();
$norar = 0;
// only load nzbs and check for rar files if we are doing something with them.
if ($processVideoSample || $processMediainfo || $processPasswords || $processAudioSample) {
$nzbfile = $nzb->getNZBPath($rel['guid'], $this->site->nzbpath);
if (!$nzbInfo->loadFromFile($nzbfile)) {
continue;
}
foreach ($nzbInfo->nzb as $nzbsubject) {
if (preg_match("/\\w\\.r00/i", $nzbsubject['subject'])) {
$norar = 1;
}
}
}
// attempt to process video sample file
if (!empty($nzbInfo->samplefiles) && $processVideoSample && $blnTookSample === false) {
$sampleFile = $nzbInfo->samplefiles[0];
//first detected sample
$sampleMsgids = array_slice($sampleFile['segments'], 0, 1);
//get first segment, increase to get more of the sample
$sampleGroup = $sampleFile['groups'][0];
//echo "PostPrc : Fetching ".implode($sampleMsgids, ', ')." from {$sampleGroup}\n";
if (!$nntpconnected) {
$nntpconnected = $nntp->doConnect();
}
$sampleBinary = $nntp->getMessages($sampleGroup, $sampleMsgids);
if ($sampleBinary === false) {
echo "\nPostPrc : Couldnt fetch sample\n";
} else {
$samplefile = $tmpPath . 'sample.avi';
file_put_contents($samplefile, $sampleBinary);
$blnTookSample = $this->getSample($tmpPath, $this->site->ffmpegpath, $rel['guid']);
if ($blnTookSample) {
$this->updateReleaseHasPreview($rel['guid']);
}
unlink($samplefile);
}
unset($sampleBinary);
}
// attempt to process loose media file
if (!empty($nzbInfo->mediafiles) && ($processVideoSample && $blnTookSample === false || $processMediainfo)) {
$mediaFile = $nzbInfo->mediafiles[0];
//first detected media file
$mediaMsgids = array_slice($mediaFile['segments'], 0, 2);
//get first two segments
$mediaGroup = $mediaFile['groups'][0];
//echo "PostPrc : Fetching ".implode($mediaMsgids, ', ')." from {$mediaGroup}\n";
if (!$nntpconnected) {
$nntpconnected = $nntp->doConnect();
}
$mediaBinary = $nntp->getMessages($mediaGroup, $mediaMsgids);
if ($mediaBinary === false) {
echo "\nPostPrc : Couldnt fetch media file\n";
} else {
$mediafile = $tmpPath . 'sample.avi';
file_put_contents($mediafile, $mediaBinary);
if ($processVideoSample && $blnTookSample === false) {
$blnTookSample = $this->getSample($tmpPath, $this->site->ffmpegpath, $rel['guid']);
if ($blnTookSample) {
$this->updateReleaseHasPreview($rel['guid']);
}
}
if ($processMediainfo) {
$blnTookMediainfo = $this->getMediainfo($tmpPath, $this->site->mediainfopath, $rel['ID']);
}
unlink($mediafile);
}
unset($mediaBinary);
}
// attempt to process audio sample file
if (!empty($nzbInfo->audiofiles) && $processAudioSample && $blnTookSample === false) {
$audioFile = $nzbInfo->audiofiles[0];
//first detected audio file
$audioMsgids = array_slice($audioFile['segments'], 0, 1);
//get first segment
$audioGroup = $audioFile['groups'][0];
//echo "PostPrc : Fetching ".implode($audioMsgids, ', ')." from {$audioGroup}\n";
if (!$nntpconnected) {
$nntpconnected = $nntp->doConnect();
}
$audioBinary = $nntp->getMessages($audioGroup, $audioMsgids);
if ($audioBinary === false) {
echo "\nPostPrc : Couldnt fetch audio sample\n";
} else {
$audiofile = $tmpPath . 'sample.mp3';
file_put_contents($audiofile, $audioBinary);
$blnTookSample = $this->getAudioSample($tmpPath, $rel['guid']);
if ($blnTookSample !== false) {
$this->updateReleaseHasPreview($rel['guid'], 2);
}
if ($processMediainfo) {
$blnTookMediainfo = $this->getMediainfo($tmpPath, $this->site->mediainfopath, $rel['ID']);
}
if ($this->site->lamepath != "") {
$this->lameAudioSample($this->site->lamepath, $rel['guid']);
}
unlink($audiofile);
}
unset($audioBinary);
}
if (!empty($nzbInfo->rarfiles) && ($this->site->checkpasswordedrar > 0 || ($processVideoSample || $processAudioSample) && $blnTookSample === false || $processMediainfo)) {
$mysqlkeepalive = 0;
foreach ($nzbInfo->rarfiles as $rarFile) {
//dont process any more rars if a passworded rar has been detected and the site is set to automatically delete them
if ($this->site->deletepasswordedrelease == 1 && max($passStatus) == Releases::PASSWD_RAR) {
echo "-Skipping processing of rar {$rarFile['subject']} as this release has already been marked as passworded.\n";
continue;
}
$rarMsgids = array_slice($rarFile['segments'], 0, 1);
//get first segment
$rarGroup = $rarFile['groups'][0];
//echo "PostPrc : Fetching ".implode($rarMsgids, ', ')." from {$rarGroup} (".++$mysqlkeepalive.")\n";
if (!$nntpconnected) {
$nntpconnected = $nntp->doConnect();
}
$fetchedBinary = $nntp->getMessages($rarGroup, $rarMsgids);
if ($fetchedBinary === false) {
//echo "\nPostPrc : Failed fetching rar file\n";
$db->exec(sprintf("update releases set passwordstatus = passwordstatus - 1 where ID = %d", $rel['ID']));
continue;
} else {
$relFiles = $this->processReleaseFiles($fetchedBinary, $rel['ID']);
if ($this->site->checkpasswordedrar > 0 && $processPasswords) {
$passStatus[] = $this->processReleasePasswords($fetchedBinary, $tmpPath, $this->site->unrarpath, $this->site->checkpasswordedrar);
}
// we need to unrar the fetched binary if checkpasswordedrar wasnt 2
if ($this->site->checkpasswordedrar < 2 && $processPasswords) {
$rarfile = $tmpPath . 'rarfile.rar';
file_put_contents($rarfile, $fetchedBinary);
$execstring = '"' . $this->site->unrarpath . '" e -ai -ep -c- -id -r -kb -p- -y -inul "' . $rarfile . '" "' . $tmpPath . '"';
$output = runCmd($execstring, false, true);
unlink($rarfile);
}
if ($processVideoSample && $blnTookSample === false) {
$blnTookSample = $this->getSample($tmpPath, $this->site->ffmpegpath, $rel['guid']);
if ($blnTookSample) {
$this->updateReleaseHasPreview($rel['guid']);
}
}
$blnTookAudioSample = false;
if ($processAudioSample && $blnTookSample === false) {
$blnTookSample = $this->getAudioSample($tmpPath, $rel['guid']);
if ($blnTookSample) {
$blnTookAudioSample = true;
$this->updateReleaseHasPreview($rel['guid'], 2);
}
}
if ($processMediainfo && $blnTookMediainfo === false) {
$blnTookMediainfo = $this->getMediainfo($tmpPath, $this->site->mediainfopath, $rel['ID']);
}
//
// Has to be done after mediainfo
//
if ($blnTookAudioSample && $this->site->lamepath != "") {
$this->lameAudioSample($this->site->lamepath, $rel['guid']);
}
if ($mysqlkeepalive % 25 == 0) {
$db->query("select 1");
}
}
//clean up all files
foreach (glob($tmpPath . '*') as $v) {
unlink($v);
}
}
//end foreach msgid
} elseif (empty($nzbInfo->rarfiles) && $norar == 1) {
$passStatus[] = Releases::PASSWD_POTENTIAL;
}
$hpsql = '';
if (!$blnTookSample) {
$hpsql = ', haspreview = 0';
}
$sql = sprintf("update releases set passwordstatus = %d %s where ID = %d", max($passStatus), $hpsql, $rel["ID"]);
$db->exec($sql);
}
//end foreach result
if ($nntpconnected) {
$nntp->doQuit();
}
echo "\n";
}
}
/**
* Download a range of usenet messages. Store binaries with subjects matching a
* specific pattern in the database.
*/
function scan($nntp, $groupArr, $first, $last, $type = 'update')
{
$db = new Db();
$releaseRegex = new ReleaseRegex();
$n = $this->n;
$this->startHeaders = microtime(true);
if ($this->compressedHeaders) {
$nntpn = new Nntp();
$nntpn->doConnect(5, false, true);
$response = $nntpn->_sendCommand('XFEATURE COMPRESS GZIP');
if ($nntpn->isError($response) || $response != 290) {
$response2 = $nntpn->_sendCommand('XZVER');
if ($nntpn->isError($response2) || $response2 != 412) {
$msgs = $nntp->getOverview($first . "-" . $last, true, false);
$nntpn->doQuit();
} else {
$msgs = $nntp->getXOverview($first . "-" . $last, true, false);
$nntpn->doQuit();
}
} else {
$msgs = $nntp->getOverview($first . "-" . $last, true, false);
$nntpn->doQuit();
}
} else {
$msgs = $nntp->getOverview($first . "-" . $last, true, false);
}
if ($nntp->isError($msgs) && ($msgs->code == 400 || $msgs->code == 503)) {
echo "NNTP connection timed out. Reconnecting...{$n}";
if (!$nntp->doConnect()) {
// TODO: What now?
echo "Failed to get NNTP connection.{$n}";
return;
}
$nntp->selectGroup($groupArr['name']);
if ($this->compressedHeaders) {
$nntpn = new Nntp();
$nntpn->doConnect(5, false, true);
$response = $nntpn->_sendCommand('XFEATURE COMPRESS GZIP');
if ($nntpn->isError($response) || $response != 290) {
$response2 = $nntpn->_sendCommand('XZVER');
if ($nntpn->isError($response2) || $response2 != 412) {
$msgs = $nntp->getOverview($first . "-" . $last, true, false);
$nntpn->doQuit();
} else {
$msgs = $nntp->getXOverview($first . "-" . $last, true, false);
$nntpn->doQuit();
}
} else {
$msgs = $nntp->getOverview($first . "-" . $last, true, false);
$nntpn->doQuit();
}
} else {
$msgs = $nntp->getOverview($first . "-" . $last, true, false);
}
}
$rangerequested = range($first, $last);
$msgsreceived = array();
$msgsblacklisted = array();
$msgsignored = array();
$msgsinserted = array();
$msgsnotinserted = array();
$timeHeaders = number_format(microtime(true) - $this->startHeaders, 2);
if ($nntp->isError($msgs)) {
echo "Error {$msgs->code}: {$msgs->message}{$n}";
echo "Skipping group{$n}";
return false;
}
$this->startUpdate = microtime(true);
if (is_array($msgs)) {
//loop headers, figure out parts
foreach ($msgs as $msg) {
if (!isset($msg['Number'])) {
continue;
}
$msgsreceived[] = $msg['Number'];
$msgPart = $msgTotalParts = 0;
$pattern = '|\\((\\d+)[\\/](\\d+)\\)|i';
preg_match_all($pattern, $msg['Subject'], $matches, PREG_PATTERN_ORDER);
$matchcnt = sizeof($matches[0]);
for ($i = 0; $i < $matchcnt; $i++) {
$msgPart = $matches[1][$i];
$msgTotalParts = $matches[2][$i];
}
if (!isset($msg['Subject']) || $matchcnt == 0) {
$msgsignored[] = $msg['Number'];
continue;
}
if ((int) $msgPart > 0 && (int) $msgTotalParts > 0) {
$subject = utf8_encode(trim(preg_replace('|\\(' . $msgPart . '[\\/]' . $msgTotalParts . '\\)|i', '', $msg['Subject'])));
if (!isset($this->message[$subject])) {
$this->message[$subject] = $msg;
$this->message[$subject]['MaxParts'] = (int) $msgTotalParts;
$this->message[$subject]['Date'] = strtotime($this->message[$subject]['Date']);
}
if ((int) $msgPart > 0) {
$this->message[$subject]['Parts'][(int) $msgPart] = array('Message-ID' => substr($msg['Message-ID'], 1, -1), 'number' => $msg['Number'], 'part' => (int) $msgPart, 'size' => $msg['Bytes']);
$this->message[$subject]['PartNumbers'][(int) $msgPart] = $msg['Number'];
}
}
}
unset($msg);
unset($msgs);
$count = 0;
$updatecount = 0;
$partcount = 0;
$rangenotreceived = array_diff($rangerequested, $msgsreceived);
if ($type != 'partrepair') {
echo "Received " . sizeof($msgsreceived) . " articles of " . ($last - $first + 1) . " requested, " . sizeof($msgsignored) . " not binaries {$n}";
}
if ($type == 'update' && sizeof($msgsreceived) == 0) {
echo "Error: Server did not return any articles.{$n}";
echo "Skipping group{$n}";
return false;
}
if (sizeof($rangenotreceived) > 0) {
switch ($type) {
case 'backfill':
//don't add missing articles
break;
case 'partrepair':
case 'update':
default:
$this->addMissingParts($rangenotreceived, $groupArr['ID']);
break;
}
echo "Server did not return " . count($rangenotreceived) . " article(s).{$n}";
}
if (isset($this->message) && count($this->message)) {
$groupRegexes = $releaseRegex->getForGroup($groupArr['name']);
//insert binaries and parts into database. when binary already exists; only insert new parts
foreach ($this->message as $subject => $data) {
//Filter binaries based on black/white list
if ($this->isBlackListed($data, $groupArr['name'])) {
$msgsblacklisted[] = count($data['Parts']);
if ($type == 'partrepair') {
$partIds = array();
foreach ($data['Parts'] as $partdata) {
$partIds[] = $partdata['number'];
}
$db->exec(sprintf("DELETE FROM partrepair WHERE numberID IN (%s) AND groupID=%d", implode(',', $partIds), $groupArr['ID']));
}
continue;
}
if (isset($data['Parts']) && count($data['Parts']) > 0 && $subject != '') {
//Check for existing binary
$binaryID = 0;
$binaryHash = md5($subject . $data['From'] . $groupArr['ID']);
$res = $db->queryOneRow(sprintf("SELECT ID FROM binaries WHERE binaryhash = %s", $db->escapeString($binaryHash)));
if (!$res) {
//Apply Regexes
$regexMatches = array();
foreach ($groupRegexes as $groupRegex) {
$regexCheck = $releaseRegex->performMatch($groupRegex, $subject);
if ($regexCheck !== false) {
$regexMatches = $regexCheck;
break;
}
}
$sql = '';
if (!empty($regexMatches)) {
$relparts = explode("/", $regexMatches['parts']);
$sql = sprintf("INSERT INTO binaries (name, fromname, date, xref, totalparts, groupID, procstat, categoryID, regexID, reqID, relpart, reltotalpart, binaryhash, relname, dateadded) VALUES (%s, %s, FROM_UNIXTIME(%s), %s, %s, %d, %d, %s, %d, %s, %d, %d, %s, %s, now())", $db->escapeString($subject), $db->escapeString(utf8_encode($data['From'])), $db->escapeString($data['Date']), $db->escapeString($data['Xref']), $db->escapeString($data['MaxParts']), $groupArr['ID'], Releases::PROCSTAT_TITLEMATCHED, $regexMatches['regcatid'], $regexMatches['regexID'], $db->escapeString($regexMatches['reqID']), $relparts[0], $relparts[1], $db->escapeString($binaryHash), $db->escapeString(str_replace('_', ' ', $regexMatches['name'])));
} elseif ($this->onlyProcessRegexBinaries === false) {
$sql = sprintf("INSERT INTO binaries (name, fromname, date, xref, totalparts, groupID, binaryhash, dateadded) VALUES (%s, %s, FROM_UNIXTIME(%s), %s, %s, %d, %s, now())", $db->escapeString($subject), $db->escapeString(utf8_encode($data['From'])), $db->escapeString($data['Date']), $db->escapeString($data['Xref']), $db->escapeString($data['MaxParts']), $groupArr['ID'], $db->escapeString($binaryHash));
} elseif ($type == 'partrepair') {
$partIds = array();
foreach ($data['Parts'] as $partdata) {
$partIds[] = $partdata['number'];
}
$db->exec(sprintf("DELETE FROM partrepair WHERE numberID IN (%s) AND groupID=%d", implode(',', $partIds), $groupArr['ID']));
continue;
}
if ($sql != '') {
$binaryID = $db->queryInsert($sql);
$count++;
if ($count % 500 == 0) {
echo "{$count} bin adds...";
}
}
} else {
$binaryID = $res["ID"];
$updatecount++;
if ($updatecount % 500 == 0) {
echo "{$updatecount} bin updates...";
}
}
if ($binaryID != 0) {
$partParams = array();
$partNumbers = array();
$totsize = 0;
foreach ($data['Parts'] as $partdata) {
$partcount++;
$totsize += $partdata['size'];
$partParams[] = sprintf("(%d, %s, %s, %s, %s)", $binaryID, $db->escapeString($partdata['Message-ID']), $db->escapeString($partdata['number']), $db->escapeString(round($partdata['part'])), $db->escapeString($partdata['size']));
$partNumbers[] = $partdata['number'];
}
$partSql = "INSERT INTO parts (binaryID, messageID, number, partnumber, size) VALUES " . implode(', ', $partParams);
$pidata = $db->queryInsert($partSql, false);
if (!$pidata) {
$msgsnotinserted = array_merge($msgsnotinserted, $partNumbers);
} else {
$msgsinserted = array_merge($msgsinserted, $partNumbers);
}
// update bin size
$upsql = sprintf("update binaries set size = size + %d where ID = %d", $totsize, $binaryID);
$db->exec($upsql);
}
}
}
//TODO: determine whether to add to missing articles if insert failed
if (sizeof($msgsnotinserted) > 0) {
echo 'WARNING: ' . count($msgsnotinserted) . ' Parts failed to insert' . $n;
$this->addMissingParts($msgsnotinserted, $groupArr['ID']);
}
if ($count >= 500 || $updatecount >= 500) {
echo $n;
}
//line break for bin adds output
}
$timeUpdate = number_format(microtime(true) - $this->startUpdate, 2);
$timeLoop = number_format(microtime(true) - $this->startLoop, 2);
if (sizeof($msgsblacklisted) > 0) {
echo "Blacklisted " . array_sum($msgsblacklisted) . " parts in " . sizeof($msgsblacklisted) . " binaries" . $n;
}
if ($type != 'partrepair') {
echo number_format($count) . ' new, ' . number_format($updatecount) . ' updated, ' . number_format($partcount) . ' parts.';
echo " {$timeHeaders} headers, {$timeUpdate} update, {$timeLoop} range.{$n}";
}
unset($this->message);
unset($data);
return $last;
} else {
echo "Error: Can't get parts from server (msgs not array) {$n}";
echo "Skipping group{$n}";
return false;
}
}
public function processPasswordedReleases($echooutput = false)
{
$maxattemptstocheckpassworded = 5;
$potentiallypasswordedfileregex = "/\\.(ace|cab|tar|gz)\$/i";
$numfound = 0;
$numpasswd = 0;
$numpot = 0;
$numnone = 0;
$db = new DB();
$nntp = new Nntp();
$rar = new RarInfo();
$rar->setMaxBytes(4000);
if ($echooutput) {
echo "Checking for passworded releases.\n\n";
}
//
// Get out all releases which have not been checked more than max attempts for password.
//
$result = $db->query(sprintf("select ID from releases where passwordstatus between %d and -1", ($maxattemptstocheckpassworded + 1) * -1));
if (count($result) > 0) {
$nntp->doConnect();
foreach ($result as $row) {
//
// get out all files for this release, if it contains no rars, mark as Releases::PASSWD_NONE
// if it contains rars, try and retrieve the message for the first rar and inspect its filename
// if rar file is encrypted set as Releases::PASSWD_RAR, if it contains an ace/cab etc
// mark as Releases::PASSWD_POTENTIAL, otherwise set as Releases::PASSWD_NONE.
//
$numfound++;
//
// Go through the binaries for this release looking for a rar
//
$binresult = $db->query(sprintf("select binaries.ID, binaries.name, groups.name as groupname from binaries inner join groups on groups.ID = binaries.groupID where releaseID = %d order by relpart", $row["ID"]));
$msgid = -1;
$bingroup = "";
foreach ($binresult as $binrow) {
if (preg_match("/\\W(?:part0*1|(?!part\\d+)[^.]+)\\.rar(?!\\.)/i", $binrow["name"])) {
$bingroup = $binrow["groupname"];
echo "Checking " . $binrow["name"] . " for password.\n";
$part = $db->queryOneRow(sprintf("select messageID from parts where binaryID = %d order by partnumber", $binrow["ID"]));
if (isset($part["messageID"])) {
$msgid = $part["messageID"];
break;
}
}
}
$passStatus = Releases::PASSWD_NONE;
//
// no part of binary found matching a rar, so it cant be progressed further
//
if ($msgid != -1) {
$fetchedBinary = $nntp->getMessage($bingroup, $msgid);
if ($fetchedBinary === false) {
$db->query(sprintf("update releases set passwordstatus = passwordstatus - 1 where ID = %d", $row["ID"]));
continue;
}
if ($rar->setData($fetchedBinary)) {
//
// whole archive password protected
//
if ($rar->isEncrypted) {
$passStatus = Releases::PASSWD_RAR;
} else {
$files = $rar->getFileList();
foreach ($files as $file) {
//
// individual file rar passworded
//
if ($file['pass'] == true) {
$passStatus = Releases::PASSWD_RAR;
break;
} else {
if (preg_match($potentiallypasswordedfileregex, $file["name"])) {
$passStatus = Releases::PASSWD_POTENTIAL;
break;
}
}
}
}
}
}
//
// increment reporting stats
//
if ($passStatus == Releases::PASSWD_RAR) {
$numpasswd++;
} elseif ($passStatus == Releases::PASSWD_POTENTIAL) {
$numpot++;
} else {
$numnone++;
}
$db->query(sprintf("update releases set passwordstatus = %d where ID = %d", $passStatus, $row["ID"]));
}
$nntp->doQuit();
}
if ($echooutput) {
echo sprintf("Finished checking for passwords for %d releases (%d passworded, %d potential, %d none).\n\n", $numfound, $numpasswd, $numpot, $numnone);
}
}
/**
* Get nzpre data from usenet and parse.
*/
public function nzpreUpdate()
{
require_once WWW_DIR . "/lib/nntp.php";
$s = new Sites();
$site = $s->get();
if (empty($site->nzpregroup) || empty($site->nzpresubject) || empty($site->nzpreposter) || empty($site->nzprefield) || empty($site->nzprekey)) {
return false;
}
if ($this->echooutput) {
echo "Predb : Checking for new pre data ";
}
$db = new DB();
$nntp = new Nntp();
if (!$nntp->doConnect()) {
echo "Failed to get NNTP connection\n";
return false;
}
$ret = $groupData = $nntp->selectGroup($site->nzpregroup);
if ($nntp->isError($ret)) {
echo "Predb : Error " . $ret->getMessage() . "\n";
return false;
}
$ret = $groupMsgs = $nntp->getOverview($groupData['last'] - (!empty($site->nzprearticles) ? $site->nzprearticles : 500) . '-' . $groupData['last']);
if ($nntp->isError($ret)) {
echo "Predb : Error " . $ret->getMessage() . "\n";
return false;
}
$added_updated = 0;
$nzprekey = $site->nzprekey;
while (strlen($nzprekey) < 1024) {
$nzprekey = $nzprekey . $nzprekey;
}
$cnt = !empty($site->nzprearticles) ? $site->nzprearticles : 500;
foreach ($groupMsgs as $groupMsg) {
if ($cnt % 50 == 0 && $cnt != 0 && $this->echooutput) {
echo $cnt . "..";
}
$cnt--;
if (preg_match('/^' . $site->nzpresubject . '$/', $groupMsg['Subject']) && preg_match('/^' . $site->nzpreposter . '$/', $groupMsg['From'])) {
$ret = $msgHeader = $nntp->getHeader($groupMsg['Message-ID']);
if ($nntp->isError($ret)) {
continue;
}
for ($i = 0; $i < count($msgHeader); $i++) {
if (preg_match('/^' . $site->nzprefield . ': /', $msgHeader[$i])) {
if ($nzpreParse = $this->nzpreParse(str_replace($site->nzprefield . ': ', '', $msgHeader[$i]), $nzprekey)) {
if ($this->updatePreDB($db, $nzpreParse)) {
$added_updated++;
}
}
break;
}
}
}
}
$nntp->disconnect();
if ($this->echooutput) {
echo "\nPredb : Added/Updated " . $added_updated . " records\n";
}
}
public function processAdditional()
{
$maxattemptstocheckpassworded = 5;
$processSample = $this->site->ffmpegpath != '' ? true : false;
$processMediainfo = $this->site->mediainfopath != '' ? true : false;
$processPasswords = $this->site->unrarpath != '' ? true : false;
$tmpPath = $this->site->tmpunrarpath;
if (substr($tmpPath, -strlen('/')) != '/') {
$tmpPath = $tmpPath . '/';
}
$db = new DB();
$nntp = new Nntp();
//
// Get out all releases which have not been checked more than max attempts for password.
//
$result = $db->query(sprintf("select r.ID, r.guid, r.name, c.disablepreview from releases r \n\t\t\tleft join category c on c.ID = r.categoryID\n\t\t\twhere (r.passwordstatus between %d and -1)\n\t\t\tor (r.haspreview = -1 and c.disablepreview = 0)\t\t\t\n\t\t", ($maxattemptstocheckpassworded + 1) * -1));
$rescount = sizeof($result);
echo "Post-processing {$rescount} releases\n";
if ($rescount > 0) {
$nntp->doConnect();
foreach ($result as $rel) {
// Per release defaults
$passStatus = array(Releases::PASSWD_NONE);
$blnTookMediainfo = false;
$blnTookSample = $rel['disablepreview'] == 1 ? true : false;
//only attempt sample if not disabled
if ($blnTookSample) {
$db->query(sprintf("update releases set haspreview = 0 where id = %d", $rel['ID']));
}
//
// Go through the binaries for this release looking for a rar, a sample, and a mediafile
//
$sql = sprintf("select binaries.ID, binaries.name, groups.name as groupname from binaries inner join groups on groups.ID = binaries.groupID where releaseID = %d order by relpart", $rel["ID"]);
$binresult = $db->query($sql);
$msgid = array();
$samplemsgid = $mediamsgid = -1;
$bingroup = $samplegroup = $mediagroup = "";
$norar = 0;
echo "\n" . $rel['name'] . " has " . sizeof($binresult) . " binaries\n";
foreach ($binresult as $binrow) {
if (preg_match("/\\W\\.r00/i", $binrow["name"])) {
$norar = 1;
}
if (preg_match("/sample/i", $binrow["name"]) && !preg_match("/\\.par2|\\.srs/i", $binrow["name"])) {
echo "Detected sample file " . $binrow["name"] . "\n";
$samplepart = $db->queryOneRow(sprintf("select messageID from parts where binaryID = %d order by partnumber limit 1", $binrow["ID"]));
if (isset($samplepart["messageID"])) {
$samplegroup = $binrow["groupname"];
$samplemsgid = $samplepart["messageID"];
}
}
if (preg_match('/\\.(' . $this->mediafileregex . ')[\\. "\\)\\]]/i', $binrow["name"]) && !preg_match("/\\.par2|\\.srs/i", $binrow["name"])) {
$mediapart = $db->queryOneRow(sprintf("select messageID from parts where binaryID = %d order by partnumber limit 1", $binrow["ID"]));
if (isset($mediapart["messageID"]) && $mediapart['messageID'] != $samplemsgid) {
echo "Detected media file " . $binrow["name"] . "\n";
$mediagroup = $binrow["groupname"];
$mediamsgid = $mediapart["messageID"];
}
}
if (preg_match("/\\W(?:part0*1|(?!part\\d+)[^.]+)\\.rar(?!\\.)/i", $binrow["name"]) && !preg_match("/[-_\\.]sub/i", $binrow["name"])) {
echo "Detected RAR " . $binrow["name"] . "\n";
$part = $db->queryOneRow(sprintf("select messageID from parts where binaryID = %d order by partnumber limit 1", $binrow["ID"]));
if (isset($part["messageID"])) {
$bingroup = $binrow["groupname"];
$msgid[] = $part["messageID"];
}
}
}
// attempt to process sample file
if ($samplemsgid != -1 && $processSample && $blnTookSample === false) {
echo "Processing Sample\n";
echo "-Fetching binary {$samplemsgid}\n";
$sampleBinary = $nntp->getMessage($samplegroup, $samplemsgid);
if ($sampleBinary === false) {
echo "-Couldnt fetch binary {$samplemsgid}\n";
$samplemsgid = -1;
} else {
$samplefile = $tmpPath . 'sample.avi';
file_put_contents($samplefile, $sampleBinary);
$blnTookSample = $this->getSample($tmpPath, $this->site->ffmpegpath, $rel['guid']);
if ($blnTookSample) {
$this->updateReleaseHasPreview($rel['guid']);
}
unlink($samplefile);
}
unset($sampleBinary);
}
// attempt to process loose media file
if ($mediamsgid != -1 && ($processSample && $blnTookSample === false || $processMediainfo)) {
echo "Processing loose media file\n";
echo "-Fetching binary {$mediamsgid}\n";
$mediaBinary = $nntp->getMessage($mediagroup, $mediamsgid);
if ($mediaBinary === false) {
echo "-Couldnt fetch binary {$mediamsgid}\n";
$mediamsgid = -1;
// can't get the media so we'll try from the .rar
} else {
$mediafile = $tmpPath . 'sample.avi';
file_put_contents($mediafile, $mediaBinary);
if ($processSample && $blnTookSample === false) {
$blnTookSample = $this->getSample($tmpPath, $this->site->ffmpegpath, $rel['guid']);
if ($blnTookSample) {
$this->updateReleaseHasPreview($rel['guid']);
}
}
if ($processMediainfo) {
$blnTookMediainfo = $this->getMediainfo($tmpPath, $this->site->mediainfopath, $rel['ID']);
}
unlink($mediafile);
}
unset($mediaBinary);
}
if (!empty($msgid) && ($this->site->checkpasswordedrar > 0 || $processSample && $blnTookSample === false || $processMediainfo)) {
echo "Processing RAR files\n";
$mysqlkeepalive = 0;
foreach ($msgid as $mid) {
echo "-Fetching binary " . $mid . " (" . ++$mysqlkeepalive . ")\n";
$fetchedBinary = $nntp->getMessage($bingroup, $mid);
if ($fetchedBinary === false) {
echo "-Failed fetching binary\n";
$db->query(sprintf("update releases set passwordstatus = passwordstatus - 1 where ID = %d", $rel['ID']));
continue;
} else {
$relFiles = $this->processReleaseFiles($fetchedBinary, $rel['ID']);
if ($this->site->checkpasswordedrar > 0 && $processPasswords) {
$passStatus[] = $this->processReleasePasswords($fetchedBinary, $tmpPath, $this->site->unrarpath, $this->site->checkpasswordedrar);
}
// we need to unrar the fetched binary if checkpasswordedrar wasnt 2
if ($this->site->checkpasswordedrar < 2 && $processPasswords) {
$rarfile = $tmpPath . 'rarfile.rar';
file_put_contents($rarfile, $fetchedBinary);
$execstring = '"' . $this->site->unrarpath . '" e -ai -ep -c- -id -r -kb -p- -y -inul "' . $rarfile . '" "' . $tmpPath . '"';
$output = runCmd($execstring);
unlink($rarfile);
}
if ($processSample && $blnTookSample === false) {
$blnTookSample = $this->getSample($tmpPath, $this->site->ffmpegpath, $rel['guid']);
if ($blnTookSample) {
$this->updateReleaseHasPreview($rel['guid']);
}
}
if ($processMediainfo && $blnTookMediainfo === false) {
$blnTookMediainfo = $this->getMediainfo($tmpPath, $this->site->mediainfopath, $rel['ID']);
}
if ($mysqlkeepalive % 25 == 0) {
$db->query("select 1");
}
}
//clean up all files
foreach (glob($tmpPath . '*') as $v) {
unlink($v);
}
}
//end foreach msgid
} elseif (empty($msgid) && $norar == 1) {
$passStatus[] = Releases::PASSWD_POTENTIAL;
}
$hpsql = '';
if (!$blnTookSample) {
$hpsql = ', haspreview = 0';
}
$sql = sprintf("update releases set passwordstatus = %d %s where ID = %d", max($passStatus), $hpsql, $rel["ID"]);
$db->query($sql);
}
//end foreach result
$nntp->doQuit();
}
}
public function processNfoFiles($processImdb = 1, $processTvrage = 1)
{
$ret = 0;
$db = new DB();
$nntp = new Nntp();
$res = $db->queryDirect(sprintf("SELECT rn.*, r.searchname FROM releasenfo rn left outer join releases r ON r.ID = rn.releaseID WHERE rn.nfo IS NULL AND rn.attempts < 5"));
if (mysql_num_rows($res) > 0) {
if ($this->echooutput) {
echo "Processing " . mysql_num_rows($res) . " nfos\n";
}
$nntp->doConnect();
while ($arr = mysql_fetch_assoc($res)) {
$fetchedBinary = $nntp->getBinary($arr['binaryID'], true);
if ($fetchedBinary !== false) {
//insert nfo into database
$db->query(sprintf("UPDATE releasenfo SET nfo = compress(%s) WHERE ID = %d", $db->escapeString($fetchedBinary), $arr["ID"]));
$ret++;
$imdbId = $this->parseImdb($fetchedBinary);
if ($imdbId !== false) {
//update release with imdb id
$db->query(sprintf("UPDATE releases SET imdbID = %s WHERE ID = %d", $db->escapeString($imdbId), $arr["releaseID"]));
//if set scan for imdb info
if ($processImdb == 1) {
$movie = new Movie($this->echooutput);
//check for existing movie entry
$movCheck = $movie->getMovieInfo($imdbId);
if ($movCheck === false || isset($movCheck['updateddate']) && time() - strtotime($movCheck['updateddate']) > 2592000) {
$movieId = $movie->updateMovieInfo($imdbId);
}
}
}
$rageId = $this->parseRageId($fetchedBinary);
if ($rageId !== false) {
//if set scan for tvrage info
if ($processTvrage == 1) {
$tvrage = new Tvrage($this->echooutput);
$show = $tvrage->parseNameEpSeason($arr['searchname']);
if (is_array($show) && $show['name'] != '') {
// update release with season, ep, and airdate info (if available) from releasetitle
$tvrage->updateEpInfo($show, $arr['releaseID']);
$rid = $tvrage->getByRageID($rageId);
if (!$rid) {
$tvrShow = $tvrage->getRageInfoFromService($rageId);
$tvrage->updateRageInfo($rageId, $show, $tvrShow, $arr['releaseID']);
}
}
}
}
} else {
if ($this->echooutput) {
echo "NFO download failed - release " . $arr['releaseID'] . " on attempt " . $arr["attempts"]++ . "\n";
}
//nfo download failed, increment attempts
$db->query(sprintf("UPDATE releasenfo SET attempts = attempts+1 WHERE ID = %d", $arr["ID"]));
}
if ($ret != 0 && $this->echooutput && $ret % 5 == 0) {
echo "-processed " . $ret . " nfos\n";
}
}
$nntp->doQuit();
}
//remove nfo that we cant fetch after 5 attempts
$db->query("DELETE FROM releasenfo WHERE nfo IS NULL AND attempts >= 5");
if ($this->echooutput) {
echo $ret . " nfo files processed\n";
}
return $ret;
}
