Example #1
0
 /**
  * Restart the NNTP connection if an error occurs in the selectGroup
  * function, if it does not restart display the error.
  *
  * @param NNTP   $nntp  Instance of class NNTP.
  * @param string $group Name of the group.
  * @param bool   $comp Use compression or not?
  *
  * @return mixed On success : (array)  The group summary.
  *               On Failure : (object) PEAR_Error.
  *
  * @access public
  */
 public function dataError($nntp, $group, $comp = true)
 {
     // Disconnect.
     $nntp->doQuit();
     // Try reconnecting. This uses another round of max retries.
     if ($nntp->doConnect($comp) !== true) {
         if ($this->_debugBool) {
             $this->_debugging->log(get_class(), __FUNCTION__, 'Unable to reconnect to usenet!', Logger::LOG_NOTICE);
         }
         return $this->throwError('Unable to reconnect to usenet!');
     }
     // Try re-selecting the group.
     $data = $nntp->selectGroup($group);
     if ($this->isError($data)) {
         $message = "Code {$data->code}: {$data->message}\nSkipping group: {$group}";
         if ($this->_debugBool) {
             $this->_debugging->log(get_class(), __FUNCTION__, $message, Logger::LOG_NOTICE);
         }
         if ($this->_echo) {
             $this->pdo->log->doEcho($this->pdo->log->error($message), true);
         }
         $nntp->doQuit();
     }
     return $data;
 }
Example #2
0
 /**
  * Download a range of usenet messages. Store binaries with subjects matching a
  * specific pattern in the database.
  *
  * @param        $groupArr
  * @param        $first
  * @param        $last
  * @param string $type
  *
  * @return array
  */
 function scan($groupArr, $first, $last, $type = 'update')
 {
     $db = new Settings();
     $releaseRegex = new ReleaseRegex();
     $n = $this->n;
     // Check if MySQL tables exist, create if they do not, get their names at the same time.
     $tableNames = $this->_groups->getCBPTableNames($this->_tablePerGroup, $groupArr['id']);
     $partRepair = $type === 'partrepair';
     $returnArray = [];
     // Download the headers.
     if ($partRepair === true) {
         // This is slower but possibly is better with missing headers.
         $msgs = $this->_nntp->getOverview($first . '-' . $last, true, false);
     } else {
         $msgs = $this->_nntp->getXOVER($first . '-' . $last);
     }
     // If there was an error, try to reconnect.
     if ($this->_nntp->isError($msgs)) {
         // Increment if part repair and return false.
         if ($partRepair === true) {
             $this->_pdo->queryExec(sprintf('UPDATE %s SET attempts = attempts + 1 WHERE groupid = %d AND numberid %s', $tableNames['prname'], $groupArr['id'], $first == $last ? '= ' . $first : 'IN (' . implode(',', range($first, $last)) . ')'));
             return $returnArray;
         }
         // This is usually a compression error, so try disabling compression.
         $this->_nntp->doQuit();
         if ($this->_nntp->doConnect(false) !== true) {
             return $returnArray;
         }
         // Re-select group, download headers again without compression and re-enable compression.
         $this->_nntp->selectGroup($groupArr['name']);
         $msgs = $this->_nntp->getXOVER($first . '-' . $last);
         $this->_nntp->enableCompression();
         // Check if the non-compression headers have an error.
         if ($this->_nntp->isError($msgs)) {
             $this->log("Code {$msgs->code}: {$msgs->message}\nSkipping group: {$groupArr['name']}", 'scan', \Logger::LOG_WARNING, 'error');
             return $returnArray;
         }
     }
     $rangerequested = range($first, $last);
     $msgsreceived = array();
     $msgsblacklisted = array();
     $msgsignored = array();
     $msgsinserted = array();
     $msgsnotinserted = array();
     $timeHeaders = number_format(microtime(true) - $this->startHeaders, 2);
     if ($this->_nntp->isError($msgs)) {
         echo "Error {$msgs->code}: {$msgs->message}{$n}";
         echo "Skipping group{$n}";
         return $returnArray;
     }
     // Check if we got headers.
     $msgCount = count($msgs);
     if ($msgCount < 1) {
         return $returnArray;
     }
     // Get highest and lowest article numbers/dates.
     $iterator1 = 0;
     $iterator2 = $msgCount - 1;
     while (true) {
         if (!isset($returnArray['firstArticleNumber']) && isset($msgs[$iterator1]['Number'])) {
             $returnArray['firstArticleNumber'] = $msgs[$iterator1]['Number'];
             $returnArray['firstArticleDate'] = $msgs[$iterator1]['Date'];
         }
         if (!isset($returnArray['lastArticleNumber']) && isset($msgs[$iterator2]['Number'])) {
             $returnArray['lastArticleNumber'] = $msgs[$iterator2]['Number'];
             $returnArray['lastArticleDate'] = $msgs[$iterator2]['Date'];
         }
         // Break if we found non empty articles.
         if (isset($returnArray['firstArticleNumber']) && isset($returnArray['lastArticleNumber'])) {
             break;
         }
         // Break out if we couldn't find anything.
         if ($iterator1++ >= $msgCount - 1 || $iterator2-- <= 0) {
             break;
         }
     }
     if (is_array($msgs)) {
         //loop headers, figure out parts
         foreach ($msgs as $msg) {
             if (!isset($msg['Number'])) {
                 continue;
             }
             $msgsreceived[] = $msg['Number'];
             $msgPart = $msgTotalParts = 0;
             $pattern = '|\\((\\d+)[\\/](\\d+)\\)|i';
             preg_match_all($pattern, $msg['Subject'], $matches, PREG_PATTERN_ORDER);
             $matchcnt = sizeof($matches[0]);
             for ($i = 0; $i < $matchcnt; $i++) {
                 $msgPart = $matches[1][$i];
                 $msgTotalParts = $matches[2][$i];
             }
             if (!isset($msg['Subject']) || $matchcnt == 0) {
                 $msgsignored[] = $msg['Number'];
                 continue;
             }
             if ((int) $msgPart > 0 && (int) $msgTotalParts > 0) {
                 $subject = utf8_encode(trim(preg_replace('|\\(' . $msgPart . '[\\/]' . $msgTotalParts . '\\)|i', '', $msg['Subject'])));
                 if (!isset($this->message[$subject])) {
                     $this->message[$subject] = $msg;
                     $this->message[$subject]['MaxParts'] = (int) $msgTotalParts;
                     $this->message[$subject]['Date'] = strtotime($this->message[$subject]['Date']);
                 }
                 if ((int) $msgPart > 0) {
                     $this->message[$subject]['Parts'][(int) $msgPart] = array('Message-ID' => substr($msg['Message-ID'], 1, -1), 'number' => $msg['Number'], 'part' => (int) $msgPart, 'size' => $msg['Bytes']);
                     $this->message[$subject]['PartNumbers'][(int) $msgPart] = $msg['Number'];
                 }
             }
         }
         unset($msg);
         unset($msgs);
         $count = 0;
         $updatecount = 0;
         $partcount = 0;
         $rangenotreceived = array_diff($rangerequested, $msgsreceived);
         if ($type != 'partrepair') {
             echo "Received " . sizeof($msgsreceived) . " articles of " . ($last - $first + 1) . " requested, " . sizeof($msgsignored) . " not binaries {$n}";
         }
         if ($type == 'update' && sizeof($msgsreceived) == 0) {
             echo "Error: Server did not return any articles.{$n}";
             echo "Skipping group{$n}";
             return $returnArray;
         }
         if (sizeof($rangenotreceived) > 0) {
             switch ($type) {
                 case 'backfill':
                     //don't add missing articles
                     break;
                 case 'partrepair':
                 case 'update':
                 default:
                     $this->addMissingParts($rangenotreceived, $tableNames['prname'], $groupArr['id']);
                     break;
             }
             echo "Server did not return " . count($rangenotreceived) . " article(s).{$n}";
         }
         if (isset($this->message) && count($this->message)) {
             $groupRegexes = $releaseRegex->getForGroup($groupArr['name']);
             //insert binaries and parts into database. when binary already exists; only insert new parts
             foreach ($this->message as $subject => $data) {
                 //Filter binaries based on black/white list
                 if ($this->isBlackListed($data, $groupArr['name'])) {
                     $msgsblacklisted[] = count($data['Parts']);
                     if ($type == 'partrepair') {
                         $partIds = array();
                         foreach ($data['Parts'] as $partdata) {
                             $partIds[] = $partdata['number'];
                         }
                         $db->queryExec(sprintf("DELETE FROM %s WHERE numberid IN (%s) AND groupid=%d", $tableNames['prname'], implode(',', $partIds), $groupArr['id']));
                     }
                     continue;
                 }
                 if (isset($data['Parts']) && count($data['Parts']) > 0 && $subject != '') {
                     //Check for existing binary
                     $binaryID = 0;
                     $binaryHash = md5($subject . $data['From'] . $groupArr['id']);
                     $res = $db->queryOneRow(sprintf("SELECT id FROM %s WHERE binaryhash = %s", $tableNames['bname'], $db->escapeString($binaryHash)));
                     if (!$res) {
                         //Apply Regexes
                         $regexMatches = array();
                         foreach ($groupRegexes as $groupRegex) {
                             $regexCheck = $releaseRegex->performMatch($groupRegex, $subject);
                             if ($regexCheck !== false) {
                                 $regexMatches = $regexCheck;
                                 break;
                             }
                         }
                         $sql = '';
                         if (!empty($regexMatches)) {
                             $relparts = explode("/", $regexMatches['parts']);
                             $sql = sprintf('INSERT INTO %s (name, fromname, date, xref, totalparts, groupid, procstat, categoryid, regexid, reqid, relpart, reltotalpart, binaryhash, relname, dateadded) VALUES (%s, %s, FROM_UNIXTIME(%s), %s, %s, %d, %d, %s, %d, %s, %d, %d, %s, %s, now())', $tableNames['bname'], $db->escapeString($subject), $db->escapeString(utf8_encode($data['From'])), $db->escapeString($data['Date']), $db->escapeString($data['Xref']), $db->escapeString($data['MaxParts']), $groupArr['id'], Releases::PROCSTAT_TITLEMATCHED, $regexMatches['regcatid'], $regexMatches['regexid'], $db->escapeString($regexMatches['reqid']), $relparts[0], $relparts[1], $db->escapeString($binaryHash), $db->escapeString(str_replace('_', ' ', $regexMatches['name'])));
                         } elseif ($this->onlyProcessRegexBinaries === false) {
                             $sql = sprintf('INSERT INTO %s (name, fromname, date, xref, totalparts, groupid, binaryhash, dateadded) VALUES (%s, %s, FROM_UNIXTIME(%s), %s, %s, %d, %s, now())', $tableNames['bname'], $db->escapeString($subject), $db->escapeString(utf8_encode($data['From'])), $db->escapeString($data['Date']), $db->escapeString($data['Xref']), $db->escapeString($data['MaxParts']), $groupArr['id'], $db->escapeString($binaryHash));
                         } elseif ($type == 'partrepair') {
                             $partIds = array();
                             foreach ($data['Parts'] as $partdata) {
                                 $partIds[] = $partdata['number'];
                             }
                             $db->queryExec(sprintf('DELETE FROM %s WHERE numberid IN (%s) AND groupid = %d', $tableNames['prname'], implode(',', $partIds), $groupArr['id']));
                             continue;
                         }
                         if ($sql != '') {
                             $binaryID = $db->queryInsert($sql);
                             $count++;
                             //if ($count % 500 == 0) echo "$count bin adds...";
                         }
                     } else {
                         $binaryID = $res["id"];
                         $updatecount++;
                         //if ($updatecount % 500 == 0) echo "$updatecount bin updates...";
                     }
                     if ($binaryID != 0) {
                         $partParams = array();
                         $partNumbers = array();
                         foreach ($data['Parts'] as $partdata) {
                             $partcount++;
                             $partParams[] = sprintf('(%d, %s, %s, %s, %s)', $binaryID, $db->escapeString($partdata['Message-ID']), $db->escapeString($partdata['number']), $db->escapeString(round($partdata['part'])), $db->escapeString($partdata['size']));
                             $partNumbers[] = $partdata['number'];
                         }
                         $partSql = 'INSERT INTO ' . $tableNames['pname'] . ' (binaryid, messageid, number, partnumber, size) VALUES ' . implode(', ', $partParams);
                         $pidata = $db->queryInsert($partSql);
                         if (!$pidata) {
                             $msgsnotinserted = array_merge($msgsnotinserted, $partNumbers);
                         } else {
                             $msgsinserted = array_merge($msgsinserted, $partNumbers);
                         }
                     }
                 }
             }
             if (!empty($this->_binaryBlacklistIdsToUpdate)) {
                 $this->_pdo->queryExec(sprintf('UPDATE binaryblacklist SET last_activity = NOW() WHERE id IN (%s)', implode(',', $this->_binaryBlacklistIdsToUpdate)));
                 $this->_binaryBlacklistIdsToUpdate = [];
             }
             //TODO: determine whether to add to missing articles if insert failed
             if (sizeof($msgsnotinserted) > 0) {
                 echo 'WARNING: ' . count($msgsnotinserted) . ' Parts failed to insert' . $n;
                 $this->addMissingParts($msgsnotinserted, $tableNames['prname'], $groupArr['id']);
             }
             if ($count >= 500 || $updatecount >= 500) {
                 echo $n;
             }
             //line break for bin adds output
         }
         $timeUpdate = number_format(microtime(true) - $this->startUpdate, 2);
         $timeLoop = number_format(microtime(true) - $this->startLoop, 2);
         if (sizeof($msgsblacklisted) > 0) {
             echo "Blacklisted " . array_sum($msgsblacklisted) . " parts in " . sizeof($msgsblacklisted) . " binaries" . $n;
         }
         if ($type != 'partrepair') {
             if ($this->_echoCLI) {
                 $this->_colorCLI->doEcho($this->_colorCLI->alternateOver(number_format($count)) . $this->_colorCLI->primaryOver(' new, ') . $this->_colorCLI->alternateOver(number_format($updatecount)) . $this->_colorCLI->primaryOver(' updated, ') . $this->_colorCLI->alternateOver(number_format($partcount)) . $this->_colorCLI->primaryOver(' parts, ') . $this->_colorCLI->alternateOver($timeHeaders . 's') . $this->_colorCLI->primaryOver(' to download articles, ') . $this->_colorCLI->alternateOver($timeUpdate . 's') . $this->_colorCLI->primaryOver(' to insert binaries/parts, ') . $this->_colorCLI->alternateOver($timeLoop . 's') . $this->_colorCLI->primary(' total.'));
             }
         }
         unset($this->message);
         unset($data);
         return $returnArray;
     } else {
         echo "Error: Can't get parts from server (msgs not array) {$n}";
         echo "Skipping group{$n}";
         return $returnArray;
     }
 }
Example #3
0
 private function _nfo_grab($nfometa, &$blobhash)
 {
     // nfometa should be an array() of segments from nzb file
     // it will then populate the blobhash which uses the segments
     // as hash entries for the blob data.
     // nfometa is an array of arrays simiar to the following
     // structure:
     //
     // The list is structured in such a way that the most ideal
     // matches are at the front, while less likely ones at the
     // back of the array
     //
     //	$nfometa = array(
     //       [<releaseid>] = array(
     //           [groups] = array(
     //                       "alt.binaries.mygroupa",
     //                       "alt.binaries.mygroupb",
     //                       "alt.binaries.mygroupc",
     //                       ...
     //                    )
     //           [segment] = array(<segment id>),
     //           [groups] = array(
     //                       "alt.binaries.mygroupa",
     //                       "alt.binaries.mygroupb",
     //                       ...
     //                    )
     //       ),
     //       [<releaseid>] = array(
     //           [groups] = array(
     //                       "alt.binaries.mygroupa",
     //                       ...
     //                    )
     //           [segment] = array(<segment id>),
     //       ),
     //       ...
     //  )
     $nntp = new NNTP();
     // Connect to server (we throw an exception if we fail) which
     // is caught upstairs with the nfo_grab() function
     // no error handling is needed here
     $nntp->doConnect(1, true);
     foreach ($nfometa as $uid => $matches) {
         $blobhash[$uid] = array();
         foreach ($matches as $idx => $match) {
             $fetched = false;
             foreach ($match["groups"] as $group) {
                 // Don not try other groups if we already got it
                 if ($fetched) {
                     break;
                 }
                 // Select the group and then attempt to fetch the article
                 $blob = $nntp->getMessages($group, $match["segment"], false);
                 if ($blob === false) {
                     if ($this->verbose) {
                         echo '*';
                     }
                     continue;
                 }
                 // Mark that we fetched it to prevent fetching more
                 // of the same thing
                 $fetched = true;
                 if ($this->verbose) {
                     echo '.';
                 }
                 // Update blob with decrypted version and store
                 if ($this->is_binary($blob)) {
                     // Binary data is not acceptable, we only
                     // work with text from here on out.
                     continue;
                 }
                 // Read-able ascii at this point... store it
                 $blobhash[$uid][$idx] = $blob;
             }
             if (!$fetched) {
                 // handle empty/failed segments
                 $blobhash[$uid][$idx] = Null;
             }
         }
     }
     $nntp->doQuit();
 }
     }
 } else {
     if (isset($pieces[1]) && $pieces[0] == 'par2') {
         $nntp = new NNTP();
         if ($nntp->doConnect() === false) {
             exit($c->error("Unable to connect to usenet."));
         }
         $relID = $pieces[1];
         $guid = $pieces[2];
         $groupID = $pieces[3];
         $nzbcontents = new NZBContents(array('echo' => true, 'nntp' => $nntp, 'nfo' => new Info(), 'db' => $db, 'pp' => new PProcess(['Settings' => $pdo, 'Nfo' => $Nfo, 'NameFixer' => $namefixer])));
         $res = $nzbcontents->checkPAR2($guid, $relID, $groupID, 1, 1);
         if ($res === false) {
             echo '.';
         }
         $nntp->doQuit();
     } else {
         if (isset($pieces[1]) && $pieces[0] == 'predbft') {
             $pre = $pieces[1];
             if ($res = $db->queryOneRow(sprintf('SELECT id AS preid, title, source, searched FROM prehash ' . 'WHERE id = %d', $pre))) {
                 $namefixer->done = $namefixer->matched = false;
                 $ftmatched = $searched = 0;
                 $ftmatched = $namefixer->matchPredbFT($res, 1, 1, true, 1);
                 if ($ftmatched > 0) {
                     $searched = 1;
                 } elseif ($ftmatched < 0) {
                     $searched = -6;
                     echo "*";
                 } else {
                     $searched = $res['searched'] - 1;
                     echo ".";
Example #5
0
 /**
  * Performing parsing.
  */
 public function process()
 {
     // Default query for both full db and last 4 hours.
     $sql = "SELECT r.searchname, r.name, r.fromname, r.id as rid, r.categoryid, r.guid, r.postdate,\n\t\t\t   rn.id as nfoid,\n\t\t\t   g.name as groupname,\n\t\t\t   GROUP_CONCAT(rf.name) as filenames\n\t\tFROM releases r\n\t\tLEFT JOIN releasenfo rn ON (rn.releaseid = r.id)\n\t\tLEFT JOIN groups g ON (g.id = r.groupid)\n\t\tLEFT JOIN releasefiles rf ON (rf.releaseid = r.id)\n\t\tWHERE r.categoryid in (' . Category::CAT_TV_OTHER . ',' . Category::CAT_MOVIE_OTHER . ',' . Category::CAT_MISC_OTHER . ',' . Category::CAT_XXX_OTHER . ')\n\t\t%s\n\t\tGROUP BY r.id";
     $res = $this->pdo->query(sprintf($sql, $this->limited ? "AND r.adddate BETWEEN NOW() - INTERVAL 4 HOUR AND NOW()" : ""));
     $this->releasestocheck = sizeof($res);
     if ($res) {
         echo "PostPrc : Parsing last " . $this->releasestocheck . " releases in the Other-Misc categories\n";
         foreach ($res as $rel) {
             $tempname = $foundName = $methodused = '';
             //Knoc.One
             if (preg_match("/KNOC.ONE/i", $rel['name'])) {
                 $title = '';
                 $items = preg_split("/(\\.| )/", $rel['name']);
                 foreach ($items as $value) {
                     if (preg_match("/^[a-z]+\$/i", $value)) {
                         $len = strlen($value);
                         if ($len > 2) {
                             $title .= substr($value, -2) . substr($value, 0, -2) . " ";
                         } elseif ($len = 2) {
                             $title .= substr($value, -1) . substr($value, 0, -1) . " ";
                         } else {
                             $title .= $value . " ";
                         }
                     } else {
                         $title .= $value . " ";
                     }
                 }
                 $foundName = $title;
                 $methodused = "Knoc.One";
                 $this->determineCategory($rel, $foundName, $methodused);
             }
             ///
             ///Use the Nfo to try to get the proper Releasename.
             ///
             $nfo = $this->pdo->queryOneRow(sprintf("select uncompress(nfo) as nfo from releasenfo where releaseid = %d", $rel['rid']));
             if ($nfo && $foundName == "") {
                 $this->nfosprocessed++;
                 $nfo = $nfo['nfo'];
                 //LOUNGE releases
                 if (preg_match('/([a-z0-9.]+\\.MBLURAY)/i', $nfo, $matches)) {
                     $foundName = $matches[1];
                     $methodused = "LOUNGE";
                     $this->determineCategory($rel, $foundName, $methodused);
                 }
                 //AsianDVDClub releases
                 if (preg_match('/adc-[a-z0-9]{1,10}/', $rel['name'])) {
                     if (preg_match('/.*\\(\\d{4}\\).*/i', $nfo, $matches)) {
                         $foundName = $matches[0];
                         $methodused = "AsianDVDClub";
                         $this->determineCategory($rel, $foundName, $methodused);
                     }
                 }
                 //ACOUSTiC  releases
                 if (preg_match('/ACOUSTiC presents \\.\\.\\..*?([a-z0-9].*?\\(.*?\\))/is', $nfo, $matches)) {
                     $foundName = $matches[1] . ".MBLURAY";
                     $methodused = "ACOUSTiC ";
                     $this->determineCategory($rel, $foundName, $methodused);
                 }
                 //Japhson  releases
                 if (preg_match('/Japhson/i', $nfo, $matches)) {
                     $movie = new Movie();
                     $imdbID = null;
                     if (preg_match('/tt(\\d{7})/i', $nfo, $matches)) {
                         $imdbId = $matches[1];
                         $movCheck = $movie->fetchImdbProperties($imdbId);
                         $foundName = $movCheck['title'];
                         if (!preg_match('/(19|20)\\d{2}/i', $foundName)) {
                             $foundName = $foundName . "." . $movCheck['year'];
                         }
                         if (preg_match('/language.*?\\b([a-z0-9]+)\\b/i', $nfo, $matches)) {
                             if (!preg_match('/English/i', $matches[1])) {
                                 $foundName = $foundName . "." . $matches[1];
                             }
                         }
                         if (preg_match('/audio.*?\\b(\\w+)\\b/i', $nfo, $matches)) {
                             if (preg_match('/(Chinese|German|Dutch|Spanish|Hebrew|Finnish|Norwegian)/i', $matches[1])) {
                                 $foundName = $foundName . "." . $matches[1];
                             }
                         }
                         if (preg_match('/(video|resolution|video res).*?(1080|720|816|820|272|1280 @|528|1920)/i', $nfo, $matches)) {
                             if ($matches[2] == '1280 @') {
                                 $matches[2] = '720';
                             }
                             if ($matches[2] == '1920') {
                                 $matches[2] = '1080';
                             }
                             $foundName = $foundName . "." . $matches[2];
                         }
                         if (preg_match('/source.*?\\b(DVD9|DVD5|BDRIP|DVD\\-?RIP|BLURAY)\\b/i', $nfo, $matches)) {
                             $foundName = $foundName . "." . $matches[1];
                         }
                         if (preg_match('/(video|resolution|video res).*?(XVID|X264|WMV)/i', $nfo, $matches)) {
                             $foundName = $foundName . "." . $matches[2];
                         }
                         if (preg_match('/audio.*?\\b(DTS|AC3)\\b/i', $nfo, $matches)) {
                             $foundName = $foundName . "." . $matches[1];
                         }
                         $foundName = $foundName . "-Japhson";
                         $methodused = "Japhson";
                         $this->determineCategory($rel, $foundName, $methodused);
                     }
                 }
                 //AIHD  releases
                 if (preg_match('/ALWAYS iN HiGH/i', $nfo, $matches)) {
                     $movie = new Movie();
                     $imdbID = null;
                     if (preg_match('/tt(\\d{7})/i', $nfo, $matches)) {
                         $imdbId = $matches[1];
                         $movCheck = $movie->fetchImdbProperties($imdbId);
                         $foundName = $movCheck['title'];
                         if (!preg_match('/(19|20)\\d{2}/i', $foundName)) {
                             $foundName = $foundName . "." . $movCheck['year'];
                         }
                         if (preg_match('/L\\.([a-z0-9]+)\\b/i', $nfo, $matches)) {
                             if (!preg_match('/En/i', $matches[1])) {
                                 $foundName = $foundName . "." . $matches[1];
                             }
                         }
                         if (preg_match('/(V).*?(1080|720|816|820|272|1280 @|528|1920)/i', $nfo, $matches)) {
                             if ($matches[2] == '1280 @') {
                                 $matches[2] = '720';
                             }
                             if ($matches[2] == '1920') {
                                 $matches[2] = '1080';
                             }
                             $foundName = $foundName . "." . $matches[2];
                         }
                         if (preg_match('/V.*?\\b(DVD9|DVD5|BDRIP|DVD\\-?RIP|BLURAY)\\b/i', $nfo, $matches)) {
                             $foundName = $foundName . "." . $matches[1];
                         }
                         if (preg_match('/(V).*?(XVID|X264|WMV)/i', $nfo, $matches)) {
                             $foundName = $foundName . "." . $matches[2];
                         }
                         if (preg_match('/A.*?\\b(DTS|AC3)\\b/i', $nfo, $matches)) {
                             $foundName = $foundName . "." . $matches[1];
                         }
                         $foundName = $foundName . "-AIHD";
                         $methodused = "AIHD";
                         $this->determineCategory($rel, $foundName, $methodused);
                     }
                 }
                 //IMAGiNE releases
                 if (preg_match('/\\*\\s+([a-z0-9]+(?:\\.|_| )[a-z0-9\\.\\_\\- ]+ \\- imagine)\\s+\\*/i', $nfo, $matches)) {
                     $foundName = $matches[1];
                     $methodused = "imagine";
                     $this->determineCategory($rel, $foundName, $methodused);
                 }
                 //LEGION releases
                 if (preg_match('/([a-z0-9 \\.\\-]+LEGi0N)/is', $nfo, $matches) && $foundName == "") {
                     $foundName = $matches[1];
                     $methodused = "Legion";
                     $this->determineCategory($rel, $foundName, $methodused);
                 }
                 //SWAGGER releases
                 if (preg_match('/(S  W  A  G  G  E  R|swg.*?nfo)/i', $nfo) && $foundName == "") {
                     if (preg_match('/presents.*?([a-z0-9].*?\\((19|20)\\d{2}\\))/is', $nfo, $matches)) {
                         $foundName = $matches[1];
                     }
                     if (preg_match('/language.*?\\b([a-z0-9]+)\\b/i', $nfo, $matches)) {
                         if ($matches[1] != "english") {
                             $foundName = $foundName . "." . $matches[1];
                         }
                     }
                     if (preg_match('/resolution.*?(1080|720)/i', $nfo, $matches)) {
                         $foundName = $foundName . ".BluRay." . $matches[1];
                     }
                     if (preg_match('/video.*?\\b([a-z0-9]+)\\b/i', $nfo, $matches)) {
                         $foundName = $foundName . "." . $matches[1];
                     }
                     if (preg_match('/audio.*?\\b([a-z0-9]+)\\b/i', $nfo, $matches)) {
                         $foundName = $foundName . "." . $matches[1];
                     }
                     $foundName = $foundName . "-SWAGGER";
                     $methodused = "SWAGGER";
                     $this->determineCategory($rel, $foundName, $methodused);
                 }
                 //cm8 releases
                 if (preg_match('/([a-z0-9]+(?:\\.|_| )[a-z0-9\\.\\_\\- \'\\)\\(]+\\-(futv|crimson|qcf|runner|clue|ftp|episode|momentum|PFA|topaz|vision|tdp|haggis|nogrp|shirk|imagine|santi|sys|deimos|ltu|ficodvdr|cm8|dvdr|Nodlabs|aaf|sprinter|exvid|flawl3ss|rx|magicbox|done|unveil))\\b/i', $nfo, $matches) && $foundName == "") {
                     //echo "this: ".$matches[1]."\n";
                     $foundName = $matches[1];
                     $methodused = "cm8";
                     $this->determineCategory($rel, $foundName, $methodused);
                 }
                 //river
                 if (preg_match('/([a-z0-9\\.\\_\\-]+\\-(webios|river|w4f|sometv|ngchd|C4|gf|bov|26k|ftw))\\b/i', $nfo, $matches) && $foundName == "") {
                     $foundName = $matches[1];
                     $methodused = "river-1";
                     $this->determineCategory($rel, $foundName, $methodused);
                 }
                 if (preg_match('/([a-z0-9]+(?:\\.|_| )[a-z0-9\\.\\_\\- \'\\)\\(]+\\-(CiA|Anarchy|RemixHD|FTW|Revott|WAF|CtrlHD|Telly|Nif|Line|NPW|Rude|EbP|CRisC|SHK|AssAss1ns|Leverage|BBW|NPW))\\b/i', $nfo, $matches) && $foundName == "") {
                     $foundName = $matches[1];
                     $methodused = "river-2";
                     $this->determineCategory($rel, $foundName, $methodused);
                 }
                 if (preg_match('/([a-z0-9]+(?:\\.|_| )[a-z0-9\\.\\_\\- \'\\)\\(]+\\-(XPD|RHyTM))\\b/i', $nfo, $matches) && $foundName == "") {
                     $foundName = $matches[1];
                     $methodused = "river-3";
                     $this->determineCategory($rel, $foundName, $methodused);
                 }
                 if (preg_match('/(-PROD$|-BOV$|-NMR$|$-HAGGiS|-JUST$|CRNTV$|-MCA$|int$|-DEiTY$|-VoMiT$|-iNCiTE$|-BRUTUS$|-DCN$|-saints$|-sfm$|-lol$|-fov$|-logies$|-c4tv$|-fqm$|-jetset$|-ils$|-miragetv$|-gfvid$|-btl$|-terra$)/i', $rel['searchname']) && $foundName == "") {
                     $foundName = $rel['searchname'];
                     $methodused = "river-4";
                     $this->determineCategory($rel, $foundName, $methodused);
                 }
                 //SANTi releases
                 if (preg_match('/\\b([a-z0-9]+(?:\\.|_| )[a-z0-9\\.\\_\\- \']+\\-santi)\\b/i', $nfo, $matches) && $foundName == "") {
                     $foundName = $matches[1];
                     $methodused = "SANTi";
                     $this->determineCategory($rel, $foundName, $methodused);
                 }
                 //INSPiRAL releases
                 if (preg_match('/^([a-z0-9]+(?:\\.|_| )[a-z0-9\\.\\_\\- ]+ \\- INSPiRAL)\\s+/im', $nfo, $matches) && $foundName == "") {
                     $foundName = $matches[1];
                     $methodused = "INSPiRAL";
                     $this->determineCategory($rel, $foundName, $methodused);
                 }
                 //CIA releases
                 if (preg_match('/Release NAME.*?\\:.*?([a-z0-9][a-z0-9\\.\\ ]+)\\b.*?([a-z0-9][a-z0-9\\.\\ ]+\\-CIA)\\b/is', $nfo, $matches) && $foundName == "") {
                     $foundName = $matches[1] . $matches[2];
                     $methodused = "CIA";
                     $this->determineCategory($rel, $foundName, $methodused);
                 }
                 //HDChina releases
                 if (preg_match('/HDChina/', $nfo) && $foundName == "") {
                     if (preg_match('/Disc Title\\:.*?\\b([a-z0-9\\ \\.\\-\\_()]+\\-HDChina)/i', $nfo, $matches)) {
                         $foundName = $matches[1];
                         $methodused = "HDChina";
                         $this->determineCategory($rel, $foundName, $methodused);
                     }
                 }
                 //Pringles
                 if (preg_match('/PRiNGLES/', $nfo) && $foundName == "") {
                     if (preg_match('/is giving you.*?\\b([a-z0-9 ]+)\\s/i', $nfo, $matches)) {
                         $foundName = $matches[1];
                         $foundName = rtrim($foundName);
                         $foundName = ltrim($foundName);
                     }
                     if (preg_match('/this release.*?((19|20)\\d{2})/i', $nfo, $matches)) {
                         $foundName = $foundName . "." . $matches[1];
                         $foundName = rtrim($foundName);
                     }
                     if (preg_match('/\\[x\\] (Danish|Norwegian|Swedish|Finish|Other)/i', $nfo, $matches)) {
                         $foundName = $foundName . "." . $matches[1];
                     }
                     if (preg_match('/\\[x\\] (DVD9|DVD5)/i', $nfo, $matches)) {
                         $foundName = $foundName . "." . $matches[1];
                     }
                     $foundName = $foundName . "-PRiNGLES";
                     $methodused = "Pringles";
                     $this->determineCategory($rel, $foundName, $methodused);
                 }
                 //Fairlight releases
                 if (preg_match('/\\/Team FairLight/', $nfo) && $foundName == "") {
                     $title = null;
                     $os = null;
                     $method = null;
                     if (preg_match('/\\b([a-z0-9\\ \\- \\_()\\.]+) \\(c\\)/i', $nfo, $matches)) {
                         $title = $matches['1'];
                         $foundName = $title;
                     }
                     $foundName = $foundName . "-FLT";
                     $methodused = "FairLight";
                     $this->determineCategory($rel, $foundName, $methodused);
                 }
                 //CORE releases
                 if (preg_match('/Supplied.*?\\:.*?(CORE)/', $nfo) || preg_match('/Packaged.*?\\:.*?(CORE)/', $nfo) && $foundName == "") {
                     $title = null;
                     $os = null;
                     $method = null;
                     if (preg_match('/\\b([a-z0-9\\.\\-\\_\\+\\ ]+) \\*[a-z0-9]+\\*/i', $nfo, $matches)) {
                         $title = $matches['1'];
                         $foundName = $title;
                     }
                     if (preg_match('/Crack\\/.*?\\:.*?([a-z]+)/i', $nfo, $matches)) {
                         $method = $matches['1'];
                         $foundName = $foundName . " " . $method;
                     }
                     if (preg_match('/OS.*?\\:.*?([a-z]+)/i', $nfo, $matches)) {
                         $os = $matches['1'];
                         $foundName = $foundName . " " . $os;
                     }
                     $foundName = $foundName . "-CORE";
                     $methodused = "CORE";
                     $this->determineCategory($rel, $foundName, $methodused);
                 }
                 //CompleteRelease
                 if (preg_match('/Complete name.*?([a-z0-9].*?\\-[a-z0-9]+)\\b/i', $nfo, $matches) && $foundName == "") {
                     $foundName = $matches[1];
                     $methodused = "CompleteRelease";
                     $this->determineCategory($rel, $foundName, $methodused);
                 }
                 //Livesets
                 if (preg_match('/\\nLivesets.*?\\n.*?\\n.*?\\n.*?\\n.*?\\n(?P<name>\\w.*?)\\n(?P<album>\\w.*?)\\n/im', $nfo, $matches) && $foundName == "") {
                     $artist = $matches['name'];
                     $title = $matches['album'];
                     $source = null;
                     $year = null;
                     if (preg_match('/Year.*?\\:{1,2} ?(?P<year>(19|20)\\d{2})/i', $nfo, $matches)) {
                         $year = $matches[1];
                     } elseif (preg_match('/date.*?\\:.*?(?P<year>(19|20)\\d{2})/i', $nfo, $matches)) {
                         $year = $matches[1];
                     }
                     if (preg_match('/(web|cable|sat)/i', $title)) {
                         $source = "";
                     } elseif (preg_match('/Source.*?\\:{1,2} ?(?P<source>.*?)(\\s{2,}|\\s{1,})/i', $nfo, $matches)) {
                         $source = $matches[1];
                         if ($source == "Satellite") {
                             $source = "Sat";
                         }
                     }
                     if ($artist) {
                         $tempname = $artist;
                         if ($title) {
                             $tempname = $tempname . "-" . $title;
                         }
                         if ($source) {
                             $tempname = $tempname . "-" . $source;
                         }
                         if ($year) {
                             $tempname = $tempname . "-" . $year;
                         }
                         $tempname = preg_replace("/[^a-zA-Z,0-9,\\-,\\s]/", "", $tempname);
                         $foundName = $tempname;
                         $methodused = "Live Sets";
                         $this->determineCategory($rel, $foundName, $methodused);
                     }
                 }
                 //Typical scene regex
                 if (preg_match('/(?P<source>Source[\\s\\.]*?:|fix for nuke)?(?:\\s|\\]|\\[)?(?P<name>[a-z0-9\'\\-]+(?:\\.|_)[a-z0-9\\.\\-_\'&]+\\-[a-z0-9&]+)(?:\\s|\\[|\\])/i', $nfo, $matches) && $foundName == "") {
                     if (empty($matches['source'])) {
                         if (!preg_match('/usenet\\-space/i', $matches['name'])) {
                             $foundName = $matches['name'];
                             $methodused = "Scene";
                             $this->determineCategory($rel, $foundName, $methodused);
                         }
                     }
                 }
             }
             //The Big One
             if (preg_match_all('/([a-z0-9\\ ]+)\\.{1,}(\\:|\\[)(?P<name>.*)(\\s{2}|\\s{1})/i', $nfo, $matches) && $foundName == "") {
                 $lut = array();
                 foreach ($matches[1] as $key => $k) {
                     $lut[str_replace(' ', '', strtolower(trim($k)))] = trim($matches[3][$key]);
                 }
                 $year = null;
                 $vidsource = null;
                 $series = null;
                 $season = null;
                 $episode = null;
                 $language = null;
                 $artist = null;
                 $source = null;
                 foreach ($lut as $k => $v) {
                     $v = rtrim($v);
                     if (!$year && preg_match('/((19|20)\\d{2})/', $v, $matches)) {
                         $year = $matches[1];
                     }
                     if (!$vidsource && preg_match('/(xvid|x264|h264|wmv|divx)/i', $v, $matches)) {
                         $vidsource = $matches[1];
                     }
                     if (!$season && preg_match('/(season|seizon).*?(\\d{1,3})/i', $v, $matches)) {
                         $season = $matches[2];
                     }
                     if (!$episode && preg_match('/(Episode|ep).*?(\\d{1,3})/i', $v, $matches)) {
                         $episode = $matches[2];
                     }
                 }
                 if (isset($lut['artist'])) {
                     $del = "-";
                     if (isset($lut['artist'])) {
                         $lut['artist'] = trim($lut['artist'], " ");
                         $tempname = $lut['artist'];
                     }
                     if (isset($lut['title'])) {
                         $tempname = $tempname . $del . $lut['title'];
                     }
                     if (isset($lut['album']) && !isset($lut['title'])) {
                         $tempname = $tempname . $del . $lut['album'];
                     }
                     if (isset($lut['track']) && !isset($lut['title']) && !isset($lut['album'])) {
                         $tempname = $tempname . $del . $lut['track'];
                     }
                     if (!isset($lut['source'])) {
                         $lut['source'] = 'WEB';
                     }
                     if (isset($lut['source']) && !preg_match('/SAT/i', $tempname)) {
                         $tempname = $tempname . $del . $lut['source'];
                     }
                     if (!preg_match('/(19|20)\\d{2}/', $tempname) && $year) {
                         $tempname = $tempname . $del . $year;
                     }
                     if (isset($lut['ripper'])) {
                         $tempname = $tempname . $del . $lut['ripper'];
                     }
                     $tempname = preg_replace("/[^a-zA-Z,0-9,\\-,\\&,\\s]/", "", $tempname);
                     $tempname = preg_replace("/[ ]{2,}/", "", $tempname);
                     $methodused = "The Big One Music";
                     $foundName = $tempname;
                     $this->determineCategory($rel, $foundName, $methodused);
                 } else {
                     if (isset($lut['title'])) {
                         $del = " ";
                         if (isset($lut['series'])) {
                             $tempname = $lut['series'];
                         }
                         $tempname = $tempname . $del . $lut['title'];
                         if ($season && $episode) {
                             $tempname = $tempname . $del . "S" . str_pad($season, 2, '0', STR_PAD_LEFT) . 'E' . str_pad($episode, 2, '0', STR_PAD_LEFT);
                         } else {
                             if ($season) {
                                 $tempname = $tempname . $del . "S" . $season;
                             }
                             if ($episode) {
                                 $tempname = $tempname . $del . "Ep" . $episode;
                             }
                         }
                         if (isset($lut['source']) && !preg_match('/SAT/i', $lut['title'])) {
                             $tempname = $tempname . $del . $lut['source'];
                         }
                         if (!preg_match('/(19|20)\\d{2}/', $tempname) && $year) {
                             $tempname = $tempname . $del . $year;
                         }
                         if (isset($lut['language'])) {
                             $tempname = $tempname . $del . $lut['language'];
                         }
                         if ($vidsource) {
                             $tempname = $tempname . $del . $vidsource;
                         }
                         $tempname = preg_replace("/ /", " ", $tempname);
                         $tempname = preg_replace("/[^a-zA-Z,0-9,\\-,\\&,\\s]/", " ", $tempname);
                         $tempname = preg_replace("/[ ]+/", " ", $tempname);
                         $methodused = "The Big One Other";
                         $foundName = $tempname;
                         $this->determineCategory($rel, $foundName, $methodused);
                     }
                 }
             }
             ///
             ///unable to extract releasename from nfo, try the rar file
             ///
             if ($rel['filenames'] && $foundName == '') {
                 $this->releasefilesprocessed++;
                 $files = explode(',', $rel['filenames']);
                 if (![$files]) {
                     $files = [$files];
                 }
                 // Scene regex
                 $sceneRegex = '/([a-z0-9\'\\-\\.\\_\\(\\)\\+\\ ]+\\-[a-z0-9\'\\-\\.\\_\\(\\)\\ ]+)(.*?\\\\.*?|)\\.(?:\\w{3,4})$/i';
                 foreach ($files as $file) {
                     // Petje Releases
                     if (preg_match('/Petje \\<petje\\@pietamientje\\.com\\>/', $rel['fromname'], $matches3) && $foundName == '') {
                         if (preg_match('/.*\\.(mkv|avi|mp4|wmv|divx)/', $file, $matches4)) {
                             $array_new = explode('\\', $matches4[0]);
                             foreach ($array_new as $item) {
                                 if (preg_match('/.*\\((19|20\\d{2})\\)$/', $item, $matched)) {
                                     //echo $matched[0].".720p.x264-Petje";
                                     //print_r($matched);
                                     $foundName = $matched[0] . ".720p.x264-Petje";
                                     $methodused = "Petje";
                                     $this->determineCategory($rel, $foundName, $methodused);
                                     break 2;
                                 }
                             }
                         }
                     }
                     //3D Remux
                     if (preg_match('/.*Remux\\.mkv/', $file, $matches4)) {
                         $foundName = str_replace(".mkv", "", $matches4[0]);
                         $methodused = "3D Remux";
                         $this->determineCategory($rel, $foundName, $methodused);
                     }
                     //QoQ Extended
                     if (preg_match('/Q\\-sbuSLN.*/i', $file, $matches4)) {
                         $new1 = preg_match('/( )?(\\.wmv|\\.divx|\\.avi|\\.mkv)/i', $matches4[0], $matched);
                         $new2 = str_replace($matched[0], "", $matches4[0]);
                         $foundName = strrev($new2);
                         $methodused = "QoQ Extended";
                         $this->determineCategory($rel, $foundName, $methodused);
                     }
                     // Directory\Title.Year.Format.Group.mkv
                     if (preg_match('/(?<=\\\\).*?BLURAY.(1080|720)P.*?KNORLOADING(?=\\.MKV)/i', $file, $matches3) && $foundName == '') {
                         $foundName = $matches3['0'];
                         $methodused = "a.b.hdtv.x264";
                         $this->determineCategory($rel, $foundName, $methodused);
                     }
                     // ReleaseGroup.Title.Format.mkv
                     if (preg_match('/(?<=swg_|swghd\\-|lost\\-|veto\\-|kaka\\-|abd\\-|airline\\-|daa\\-|data\\-|japhson\\-|ika\\-|lng\\-|nrdhd\\-|saimorny\\-|sparks\\-|ulshd\\-|nscrns\\-|ifpd\\-|invan\\-|an0\\-|besthd\\-|muxhd\\-|s7\\-).*?((1080|720)|P)(?=\\.MKV)/i', $file, $matches3) && $foundName == '') {
                         $foundName = str_replace("_", ".", $matches3['0']);
                         $methodused = "a.b.hdtv.x264";
                         $this->determineCategory($rel, $foundName, $methodused);
                     }
                     // Title.Format.ReleaseGroup.mkv
                     if (preg_match('/.*?(1080|720)(|P).(SON)/i', $file, $matches3) && $foundName == '') {
                         $foundName = str_replace("_", ".", $matches3['0']);
                         $methodused = "a.b.hdtv.x264";
                         $this->determineCategory($rel, $foundName, $methodused);
                     }
                     //epubmobi
                     if (preg_match('/.*\\.(epub|mobi|azw3|pdf|prc|lit|rtf|azw|cbr|doc)/', $file, $matches4)) {
                         $foundName = str_replace(".doc", "", str_replace(".cbr", "", str_replace(".prc", "", str_replace(".pdf", "", str_replace(".azw3", "", str_replace(".mobi", "", str_replace(".epub", "", str_replace(".rtf", "", str_replace(".azw", "", str_replace(".lit", "", $matches4[0]))))))))));
                         $methodused = "EpubMobi";
                         $this->determineCategory($rel, $foundName, $methodused);
                     }
                     //Reversed name
                     if (preg_match('/[a-z0-9\\(\\)\'\\!\\,\\.\\-\\ \\_]+(BEW|p027|p0801)[a-z0-9\\(\\)\\,\'\\!\\ \\-\\.]+/i', $file, $matches4)) {
                         $new1 = preg_match('/( )?(\\.m2ts|\\.wmv|\\.avi|.mp4|\\.mkv)/i', $matches4[0], $matched);
                         $new2 = str_replace($matched[0], "", $matches4[0]);
                         $foundName = strrev($new2);
                         $methodused = "Reversed";
                         $this->determineCategory($rel, $foundName, $methodused);
                     }
                     //Check rarfile contents for a scene name
                     if (preg_match($sceneRegex, $file, $matches) && $foundName == '') {
                         //Simply Releases Toppers
                         if (preg_match('/(\\\\)(?P<name>.*?ReleaseS Toppers)/', $file, $matches1) && $foundName == '') {
                             $foundName = $matches1['name'];
                             $methodused = "Release Files-1";
                             $this->determineCategory($rel, $foundName, $methodused);
                         }
                         //Scene format no folder.
                         if (preg_match('/^([a-z0-9\\.\\_\\- ]+\\-[a-z0-9\\_]+)(\\\\|)$/i', $matches[1]) && $foundName == '') {
                             if (strlen($matches['1']) >= 15) {
                                 $foundName = $matches['1'];
                                 $methodused = "Scene format no folder.";
                                 $this->determineCategory($rel, $foundName, $methodused);
                             }
                         }
                         //Check to see if file is inside of a folder. Use folder name if it is
                         if (preg_match('/^(.*?\\\\)(.*?\\\\|)(.*?)$/i', $file, $matches1) && $foundName == '') {
                             if (preg_match('/^([a-z0-9\\.\\_\\- ]+\\-[a-z0-9\\_]+)(\\\\|)$/i', $matches1['1'], $res)) {
                                 $foundName = $res['1'];
                                 $methodused = "Release Files-1";
                                 $this->determineCategory($rel, $foundName, $methodused);
                             }
                             if (preg_match('/(?!UTC)([a-z0-9]+[a-z0-9\\.\\_\\- \'\\)\\(]+(\\d{4}|HDTV).*?\\-[a-z0-9]+)/i', $matches1['1'], $res) && $foundName == '') {
                                 $foundName = $res['1'];
                                 $methodused = "Release Files-2";
                                 $this->determineCategory($rel, $foundName, $methodused);
                             }
                             if (preg_match('/^([a-z0-9\\.\\_\\- ]+\\-[a-z0-9\\_]+)(\\\\|)$/i', $matches1['2'], $res) && $foundName == '') {
                                 $foundName = $res['1'];
                                 $methodused = "Release Files-3";
                                 $this->determineCategory($rel, $foundName, $methodused);
                             }
                             if (preg_match('/^([a-z0-9\\.\\_\\- ]+\\-(?:.+)\\(html\\))\\\\/i', $matches1['1'], $res) && $foundName == '') {
                                 $foundName = $res['1'];
                                 $methodused = "Release Files-4";
                                 $this->determineCategory($rel, $foundName, $methodused);
                             }
                         }
                         if (preg_match('/(?!UTC)([a-z0-9]+[a-z0-9\\.\\_\\- \'\\)\\(]+(\\d{4}|HDTV).*?\\-[a-z0-9]+)/i', $file, $matches2) && $foundName == '') {
                             $foundName = $matches2['1'];
                             $methodused = "Release Files-4";
                             $this->determineCategory($rel, $foundName, $methodused);
                         }
                     }
                 }
                 //RAR file contents release name matching
                 /*if (sizeof($files) > 0 && $foundName == '')
                 					{
                 						echo "RAR checking\n";
                 						//Loop through releaseFiles to find a match
                 						foreach($files as $rarFile)
                 						{
                 							//echo "-{$rarFile}\n";
                 							if ($foundName == '')
                 							{
                 								//Lookup name via reqid (filename)
                 								if (preg_match('/\.(avi|mkv|mp4|mov|wmv|iso|img|gcm|ps3|wad|ac3|nds|bin|cue|mdf)/i', $rarFile))
                 								{
                 									$this->pdo->getSetting('reqidurl;
                 									$lookupUrl = 'http://allfilled/query.php?t=alt.binaries.srrdb&reqid='.urlencode(basename($rarFile));
                 									echo '-lookup: '.$lookupUrl."\n";
                 									$xml = Utility::getUrl(['url' => $lookupUrl]);
                 									//$xml = false;
                 
                 									if ($xml !== false)
                 									{
                 										$xmlObj = @simplexml_load_string($xml);
                 										$arrXml = objectsIntoArray($xmlObj);
                 
                 										if (isset($arrXml["item"]) && is_array($arrXml["item"]) && isset($arrXml["item"]["@attributes"]) && is_array($arrXml["item"]["@attributes"]))
                 										{
                 											$foundName = $arrXml["item"]["@attributes"]["title"];
                 										}
                 									}
                 								}
                 							}
                 						}
                 					}*/
             }
             // do par check if user has elected for downloading extra stuff
             if ($this->pdo->getSetting('unrarpath') != '' && $foundName == "") {
                 $nzb = new NZB();
                 $nzbfile = $nzb->getNZBPath($rel['guid']);
                 $nzbInfo = new nzbInfo();
                 $nzbInfo->loadFromFile($nzbfile);
                 if (!empty($nzbInfo->parfiles) && empty($nzbInfo->rarfiles) && empty($nzbInfo->audiofiles)) {
                     $nntp = new NNTP();
                     $nntp->doConnect();
                     if ($this->verbose) {
                         echo "Checking Par\n";
                     }
                     foreach ($nzbInfo->parfiles as $parfile) {
                         $this->parsprocessed++;
                         $parBinary = $nntp->getMessages($parfile['groups'][0], $parfile['segments'], $this->verbose);
                         if ($parBinary) {
                             $par2 = new Par2info();
                             $par2->setData($parBinary);
                             if (!$par2->error) {
                                 $parFiles = $par2->getFileList();
                                 foreach ($parFiles as $file) {
                                     if (isset($file['name']) && (preg_match('/.*part0*1\\.rar$/iS', $file['name'], $matches) || preg_match('/(?!part0*1)\\.rar$/iS', $file['name'], $matches) || preg_match('/\\.001$/iS', $file['name'], $matches))) {
                                         $foundName = preg_replace('/^(.*)(\\.part0*1\\.rar|\\.rar|\\.001)$/i', '\\1', $file['name']);
                                         $methodused = "Par file";
                                         $this->determineCategory($rel, $foundName, $methodused);
                                         break;
                                     }
                                 }
                             }
                         }
                         unset($parBinary);
                         if ($foundName != "") {
                             break;
                         }
                     }
                     $nntp->doQuit();
                 }
             }
             ///
             ///	This is a last ditch effort, build a ReleaseName from the Nfo
             ///
             if ($nfo && ($foundName == "" || $methodused == 'Scene format no folder.')) {
                 //LastNfoAttempt
                 if (preg_match('/tt(\\d{7})/i', $nfo, $matches) && $foundName == "") {
                     $movie = new Movie();
                     $imdbId = $matches[1];
                     $movCheck = $movie->fetchImdbProperties($imdbId);
                     $buffer = Utility::getUrl(['url' => 'http://akas.imdb.com/title/tt' . $imdbId . '/']);
                     if (!preg_match('/content\\=\\"video\\.tv\\_show\\"/i', $buffer)) {
                         if (isset($movCheck['title'])) {
                             $foundName = $movCheck['title'];
                             if (!preg_match('/(19|20)\\d{2}/i', $foundName)) {
                                 $foundName = $foundName . "." . (isset($movCheck['year']) ? $movCheck['year'] : "");
                             }
                             if (preg_match('/language.*?\\b([a-z0-9]+)\\b/i', $nfo, $matches)) {
                                 if (!preg_match('/English/i', $matches[1])) {
                                     $foundName = $foundName . "." . $matches[1];
                                 }
                             }
                             if (preg_match('/audio.*?\\b(\\w+)\\b/i', $nfo, $matches)) {
                                 if (preg_match('/(Chinese|German|Dutch|Spanish|Hebrew|Finnish|Norwegian)/i', $matches[1])) {
                                     $foundName = $foundName . "." . $matches[1];
                                 }
                             }
                             if (preg_match('/(video|resolution|video res).*?(1080|720|816|820|272|1280 @|528|1920)/i', $nfo, $matches)) {
                                 if ($matches[2] == '1280 @') {
                                     $matches[2] = '720';
                                 }
                                 if ($matches[2] == '1920') {
                                     $matches[2] = '1080';
                                 }
                                 $foundName = $foundName . "." . $matches[2];
                             }
                             if (preg_match('/source.*?\\b(DVD9|DVD5|BDRIP|DVD\\-?RIP|BLURAY|BD)\\b/i', $nfo, $matches)) {
                                 if ($matches[1] == 'BD') {
                                     $matches[1] = 'Bluray.x264';
                                 }
                                 $foundName = $foundName . "." . $matches[1];
                             }
                             if (preg_match('/(video|resolution|video res).*?(XVID|X264|WMV)/i', $nfo, $matches)) {
                                 $foundName = $foundName . "." . $matches[2];
                             }
                             if (preg_match('/audio.*?\\b(DTS|AC3)\\b/i', $nfo, $matches)) {
                                 $foundName = $foundName . "." . $matches[1];
                             }
                             $foundName = $foundName . "-NoGroup";
                             $methodused = "LastNfoAttempt";
                             $this->determineCategory($rel, $foundName, $methodused);
                         }
                     }
                 }
             }
             if ($foundName == '' && $this->verbose) {
                 echo "ReleaseID: \t\t" . $rel["rid"] . "\n" . " Group: \t\t" . $rel["groupname"] . "\n" . " Old Name: \t\t" . $rel["name"] . "\n" . " Old SearchName: \t" . $rel["searchname"] . "\n" . " Status: \t\tNo new name found.\n\n";
             }
         }
     }
     if ($this->verbose) {
         echo $this->releasestocheck . " releases checked\n" . $this->nfosprocessed . " of " . $this->releasestocheck . " releases had nfo's processed\n" . $this->parsprocessed . " of " . $this->releasestocheck . " releases had par's processed\n" . $this->releasefilesprocessed . " of " . $this->releasestocheck . " releases had releasefiles processed\n" . $this->numupdated . " of " . $this->releasestocheck . " releases " . ($this->releasestocheck > 0 ? floor($this->numupdated / $this->releasestocheck * 100) . "%" : "") . " changed\n";
     }
 }
Example #6
0
 /**
  * Update the list of newsgroups and return an array of messages.
  *
  * @param string $groupList
  * @param int    $active
  * @param int    $backfill
  *
  * @return array
  */
 public function addBulk($groupList, $active = 1, $backfill = 1)
 {
     if (preg_match('/^\\s*$/m', $groupList)) {
         $ret = "No group list provided.";
     } else {
         $nntp = new NNTP(['Echo' => false]);
         if ($nntp->doConnect() !== true) {
             return 'Problem connecting to usenet.';
         }
         $groups = $nntp->getGroups();
         $nntp->doQuit();
         if ($nntp->isError($groups)) {
             return 'Problem fetching groups from usenet.';
         }
         $regFilter = '/' . $groupList . '/i';
         $ret = [];
         foreach ($groups as $group) {
             if (preg_match($regFilter, $group['group']) > 0) {
                 $res = $this->pdo->queryOneRow(sprintf('SELECT id FROM groups WHERE name = %s', $this->pdo->escapeString($group['group'])));
                 if ($res === false) {
                     $this->pdo->queryInsert(sprintf('INSERT INTO groups (name, active, backfill) VALUES (%s, %d, %d)', $this->pdo->escapeString($group['group']), $active, $backfill));
                     $ret[] = ['group' => $group['group'], 'msg' => 'Created'];
                 }
             }
         }
         if (count($ret) === 0) {
             $ret = 'No groups found with your regex, try again!';
         }
     }
     return $ret;
 }
Example #7
0
 /**
  * Check for passworded releases, RAR contents and Sample/Media info
  */
 public function processAdditional($numtoProcess = 100)
 {
     $maxattemptstocheckpassworded = 5;
     $processVideoSample = $this->pdo->getSetting('ffmpegpath') != '' ? true : false;
     $processMediainfo = $this->pdo->getSetting('mediainfopath') != '' ? true : false;
     $processPasswords = $this->pdo->getSetting('unrarpath') != '' ? true : false;
     $processAudioSample = $this->pdo->getSetting('saveaudiopreview') == 1 ? true : false;
     $tmpPath = $this->pdo->getSetting('tmpunrarpath');
     if (substr($tmpPath, -strlen('/')) != '/') {
         $tmpPath = $tmpPath . '/';
     }
     if (!file_exists($tmpPath)) {
         mkdir($tmpPath, 0766, true);
     }
     $nntp = new NNTP();
     $nzb = new NZB();
     //
     // Get out all releases which have not been checked more than max attempts for password.
     //
     $sql = sprintf("select r.id, r.guid, r.name, c.disablepreview from releases r\n\t\t\tleft join category c on c.id = r.categoryid\n\t\t\twhere (r.passwordstatus between %d and -1)\n\t\t\tor (r.haspreview = -1 and c.disablepreview = 0) order by r.postdate desc limit %d ", ($maxattemptstocheckpassworded + 1) * -1, $numtoProcess);
     $result = $this->pdo->query($sql);
     $iteration = $rescount = sizeof($result);
     if ($rescount > 0) {
         echo "PostPrc : Performing additional post processing on last " . $rescount . " releases ...";
         $nntpconnected = false;
         foreach ($result as $rel) {
             echo $iteration-- . ".";
             // Per release defaults
             $passStatus = array(Releases::PASSWD_NONE);
             $blnTookMediainfo = false;
             $blnTookSample = $rel['disablepreview'] == 1 ? true : false;
             //only attempt sample if not disabled
             if ($blnTookSample) {
                 $this->pdo->queryExec(sprintf("update releases set haspreview = 0 where id = %d", $rel['id']));
             }
             //
             // Go through the binaries for this release looking for a rar, a sample, and a mediafile
             //
             $nzbInfo = new nzbInfo();
             $norar = 0;
             // only load nzbs and check for rar files if we are doing something with them.
             if ($processVideoSample || $processMediainfo || $processPasswords || $processAudioSample) {
                 $nzbfile = $nzb->getNZBPath($rel['guid'], $this->pdo->getSetting('nzbpath'));
                 if (!$nzbInfo->loadFromFile($nzbfile)) {
                     continue;
                 }
                 foreach ($nzbInfo->nzb as $nzbsubject) {
                     if (preg_match("/\\w\\.r00/i", $nzbsubject['subject'])) {
                         $norar = 1;
                     }
                 }
             }
             // attempt to process video sample file
             if (!empty($nzbInfo->samplefiles) && $processVideoSample && $blnTookSample === false) {
                 $sampleFile = $nzbInfo->samplefiles[0];
                 //first detected sample
                 $sampleMsgids = array_slice($sampleFile['segments'], 0, 1);
                 //get first segment, increase to get more of the sample
                 $sampleGroup = $sampleFile['groups'][0];
                 //echo "PostPrc : Fetching ".implode($sampleMsgids, ', ')." from {$sampleGroup}\n";
                 if (!$nntpconnected) {
                     $nntpconnected = $nntp->doConnect();
                 }
                 $sampleBinary = $nntp->getMessages($sampleGroup, $sampleMsgids);
                 if ($sampleBinary === false) {
                     echo "\nPostPrc : Couldnt fetch sample\n";
                 } else {
                     $samplefile = $tmpPath . 'sample.avi';
                     file_put_contents($samplefile, $sampleBinary);
                     $blnTookSample = $this->getSample($tmpPath, $this->pdo->getSetting('ffmpegpath'), $rel['guid']);
                     if ($blnTookSample) {
                         $this->updateReleaseHasPreview($rel['guid']);
                     }
                     unlink($samplefile);
                 }
                 unset($sampleBinary);
             }
             // attempt to process loose media file
             if (!empty($nzbInfo->mediafiles) && ($processVideoSample && $blnTookSample === false || $processMediainfo)) {
                 $mediaFile = $nzbInfo->mediafiles[0];
                 //first detected media file
                 $mediaMsgids = array_slice($mediaFile['segments'], 0, 2);
                 //get first two segments
                 $mediaGroup = $mediaFile['groups'][0];
                 //echo "PostPrc : Fetching ".implode($mediaMsgids, ', ')." from {$mediaGroup}\n";
                 if (!$nntpconnected) {
                     $nntpconnected = $nntp->doConnect();
                 }
                 $mediaBinary = $nntp->getMessages($mediaGroup, $mediaMsgids);
                 if ($mediaBinary === false) {
                     echo "\nPostPrc : Couldnt fetch media file\n";
                 } else {
                     $mediafile = $tmpPath . 'sample.avi';
                     file_put_contents($mediafile, $mediaBinary);
                     if ($processVideoSample && $blnTookSample === false) {
                         $blnTookSample = $this->getSample($tmpPath, $this->pdo->getSetting('ffmpegpath'), $rel['guid']);
                         if ($blnTookSample) {
                             $this->updateReleaseHasPreview($rel['guid']);
                         }
                     }
                     if ($processMediainfo) {
                         $blnTookMediainfo = $this->getMediainfo($tmpPath, $this->pdo->getSetting('mediainfopath'), $rel['id']);
                     }
                     unlink($mediafile);
                 }
                 unset($mediaBinary);
             }
             // attempt to process audio sample file
             if (!empty($nzbInfo->audiofiles) && $processAudioSample && $blnTookSample === false) {
                 $audioFile = $nzbInfo->audiofiles[0];
                 //first detected audio file
                 $audioMsgids = array_slice($audioFile['segments'], 0, 1);
                 //get first segment
                 $audioGroup = $audioFile['groups'][0];
                 //echo "PostPrc : Fetching ".implode($audioMsgids, ', ')." from {$audioGroup}\n";
                 if (!$nntpconnected) {
                     $nntpconnected = $nntp->doConnect();
                 }
                 $audioBinary = $nntp->getMessages($audioGroup, $audioMsgids);
                 if ($audioBinary === false) {
                     echo "\nPostPrc : Couldnt fetch audio sample\n";
                 } else {
                     $audiofile = $tmpPath . 'sample.mp3';
                     file_put_contents($audiofile, $audioBinary);
                     $blnTookSample = $this->getAudioSample($tmpPath, $rel['guid']);
                     if ($blnTookSample !== false) {
                         $this->updateReleaseHasPreview($rel['guid'], 2);
                     }
                     if ($processMediainfo) {
                         $blnTookMediainfo = $this->getMediainfo($tmpPath, $this->pdo->getSetting('mediainfopath'), $rel['id']);
                     }
                     if ($this->pdo->getSetting('lamepath') != "") {
                         $this->lameAudioSample($this->pdo->getSetting('lamepath'), $rel['guid']);
                     }
                     unlink($audiofile);
                 }
                 unset($audioBinary);
             }
             if (!empty($nzbInfo->rarfiles) && ($this->pdo->getSetting('checkpasswordedrar') > 0 || ($processVideoSample || $processAudioSample) && $blnTookSample === false || $processMediainfo)) {
                 $mysqlkeepalive = 0;
                 foreach ($nzbInfo->rarfiles as $rarFile) {
                     //dont process any more rars if a passworded rar has been detected and the site is set to automatically delete them
                     if ($this->pdo->getSetting('deletepasswordedrelease') == 1 && max($passStatus) == Releases::PASSWD_RAR) {
                         echo "-Skipping processing of rar {$rarFile['subject']} as this release has already been marked as passworded.\n";
                         continue;
                     }
                     $rarMsgids = array_slice($rarFile['segments'], 0, 1);
                     //get first segment
                     $rarGroup = $rarFile['groups'][0];
                     //echo "PostPrc : Fetching ".implode($rarMsgids, ', ')." from {$rarGroup} (".++$mysqlkeepalive.")\n";
                     if (!$nntpconnected) {
                         $nntpconnected = $nntp->doConnect();
                     }
                     $fetchedBinary = $nntp->getMessages($rarGroup, $rarMsgids);
                     if ($fetchedBinary === false) {
                         //echo "\nPostPrc : Failed fetching rar file\n";
                         $this->pdo->queryExec(sprintf("update releases set passwordstatus = passwordstatus - 1 where id = %d", $rel['id']));
                         continue;
                     } else {
                         $relFiles = $this->processReleaseFiles($fetchedBinary, $rel['id']);
                         if ($this->pdo->getSetting('checkpasswordedrar') > 0 && $processPasswords) {
                             $passStatus[] = $this->processReleasePasswords($fetchedBinary, $tmpPath, $this->pdo->getSetting('unrarpath'), $this->pdo->getSetting('checkpasswordedrar'));
                         }
                         // we need to unrar the fetched binary if checkpasswordedrar wasnt 2
                         if ($this->pdo->getSetting('checkpasswordedrar') < 2 && $processPasswords) {
                             $rarfile = $tmpPath . 'rarfile.rar';
                             file_put_contents($rarfile, $fetchedBinary);
                             $execstring = '"' . $this->pdo->getSetting('unrarpath') . '" e -ai -ep -c- -id -r -kb -p- -y -inul "' . $rarfile . '" "' . $tmpPath . '"';
                             $output = Utility::runCmd($execstring, false, true);
                             unlink($rarfile);
                         }
                         if ($processVideoSample && $blnTookSample === false) {
                             $blnTookSample = $this->getSample($tmpPath, $this->pdo->getSetting('ffmpegpath'), $rel['guid']);
                             if ($blnTookSample) {
                                 $this->updateReleaseHasPreview($rel['guid']);
                             }
                         }
                         $blnTookAudioSample = false;
                         if ($processAudioSample && $blnTookSample === false) {
                             $blnTookSample = $this->getAudioSample($tmpPath, $rel['guid']);
                             if ($blnTookSample) {
                                 $blnTookAudioSample = true;
                                 $this->updateReleaseHasPreview($rel['guid'], 2);
                             }
                         }
                         if ($processMediainfo && $blnTookMediainfo === false) {
                             $blnTookMediainfo = $this->getMediainfo($tmpPath, $this->pdo->getSetting('mediainfopath'), $rel['id']);
                         }
                         //
                         // Has to be done after mediainfo
                         //
                         if ($blnTookAudioSample && $this->pdo->getSetting('lamepath') != "") {
                             $this->lameAudioSample($this->pdo->getSetting('lamepath'), $rel['guid']);
                         }
                         if ($mysqlkeepalive % 25 == 0) {
                             $this->pdo->query("select 1");
                         }
                     }
                     //clean up all files
                     foreach (glob($tmpPath . '*') as $v) {
                         unlink($v);
                     }
                 }
                 //end foreach msgid
             } elseif (empty($nzbInfo->rarfiles) && $norar == 1) {
                 $passStatus[] = Releases::PASSWD_POTENTIAL;
             }
             $hpsql = '';
             if (!$blnTookSample) {
                 $hpsql = ', haspreview = 0';
             }
             $sql = sprintf("update releases set passwordstatus = %d %s where id = %d", max($passStatus), $hpsql, $rel["id"]);
             $this->pdo->queryExec($sql);
         }
         //end foreach result
         if ($nntpconnected) {
             $nntp->doQuit();
         }
         echo "\n";
     }
 }
Example #8
0
 /**
  * Update the list of newsgroups from nntp provider matching a regex and return an array of messages.
  */
 function addBulk($groupList, $active = 1, $backfill = 1)
 {
     $ret = array();
     if ($groupList == "") {
         $ret[] = "No group list provided.";
     } else {
         $nntp = new NNTP(['Echo' => false]);
         if (!$nntp->doConnect()) {
             $ret[] = "Failed to get NNTP connection";
             return $ret;
         }
         $groups = $nntp->getGroups();
         $nntp->doQuit();
         $regfilter = "/(" . str_replace(array('.', '*'), array('\\.', '.*?'), $groupList) . ")\$/";
         foreach ($groups as $group) {
             if (preg_match($regfilter, $group['group']) > 0) {
                 $res = $this->pdo->queryOneRow(sprintf("SELECT id FROM groups WHERE name = %s ", $this->pdo->escapeString($group['group'])));
                 if ($res) {
                     $this->pdo->queryExec(sprintf("update groups SET active = %d where id = %d", $active, $res["id"]));
                     $ret[] = array('group' => $group['group'], 'msg' => 'Updated');
                 } else {
                     $desc = "";
                     $this->pdo->queryInsert(sprintf("INSERT INTO groups (name, description, active, backfill) VALUES (%s, %s, %d, %s)", $this->pdo->escapeString($group['group']), $this->pdo->escapeString($desc), $active, $backfill));
                     $ret[] = array('group' => $group['group'], 'msg' => 'Created');
                 }
             }
         }
     }
     return $ret;
 }
Example #9
0
 public function fetchTestBinaries($groupname, $numarticles, $clearexistingbins)
 {
     $nntp = new NNTP();
     $binaries = new Binaries();
     $groups = new Groups();
     $ret = [];
     if ($clearexistingbins == true) {
         $this->pdo->queryExec('truncate releaseregextesting');
     }
     $nntp->doConnect();
     $groupsToFetch = [];
     if (preg_match('/^[a-z]{2,3}(\\.[a-z0-9\\-]+)+$/', $groupname)) {
         $groupsToFetch[] = array('name' => $groupname);
     } elseif ($groupname === 0) {
         $groupsToFetch = $groups->getAll();
     } else {
         $newsgroups = $nntp->getGroups();
         foreach ($newsgroups as $ngroup) {
             if (preg_match('/' . $groupname . '/', $ngroup['group'])) {
                 $groupsToFetch[] = array('name' => $ngroup['group']);
             }
         }
     }
     foreach ($groupsToFetch as $groupArr) {
         $group = $groupArr['name'];
         $data = $nntp->selectGroup($group);
         if (NNTP::isError($data)) {
             $ret[] = "Could not select group (doesnt exist on USP): {$group}";
             continue;
         } else {
             $rangeStart = $data['last'] - $numarticles;
             $rangeEnd = $groupEnd = $data['last'];
             $rangeTotal = $rangeEnd - $rangeStart;
             $done = false;
             while ($done === false) {
                 if ($rangeTotal > $binaries->messageBuffer) {
                     if ($rangeStart + $binaries->messageBuffer > $groupEnd) {
                         $rangeEnd = $groupEnd;
                     } else {
                         $rangeEnd = $rangeStart + $binaries->messageBuffer;
                     }
                 }
                 $msgs = $nntp->getXOver($rangeStart . "-" . $rangeEnd, true, false);
                 if (NNTP::isError($msgs)) {
                     $ret[] = "Error {$msgs->code}: {$msgs->message} on " . $group;
                     continue 2;
                 }
                 $headers = [];
                 if (is_array($msgs)) {
                     //loop headers, figure out parts
                     foreach ($msgs as $msg) {
                         if (!isset($msg['Number'])) {
                             continue;
                         }
                         $msgPart = $msgTotalParts = 0;
                         $pattern = '|\\((\\d+)[\\/](\\d+)\\)|i';
                         preg_match_all($pattern, $msg['Subject'], $matches, PREG_PATTERN_ORDER);
                         $matchcnt = sizeof($matches[0]);
                         for ($i = 0; $i < $matchcnt; $i++) {
                             //not (int)'d here because of the preg_replace later on
                             $msgPart = $matches[1][$i];
                             $msgTotalParts = $matches[2][$i];
                         }
                         if (!isset($msg['Subject']) || $matchcnt == 0) {
                             // not a binary post most likely.. continue
                             continue;
                         }
                         if ((int) $msgPart > 0 && (int) $msgTotalParts > 0) {
                             $subject = utf8_encode(trim(preg_replace('|\\(' . $msgPart . '[\\/]' . $msgTotalParts . '\\)|i', '', $msg['Subject'])));
                             if (!isset($headers[$subject])) {
                                 $headers[$subject]['Subject'] = $subject;
                                 $headers[$subject]['From'] = $msg['From'];
                                 $headers[$subject]['Date'] = strtotime($msg['Date']);
                                 $headers[$subject]['Message-ID'] = $msg['Message-ID'];
                                 $headers[$subject]['Size'] = $msg['Bytes'];
                             } else {
                                 $headers[$subject]['Size'] += $msg['Bytes'];
                             }
                         }
                     }
                     unset($msgs);
                     if (isset($headers) && count($headers)) {
                         $groupRegexes = $this->getForGroup($group);
                         $binSetData = [];
                         foreach ($headers as $subject => $data) {
                             $binData = array('name' => $subject, 'fromname' => $data['From'], 'date' => $data['Date'], 'binaryhash' => md5($subject . $data['From'] . $group), 'groupname' => $group, 'regexid' => "null", 'categoryid' => "null", 'reqid' => "null", 'blacklistid' => 0, 'size' => $data['Size'], 'relname' => "null", 'relpart' => "null", 'reltotalpart' => "null");
                             //Filter binaries based on black/white list
                             if ($binaries->isBlackListed($data, $group)) {
                                 //binary is blacklisted
                                 $binData['blacklistid'] = 1;
                             }
                             //Apply Regexes
                             $regexMatches = [];
                             foreach ($groupRegexes as $groupRegex) {
                                 $regexCheck = $this->performMatch($groupRegex, $subject, $data['From']);
                                 if ($regexCheck !== false) {
                                     $regexMatches = $regexCheck;
                                     $binData['regexid'] = $regexCheck['regexid'];
                                     $binData['categoryid'] = $regexCheck['regcatid'];
                                     $binData['reqid'] = empty($regexCheck['reqid']) ? "null" : $regexCheck['reqid'];
                                     $binData['relname'] = $regexCheck['name'];
                                     break;
                                 }
                             }
                             $binSetData[] = $binData;
                         }
                         //insert 500 bins at a time
                         $binChunks = array_chunk($binSetData, 500);
                         foreach ($binChunks as $binChunk) {
                             foreach ($binChunk as $chunk) {
                                 $binParams[] = sprintf("(%s, %s, FROM_UNIXTIME(%s), %s, %s, %s, %s, %s, %d, %d, now())", $this->pdo->escapeString($chunk['name']), $this->pdo->escapeString($chunk['fromname']), $this->pdo->escapeString($chunk['date']), $this->pdo->escapeString($chunk['binaryhash']), $this->pdo->escapeString($chunk['groupname']), $chunk['regexid'], $chunk['categoryid'], $chunk['reqid'], $chunk['blacklistid'], $chunk['size']);
                             }
                             $binSql = "INSERT IGNORE INTO releaseregextesting (name, fromname, date, binaryhash, groupname, regexid, categoryid, reqid, blacklistid, size, dateadded) VALUES " . implode(', ', $binParams);
                             //echo $binSql;
                             $this->pdo->queryExec($binSql);
                         }
                         $ret[] = "Fetched " . number_format($numarticles) . " articles from " . $group;
                     } else {
                         $ret[] = "No headers found on " . $group;
                         continue;
                     }
                 } else {
                     $ret[] = "Can't get parts from server (msgs not array) on " . $group;
                     continue;
                 }
                 if ($rangeEnd == $groupEnd) {
                     $done = true;
                 }
                 $rangeStart = $rangeEnd + 1;
             }
         }
     }
     $nntp->doQuit();
     return $ret;
 }
Example #10
0
 /**
  * Loop over range of wanted headers, insert headers into DB.
  *
  * @param array      $groupMySQL   The group info from mysql.
  * @param int        $first        The oldest wanted header.
  * @param int        $last         The newest wanted header.
  * @param string     $type         Is this partrepair or update or backfill?
  * @param null|array $missingParts If we are running in partrepair, the list of missing article numbers.
  *
  * @return array Empty on failure.
  */
 public function scan($groupMySQL, $first, $last, $type = 'update', $missingParts = null)
 {
     // Start time of scan method and of fetching headers.
     $startLoop = microtime(true);
     // Check if MySQL tables exist, create if they do not, get their names at the same time.
     $tableNames = $this->_groups->getCBPTableNames($this->_tablePerGroup, $groupMySQL['id']);
     $returnArray = [];
     $partRepair = $type === 'partrepair';
     $addToPartRepair = $type === 'update' && $this->_partRepair;
     // Download the headers.
     if ($partRepair === true) {
         // This is slower but possibly is better with missing headers.
         $headers = $this->_nntp->getOverview($first . '-' . $last, true, false);
     } else {
         $headers = $this->_nntp->getXOVER($first . '-' . $last);
     }
     // If there was an error, try to reconnect.
     if ($this->_nntp->isError($headers)) {
         // Increment if part repair and return false.
         if ($partRepair === true) {
             $this->_pdo->queryExec(sprintf('UPDATE partrepair SET attempts = attempts + 1 WHERE group_id = %d AND numberid %s', $groupMySQL['id'], $first == $last ? '= ' . $first : 'IN (' . implode(',', range($first, $last)) . ')'));
             return $returnArray;
         }
         // This is usually a compression error, so try disabling compression.
         $this->_nntp->doQuit();
         if ($this->_nntp->doConnect(false) !== true) {
             return $returnArray;
         }
         // Re-select group, download headers again without compression and re-enable compression.
         $this->_nntp->selectGroup($groupMySQL['name']);
         $headers = $this->_nntp->getXOVER($first . '-' . $last);
         $this->_nntp->enableCompression();
         // Check if the non-compression headers have an error.
         if ($this->_nntp->isError($headers)) {
             $this->log("Code {$headers->code}: {$headers->message}\nSkipping group: {${$groupMySQL['name']}}", 'scan', Logger::LOG_WARNING, 'error');
             return $returnArray;
         }
     }
     // Start of processing headers.
     $startCleaning = microtime(true);
     // End of the getting data from usenet.
     $timeHeaders = number_format($startCleaning - $startLoop, 2);
     // Check if we got headers.
     $msgCount = count($headers);
     if ($msgCount < 1) {
         return $returnArray;
     }
     // Get highest and lowest article numbers/dates.
     $iterator1 = 0;
     $iterator2 = $msgCount - 1;
     while (true) {
         if (!isset($returnArray['firstArticleNumber']) && isset($headers[$iterator1]['Number'])) {
             $returnArray['firstArticleNumber'] = $headers[$iterator1]['Number'];
             $returnArray['firstArticleDate'] = $headers[$iterator1]['Date'];
         }
         if (!isset($returnArray['lastArticleNumber']) && isset($headers[$iterator2]['Number'])) {
             $returnArray['lastArticleNumber'] = $headers[$iterator2]['Number'];
             $returnArray['lastArticleDate'] = $headers[$iterator2]['Date'];
         }
         // Break if we found non empty articles.
         if (isset($returnArray['firstArticleNumber']) && isset($returnArray['lastArticleNumber'])) {
             break;
         }
         // Break out if we couldn't find anything.
         if ($iterator1++ >= $msgCount - 1 || $iterator2-- <= 0) {
             break;
         }
     }
     $headersRepaired = $articles = $rangeNotReceived = $collectionIDs = $binariesUpdate = $headersReceived = $headersNotInserted = [];
     $notYEnc = $headersBlackListed = 0;
     $partsQuery = $partsCheck = sprintf('INSERT INTO %s (binaryid, number, messageid, partnumber, size, collection_id) VALUES ', $tableNames['pname']);
     $this->_pdo->beginTransaction();
     // Loop articles, figure out files/parts.
     foreach ($headers as $header) {
         // Check if we got the article or not.
         if (isset($header['Number'])) {
             $headersReceived[] = $header['Number'];
         } else {
             if ($addToPartRepair) {
                 $rangeNotReceived[] = $header['Number'];
             }
             continue;
         }
         // If set we are running in partRepair mode.
         if ($partRepair === true && !is_null($missingParts)) {
             if (!in_array($header['Number'], $missingParts)) {
                 // If article isn't one that is missing skip it.
                 continue;
             } else {
                 // We got the part this time. Remove article from part repair.
                 $headersRepaired[] = $header['Number'];
             }
         }
         /*
          * Find part / total parts. Ignore if no part count found.
          *
          * \s* Trims the leading space.
          * (?!"Usenet Index Post) ignores these types of articles, they are useless.
          * (.+) Fetches the subject.
          * \s+ Trims trailing space after the subject.
          * \((\d+)\/(\d+)\) Gets the part count.
          * No ending ($) as there are cases of subjects with extra data after the part count.
          */
         if (preg_match('/^\\s*(?!"Usenet Index Post)(.+)\\s+\\((\\d+)\\/(\\d+)\\)/', $header['Subject'], $matches)) {
             // Add yEnc to subjects that do not have them, but have the part number at the end of the header.
             if (!stristr($header['Subject'], 'yEnc')) {
                 $matches[1] .= ' yEnc';
             }
         } else {
             if ($this->_showDroppedYEncParts === true && strpos($header['Subject'], '"Usenet Index Post') !== 0) {
                 file_put_contents(nZEDb_LOGS . 'not_yenc' . $groupMySQL['name'] . '.dropped.log', $header['Subject'] . PHP_EOL, FILE_APPEND);
             }
             $notYEnc++;
             continue;
         }
         // Filter subject based on black/white list.
         if ($this->_blackListEmpty === false && $this->isBlackListed($header, $groupMySQL['name'])) {
             $headersBlackListed++;
             continue;
         }
         if (!isset($header['Bytes'])) {
             $header['Bytes'] = isset($header[':bytes']) ? $header[':bytes'] : 0;
         }
         $header['Bytes'] = (int) $header['Bytes'];
         // Set up the info for inserting into parts/binaries/collections tables.
         if (!isset($articles[$matches[1]])) {
             // Attempt to find the file count. If it is not found, set it to 0.
             if (!preg_match('/[[(\\s](\\d{1,5})(\\/|[\\s_]of[\\s_]|-)(\\d{1,5})[])\\s$:]/i', $matches[1], $fileCount)) {
                 $fileCount[1] = $fileCount[3] = 0;
                 if ($this->_showDroppedYEncParts === true) {
                     file_put_contents(nZEDb_LOGS . 'no_files' . $groupMySQL['name'] . '.log', $header['Subject'] . PHP_EOL, FILE_APPEND);
                 }
             }
             // Used to group articles together when forming the release/nzb.
             $header['CollectionKey'] = $this->_collectionsCleaning->collectionsCleaner($matches[1], $groupMySQL['name']) . $header['From'] . $groupMySQL['id'] . $fileCount[3];
             if (!isset($collectionIDs[$header['CollectionKey']])) {
                 /* Date from header should be a string this format:
                  * 31 Mar 2014 15:36:04 GMT or 6 Oct 1998 04:38:40 -0500
                  * Still make sure it's not unix time, convert it to unix time if it is.
                  */
                 $header['Date'] = is_numeric($header['Date']) ? $header['Date'] : strtotime($header['Date']);
                 // Get the current unixtime from PHP.
                 $now = time();
                 $collectionID = $this->_pdo->queryInsert(sprintf("\n\t\t\t\t\t\t\tINSERT INTO %s (subject, fromname, date, xref, group_id,\n\t\t\t\t\t\t\t\ttotalfiles, collectionhash, dateadded)\n\t\t\t\t\t\t\tVALUES (%s, %s, FROM_UNIXTIME(%s), %s, %d, %d, '%s', NOW())\n\t\t\t\t\t\t\tON DUPLICATE KEY UPDATE dateadded = NOW()", $tableNames['cname'], $this->_pdo->escapeString(substr(utf8_encode($matches[1]), 0, 255)), $this->_pdo->escapeString(utf8_encode($header['From'])), is_numeric($header['Date']) ? $header['Date'] > $now ? $now : $header['Date'] : $now, $this->_pdo->escapeString(substr($header['Xref'], 0, 255)), $groupMySQL['id'], $fileCount[3], sha1($header['CollectionKey'])));
                 if ($collectionID === false) {
                     if ($addToPartRepair) {
                         $headersNotInserted[] = $header['Number'];
                     }
                     $this->_pdo->Rollback();
                     $this->_pdo->beginTransaction();
                     continue;
                 }
                 $collectionIDs[$header['CollectionKey']] = $collectionID;
             } else {
                 $collectionID = $collectionIDs[$header['CollectionKey']];
             }
             $binaryID = $this->_pdo->queryInsert(sprintf("\n\t\t\t\t\t\tINSERT INTO %s (binaryhash, name, collectionid, totalparts, currentparts, filenumber, partsize)\n\t\t\t\t\t\tVALUES ('%s', %s, %d, %d, 1, %d, %d)\n\t\t\t\t\t\tON DUPLICATE KEY UPDATE currentparts = currentparts + 1, partsize = partsize + %d", $tableNames['bname'], md5($matches[1] . $header['From'] . $groupMySQL['id']), $this->_pdo->escapeString(utf8_encode($matches[1])), $collectionID, $matches[3], $fileCount[1], $header['Bytes'], $header['Bytes']));
             if ($binaryID === false) {
                 if ($addToPartRepair) {
                     $headersNotInserted[] = $header['Number'];
                 }
                 $this->_pdo->Rollback();
                 $this->_pdo->beginTransaction();
                 continue;
             }
             $binariesUpdate[$binaryID]['Size'] = 0;
             $binariesUpdate[$binaryID]['Parts'] = 0;
             $articles[$matches[1]]['CollectionID'] = $collectionID;
             $articles[$matches[1]]['BinaryID'] = $binaryID;
         } else {
             $binaryID = $articles[$matches[1]]['BinaryID'];
             $collectionID = $articles[$matches[1]]['CollectionID'];
             $binariesUpdate[$binaryID]['Size'] += $header['Bytes'];
             $binariesUpdate[$binaryID]['Parts']++;
         }
         // Strip the < and >, saves space in DB.
         $header['Message-ID'][0] = "'";
         $partsQuery .= '(' . $binaryID . ',' . $header['Number'] . ',' . rtrim($header['Message-ID'], '>') . "'," . $matches[2] . ',' . $header['Bytes'] . ',' . $collectionID . '),';
     }
     unset($headers);
     // Reclaim memory.
     // Start of inserting into SQL.
     $startUpdate = microtime(true);
     // End of processing headers.
     $timeCleaning = number_format($startUpdate - $startCleaning, 2);
     $binariesQuery = $binariesCheck = sprintf('INSERT INTO %s (id, partsize, currentparts) VALUES ', $tableNames['bname']);
     foreach ($binariesUpdate as $binaryID => $binary) {
         $binariesQuery .= '(' . $binaryID . ',' . $binary['Size'] . ',' . $binary['Parts'] . '),';
     }
     $binariesEnd = ' ON DUPLICATE KEY UPDATE partsize = VALUES(partsize) + partsize, currentparts = VALUES(currentparts) + currentparts';
     $binariesQuery = rtrim($binariesQuery, ',') . $binariesEnd;
     // Check if we got any binaries. If we did, try to insert them.
     if (strlen($binariesCheck . $binariesEnd) === strlen($binariesQuery) ? true : $this->_pdo->queryExec($binariesQuery)) {
         if ($this->_debug) {
             $this->_colorCLI->doEcho($this->_colorCLI->debug('Sending ' . round(strlen($partsQuery) / 1024, 2) . ' KB of parts to MySQL'));
         }
         if (strlen($partsQuery) === strlen($partsCheck) ? true : $this->_pdo->queryExec(rtrim($partsQuery, ','))) {
             $this->_pdo->Commit();
         } else {
             if ($addToPartRepair) {
                 $headersNotInserted += $headersReceived;
             }
             $this->_pdo->Rollback();
         }
     } else {
         if ($addToPartRepair) {
             $headersNotInserted += $headersReceived;
         }
         $this->_pdo->Rollback();
     }
     if ($this->_echoCLI && $partRepair === false) {
         $this->_colorCLI->doEcho($this->_colorCLI->primary('Received ' . count($headersReceived) . ' articles of ' . number_format($last - $first + 1) . ' requested, ' . $headersBlackListed . ' blacklisted, ' . $notYEnc . ' not yEnc.'));
     }
     // Start of part repair.
     $startPR = microtime(true);
     // End of inserting.
     $timeInsert = number_format($startPR - $startUpdate, 2);
     if ($partRepair && count($headersRepaired) > 0) {
         $this->removeRepairedParts($headersRepaired, $tableNames['prname'], $groupMySQL['id']);
     }
     if ($addToPartRepair) {
         $notInsertedCount = count($headersNotInserted);
         if ($notInsertedCount > 0) {
             $this->addMissingParts($headersNotInserted, $tableNames['prname'], $groupMySQL['id']);
             $this->log($notInsertedCount . ' articles failed to insert!', 'scan', Logger::LOG_WARNING, 'warning');
         }
         // Check if we have any missing headers.
         if ($last - $first - $notYEnc - $headersBlackListed + 1 > count($headersReceived)) {
             $rangeNotReceived = array_merge($rangeNotReceived, array_diff(range($first, $last), $headersReceived));
         }
         $notReceivedCount = count($rangeNotReceived);
         if ($notReceivedCount > 0) {
             $this->addMissingParts($rangeNotReceived, $tableNames['prname'], $groupMySQL['id']);
             if ($this->_echoCLI) {
                 $this->_colorCLI->doEcho($this->_colorCLI->alternate('Server did not return ' . $notReceivedCount . ' articles from ' . $groupMySQL['name'] . '.'), true);
             }
         }
     }
     $currentMicroTime = microtime(true);
     if ($this->_echoCLI) {
         $this->_colorCLI->doEcho($this->_colorCLI->alternateOver($timeHeaders . 's') . $this->_colorCLI->primaryOver(' to download articles, ') . $this->_colorCLI->alternateOver($timeCleaning . 's') . $this->_colorCLI->primaryOver(' to process collections, ') . $this->_colorCLI->alternateOver($timeInsert . 's') . $this->_colorCLI->primaryOver(' to insert binaries/parts, ') . $this->_colorCLI->alternateOver(number_format($currentMicroTime - $startPR, 2) . 's') . $this->_colorCLI->primaryOver(' for part repair, ') . $this->_colorCLI->alternateOver(number_format($currentMicroTime - $startLoop, 2) . 's') . $this->_colorCLI->primary(' total.'));
     }
     return $returnArray;
 }