<?php require_once dirname(__FILE__) . '/../../../config.php'; use nzedb\db\Settings; $start = TIME(); $pdo = new Settings(); $consoleTools = new ConsoleTools(['ColorCLI' => $pdo->log]); // Create the connection here and pass $nntp = new NNTP(['Settings' => $pdo]); if ($nntp->doConnect() !== true) { exit($pdo->log->error("Unable to connect to usenet.")); } echo $pdo->log->header("Getting first/last for all your active groups."); $data = $nntp->getGroups(); if ($nntp->isError($data)) { exit($pdo->log->error("Failed to getGroups() from nntp server.")); } echo $pdo->log->header("Inserting new values into shortgroups table."); $pdo->queryExec('TRUNCATE TABLE shortgroups'); // Put into an array all active groups $res = $pdo->query('SELECT name FROM groups WHERE active = 1 OR backfill = 1'); foreach ($data as $newgroup) { if (myInArray($res, $newgroup['group'], 'name')) { $pdo->queryInsert(sprintf('INSERT INTO shortgroups (name, first_record, last_record, updated) VALUES (%s, %s, %s, NOW())', $pdo->escapeString($newgroup['group']), $pdo->escapeString($newgroup['first']), $pdo->escapeString($newgroup['last']))); echo $pdo->log->primary('Updated ' . $newgroup['group']); } } echo $pdo->log->header('Running time: ' . $consoleTools->convertTimer(TIME() - $start)); function myInArray($array, $value, $key) { //loop through the array
/** * Update the list of newsgroups from nntp provider matching a regex and return an array of messages. */ function addBulk($groupList, $active = 1, $backfill = 1) { $ret = array(); if ($groupList == "") { $ret[] = "No group list provided."; } else { $nntp = new NNTP(['Echo' => false]); if (!$nntp->doConnect()) { $ret[] = "Failed to get NNTP connection"; return $ret; } $groups = $nntp->getGroups(); $nntp->doQuit(); $regfilter = "/(" . str_replace(array('.', '*'), array('\\.', '.*?'), $groupList) . ")\$/"; foreach ($groups as $group) { if (preg_match($regfilter, $group['group']) > 0) { $res = $this->pdo->queryOneRow(sprintf("SELECT id FROM groups WHERE name = %s ", $this->pdo->escapeString($group['group']))); if ($res) { $this->pdo->queryExec(sprintf("update groups SET active = %d where id = %d", $active, $res["id"])); $ret[] = array('group' => $group['group'], 'msg' => 'Updated'); } else { $desc = ""; $this->pdo->queryInsert(sprintf("INSERT INTO groups (name, description, active, backfill) VALUES (%s, %s, %d, %s)", $this->pdo->escapeString($group['group']), $this->pdo->escapeString($desc), $active, $backfill)); $ret[] = array('group' => $group['group'], 'msg' => 'Created'); } } } } return $ret; }
/** * Update the list of newsgroups and return an array of messages. * * @param string $groupList * @param int $active * @param int $backfill * * @return array */ public function addBulk($groupList, $active = 1, $backfill = 1) { if (preg_match('/^\\s*$/m', $groupList)) { $ret = "No group list provided."; } else { $nntp = new NNTP(['Echo' => false]); if ($nntp->doConnect() !== true) { return 'Problem connecting to usenet.'; } $groups = $nntp->getGroups(); $nntp->doQuit(); if ($nntp->isError($groups)) { return 'Problem fetching groups from usenet.'; } $regFilter = '/' . $groupList . '/i'; $ret = []; foreach ($groups as $group) { if (preg_match($regFilter, $group['group']) > 0) { $res = $this->pdo->queryOneRow(sprintf('SELECT id FROM groups WHERE name = %s', $this->pdo->escapeString($group['group']))); if ($res === false) { $this->pdo->queryInsert(sprintf('INSERT INTO groups (name, active, backfill) VALUES (%s, %d, %d)', $this->pdo->escapeString($group['group']), $active, $backfill)); $ret[] = ['group' => $group['group'], 'msg' => 'Created']; } } } if (count($ret) === 0) { $ret = 'No groups found with your regex, try again!'; } } return $ret; }
public function fetchTestBinaries($groupname, $numarticles, $clearexistingbins) { $nntp = new NNTP(); $binaries = new Binaries(); $groups = new Groups(); $ret = []; if ($clearexistingbins == true) { $this->pdo->queryExec('truncate releaseregextesting'); } $nntp->doConnect(); $groupsToFetch = []; if (preg_match('/^[a-z]{2,3}(\\.[a-z0-9\\-]+)+$/', $groupname)) { $groupsToFetch[] = array('name' => $groupname); } elseif ($groupname === 0) { $groupsToFetch = $groups->getAll(); } else { $newsgroups = $nntp->getGroups(); foreach ($newsgroups as $ngroup) { if (preg_match('/' . $groupname . '/', $ngroup['group'])) { $groupsToFetch[] = array('name' => $ngroup['group']); } } } foreach ($groupsToFetch as $groupArr) { $group = $groupArr['name']; $data = $nntp->selectGroup($group); if (NNTP::isError($data)) { $ret[] = "Could not select group (doesnt exist on USP): {$group}"; continue; } else { $rangeStart = $data['last'] - $numarticles; $rangeEnd = $groupEnd = $data['last']; $rangeTotal = $rangeEnd - $rangeStart; $done = false; while ($done === false) { if ($rangeTotal > $binaries->messageBuffer) { if ($rangeStart + $binaries->messageBuffer > $groupEnd) { $rangeEnd = $groupEnd; } else { $rangeEnd = $rangeStart + $binaries->messageBuffer; } } $msgs = $nntp->getXOver($rangeStart . "-" . $rangeEnd, true, false); if (NNTP::isError($msgs)) { $ret[] = "Error {$msgs->code}: {$msgs->message} on " . $group; continue 2; } $headers = []; if (is_array($msgs)) { //loop headers, figure out parts foreach ($msgs as $msg) { if (!isset($msg['Number'])) { continue; } $msgPart = $msgTotalParts = 0; $pattern = '|\\((\\d+)[\\/](\\d+)\\)|i'; preg_match_all($pattern, $msg['Subject'], $matches, PREG_PATTERN_ORDER); $matchcnt = sizeof($matches[0]); for ($i = 0; $i < $matchcnt; $i++) { //not (int)'d here because of the preg_replace later on $msgPart = $matches[1][$i]; $msgTotalParts = $matches[2][$i]; } if (!isset($msg['Subject']) || $matchcnt == 0) { // not a binary post most likely.. continue continue; } if ((int) $msgPart > 0 && (int) $msgTotalParts > 0) { $subject = utf8_encode(trim(preg_replace('|\\(' . $msgPart . '[\\/]' . $msgTotalParts . '\\)|i', '', $msg['Subject']))); if (!isset($headers[$subject])) { $headers[$subject]['Subject'] = $subject; $headers[$subject]['From'] = $msg['From']; $headers[$subject]['Date'] = strtotime($msg['Date']); $headers[$subject]['Message-ID'] = $msg['Message-ID']; $headers[$subject]['Size'] = $msg['Bytes']; } else { $headers[$subject]['Size'] += $msg['Bytes']; } } } unset($msgs); if (isset($headers) && count($headers)) { $groupRegexes = $this->getForGroup($group); $binSetData = []; foreach ($headers as $subject => $data) { $binData = array('name' => $subject, 'fromname' => $data['From'], 'date' => $data['Date'], 'binaryhash' => md5($subject . $data['From'] . $group), 'groupname' => $group, 'regexid' => "null", 'categoryid' => "null", 'reqid' => "null", 'blacklistid' => 0, 'size' => $data['Size'], 'relname' => "null", 'relpart' => "null", 'reltotalpart' => "null"); //Filter binaries based on black/white list if ($binaries->isBlackListed($data, $group)) { //binary is blacklisted $binData['blacklistid'] = 1; } //Apply Regexes $regexMatches = []; foreach ($groupRegexes as $groupRegex) { $regexCheck = $this->performMatch($groupRegex, $subject, $data['From']); if ($regexCheck !== false) { $regexMatches = $regexCheck; $binData['regexid'] = $regexCheck['regexid']; $binData['categoryid'] = $regexCheck['regcatid']; $binData['reqid'] = empty($regexCheck['reqid']) ? "null" : $regexCheck['reqid']; $binData['relname'] = $regexCheck['name']; break; } } $binSetData[] = $binData; } //insert 500 bins at a time $binChunks = array_chunk($binSetData, 500); foreach ($binChunks as $binChunk) { foreach ($binChunk as $chunk) { $binParams[] = sprintf("(%s, %s, FROM_UNIXTIME(%s), %s, %s, %s, %s, %s, %d, %d, now())", $this->pdo->escapeString($chunk['name']), $this->pdo->escapeString($chunk['fromname']), $this->pdo->escapeString($chunk['date']), $this->pdo->escapeString($chunk['binaryhash']), $this->pdo->escapeString($chunk['groupname']), $chunk['regexid'], $chunk['categoryid'], $chunk['reqid'], $chunk['blacklistid'], $chunk['size']); } $binSql = "INSERT IGNORE INTO releaseregextesting (name, fromname, date, binaryhash, groupname, regexid, categoryid, reqid, blacklistid, size, dateadded) VALUES " . implode(', ', $binParams); //echo $binSql; $this->pdo->queryExec($binSql); } $ret[] = "Fetched " . number_format($numarticles) . " articles from " . $group; } else { $ret[] = "No headers found on " . $group; continue; } } else { $ret[] = "Can't get parts from server (msgs not array) on " . $group; continue; } if ($rangeEnd == $groupEnd) { $done = true; } $rangeStart = $rangeEnd + 1; } } } $nntp->doQuit(); return $ret; }