/**
  * Update the list of newsgroups from nntp provider matching a regex and return an array of messages.
  */
 function addBulk($groupList, $active = 1)
 {
     require_once WWW_DIR . "/lib/binaries.php";
     require_once WWW_DIR . "/lib/nntp.php";
     $ret = array();
     if ($groupList == "") {
         $ret[] = "No group list provided.";
     } else {
         $db = new DB();
         $nntp = new Nntp();
         if (!$nntp->doConnect()) {
             $ret[] = "Failed to get NNTP connection";
             return $ret;
         }
         $groups = $nntp->getGroups();
         $nntp->doQuit();
         $regfilter = "/(" . str_replace(array('.', '*'), array('\\.', '.*?'), $groupList) . ")\$/";
         foreach ($groups as $group) {
             if (preg_match($regfilter, $group['group']) > 0) {
                 $res = $db->queryOneRow(sprintf("SELECT ID FROM groups WHERE name = %s ", $db->escapeString($group['group'])));
                 if ($res) {
                     $db->exec(sprintf("update groups SET active = %d where ID = %d", $active, $res["ID"]));
                     $ret[] = array('group' => $group['group'], 'msg' => 'Updated');
                 } else {
                     $desc = "";
                     $db->queryInsert(sprintf("INSERT INTO groups (name, description, active) VALUES (%s, %s, %d)", $db->escapeString($group['group']), $db->escapeString($desc), $active));
                     $ret[] = array('group' => $group['group'], 'msg' => 'Created');
                 }
             }
         }
     }
     return $ret;
 }
 public function fetchTestBinaries($groupname, $numarticles, $clearexistingbins)
 {
     $db = new DB();
     $nntp = new Nntp();
     $binaries = new Binaries();
     $groups = new Groups();
     $ret = array();
     if ($clearexistingbins == true) {
         $db->exec('truncate releaseregextesting');
     }
     $nntp->doConnect();
     $groupsToFetch = array();
     if (preg_match('/^[a-z]{2,3}(\\.[a-z0-9\\-]+)+$/', $groupname)) {
         $groupsToFetch[] = array('name' => $groupname);
     } elseif ($groupname === 0) {
         $groupsToFetch = $groups->getAll();
     } else {
         $newsgroups = $nntp->getGroups();
         foreach ($newsgroups as $ngroup) {
             if (preg_match('/' . $groupname . '/', $ngroup['group'])) {
                 $groupsToFetch[] = array('name' => $ngroup['group']);
             }
         }
     }
     foreach ($groupsToFetch as $groupArr) {
         $group = $groupArr['name'];
         $data = $nntp->selectGroup($group);
         if ($nntp->isError($data)) {
             $ret[] = "Could not select group (doesnt exist on USP): {$group}";
             continue;
         } else {
             $rangeStart = $data['last'] - $numarticles;
             $rangeEnd = $groupEnd = $data['last'];
             $rangeTotal = $rangeEnd - $rangeStart;
             $done = false;
             while ($done === false) {
                 if ($rangeTotal > $binaries->messagebuffer) {
                     if ($rangeStart + $binaries->messagebuffer > $groupEnd) {
                         $rangeEnd = $groupEnd;
                     } else {
                         $rangeEnd = $rangeStart + $binaries->messagebuffer;
                     }
                 }
                 if ($binaries->compressedHeaders) {
                     $msgs = $nntp->getXOverview($rangeStart . "-" . $rangeEnd, true, false);
                 } else {
                     $msgs = $nntp->getOverview($rangeStart . "-" . $rangeEnd, true, false);
                 }
                 if ($nntp->isError($msgs)) {
                     $ret[] = "Error {$msgs->code}: {$msgs->message} on " . $group;
                     continue 2;
                 }
                 $headers = array();
                 if (is_array($msgs)) {
                     //loop headers, figure out parts
                     foreach ($msgs as $msg) {
                         if (!isset($msg['Number'])) {
                             continue;
                         }
                         $msgPart = $msgTotalParts = 0;
                         $pattern = '|\\((\\d+)[\\/](\\d+)\\)|i';
                         preg_match_all($pattern, $msg['Subject'], $matches, PREG_PATTERN_ORDER);
                         $matchcnt = sizeof($matches[0]);
                         for ($i = 0; $i < $matchcnt; $i++) {
                             //not (int)'d here because of the preg_replace later on
                             $msgPart = $matches[1][$i];
                             $msgTotalParts = $matches[2][$i];
                         }
                         if (!isset($msg['Subject']) || $matchcnt == 0) {
                             // not a binary post most likely.. continue
                             continue;
                         }
                         if ((int) $msgPart > 0 && (int) $msgTotalParts > 0) {
                             $subject = utf8_encode(trim(preg_replace('|\\(' . $msgPart . '[\\/]' . $msgTotalParts . '\\)|i', '', $msg['Subject'])));
                             if (!isset($headers[$subject])) {
                                 $headers[$subject]['Subject'] = $subject;
                                 $headers[$subject]['From'] = $msg['From'];
                                 $headers[$subject]['Date'] = strtotime($msg['Date']);
                                 $headers[$subject]['Message-ID'] = $msg['Message-ID'];
                                 $headers[$subject]['Size'] = $msg['Bytes'];
                             } else {
                                 $headers[$subject]['Size'] += $msg['Bytes'];
                             }
                         }
                     }
                     unset($msgs);
                     if (isset($headers) && count($headers)) {
                         $groupRegexes = $this->getForGroup($group);
                         $binSetData = array();
                         foreach ($headers as $subject => $data) {
                             $binData = array('name' => $subject, 'fromname' => $data['From'], 'date' => $data['Date'], 'binaryhash' => md5($subject . $data['From'] . $group), 'groupname' => $group, 'regexID' => "null", 'categoryID' => "null", 'reqID' => "null", 'blacklistID' => 0, 'size' => $data['Size'], 'relname' => "null", 'relpart' => "null", 'reltotalpart' => "null");
                             //Filter binaries based on black/white list
                             if ($binaries->isBlackListed($data, $group)) {
                                 //binary is blacklisted
                                 $binData['blacklistID'] = 1;
                             }
                             //Apply Regexes
                             $regexMatches = array();
                             foreach ($groupRegexes as $groupRegex) {
                                 $regexCheck = $this->performMatch($groupRegex, $subject, $data['From']);
                                 if ($regexCheck !== false) {
                                     $regexMatches = $regexCheck;
                                     $binData['regexID'] = $regexCheck['regexID'];
                                     $binData['categoryID'] = $regexCheck['regcatid'];
                                     $binData['reqID'] = empty($regexCheck['reqID']) ? "null" : $regexCheck['reqID'];
                                     $binData['relname'] = $regexCheck['name'];
                                     break;
                                 }
                             }
                             $binSetData[] = $binData;
                         }
                         //insert 500 bins at a time
                         $binChunks = array_chunk($binSetData, 500);
                         foreach ($binChunks as $binChunk) {
                             foreach ($binChunk as $chunk) {
                                 $binParams[] = sprintf("(%s, %s, FROM_UNIXTIME(%s), %s, %s, %s, %s, %s, %d, %d, now())", $db->escapeString($chunk['name']), $db->escapeString($chunk['fromname']), $db->escapeString($chunk['date']), $db->escapeString($chunk['binaryhash']), $db->escapeString($chunk['groupname']), $chunk['regexID'], $chunk['categoryID'], $chunk['reqID'], $chunk['blacklistID'], $chunk['size']);
                             }
                             $binSql = "INSERT IGNORE INTO releaseregextesting (name, fromname, date, binaryhash, groupname, regexID, categoryID, reqID, blacklistID, size, dateadded) VALUES " . implode(', ', $binParams);
                             //echo $binSql;
                             $db->exec($binSql);
                         }
                         $ret[] = "Fetched " . number_format($numarticles) . " articles from " . $group;
                     } else {
                         $ret[] = "No headers found on " . $group;
                         continue;
                     }
                 } else {
                     $ret[] = "Can't get parts from server (msgs not array) on " . $group;
                     continue;
                 }
                 if ($rangeEnd == $groupEnd) {
                     $done = true;
                 }
                 $rangeStart = $rangeEnd + 1;
             }
         }
     }
     $nntp->doQuit();
     return $ret;
 }