コード例 #1
0
 public function fetchTestBinaries($groupname, $numarticles, $clearexistingbins)
 {
     $nntp = new NNTP();
     $binaries = new Binaries();
     $groups = new Groups();
     $ret = [];
     if ($clearexistingbins == true) {
         $this->pdo->queryExec('truncate releaseregextesting');
     }
     $nntp->doConnect();
     $groupsToFetch = [];
     if (preg_match('/^[a-z]{2,3}(\\.[a-z0-9\\-]+)+$/', $groupname)) {
         $groupsToFetch[] = array('name' => $groupname);
     } elseif ($groupname === 0) {
         $groupsToFetch = $groups->getAll();
     } else {
         $newsgroups = $nntp->getGroups();
         foreach ($newsgroups as $ngroup) {
             if (preg_match('/' . $groupname . '/', $ngroup['group'])) {
                 $groupsToFetch[] = array('name' => $ngroup['group']);
             }
         }
     }
     foreach ($groupsToFetch as $groupArr) {
         $group = $groupArr['name'];
         $data = $nntp->selectGroup($group);
         if (NNTP::isError($data)) {
             $ret[] = "Could not select group (doesnt exist on USP): {$group}";
             continue;
         } else {
             $rangeStart = $data['last'] - $numarticles;
             $rangeEnd = $groupEnd = $data['last'];
             $rangeTotal = $rangeEnd - $rangeStart;
             $done = false;
             while ($done === false) {
                 if ($rangeTotal > $binaries->messageBuffer) {
                     if ($rangeStart + $binaries->messageBuffer > $groupEnd) {
                         $rangeEnd = $groupEnd;
                     } else {
                         $rangeEnd = $rangeStart + $binaries->messageBuffer;
                     }
                 }
                 $msgs = $nntp->getXOver($rangeStart . "-" . $rangeEnd, true, false);
                 if (NNTP::isError($msgs)) {
                     $ret[] = "Error {$msgs->code}: {$msgs->message} on " . $group;
                     continue 2;
                 }
                 $headers = [];
                 if (is_array($msgs)) {
                     //loop headers, figure out parts
                     foreach ($msgs as $msg) {
                         if (!isset($msg['Number'])) {
                             continue;
                         }
                         $msgPart = $msgTotalParts = 0;
                         $pattern = '|\\((\\d+)[\\/](\\d+)\\)|i';
                         preg_match_all($pattern, $msg['Subject'], $matches, PREG_PATTERN_ORDER);
                         $matchcnt = sizeof($matches[0]);
                         for ($i = 0; $i < $matchcnt; $i++) {
                             //not (int)'d here because of the preg_replace later on
                             $msgPart = $matches[1][$i];
                             $msgTotalParts = $matches[2][$i];
                         }
                         if (!isset($msg['Subject']) || $matchcnt == 0) {
                             // not a binary post most likely.. continue
                             continue;
                         }
                         if ((int) $msgPart > 0 && (int) $msgTotalParts > 0) {
                             $subject = utf8_encode(trim(preg_replace('|\\(' . $msgPart . '[\\/]' . $msgTotalParts . '\\)|i', '', $msg['Subject'])));
                             if (!isset($headers[$subject])) {
                                 $headers[$subject]['Subject'] = $subject;
                                 $headers[$subject]['From'] = $msg['From'];
                                 $headers[$subject]['Date'] = strtotime($msg['Date']);
                                 $headers[$subject]['Message-ID'] = $msg['Message-ID'];
                                 $headers[$subject]['Size'] = $msg['Bytes'];
                             } else {
                                 $headers[$subject]['Size'] += $msg['Bytes'];
                             }
                         }
                     }
                     unset($msgs);
                     if (isset($headers) && count($headers)) {
                         $groupRegexes = $this->getForGroup($group);
                         $binSetData = [];
                         foreach ($headers as $subject => $data) {
                             $binData = array('name' => $subject, 'fromname' => $data['From'], 'date' => $data['Date'], 'binaryhash' => md5($subject . $data['From'] . $group), 'groupname' => $group, 'regexid' => "null", 'categoryid' => "null", 'reqid' => "null", 'blacklistid' => 0, 'size' => $data['Size'], 'relname' => "null", 'relpart' => "null", 'reltotalpart' => "null");
                             //Filter binaries based on black/white list
                             if ($binaries->isBlackListed($data, $group)) {
                                 //binary is blacklisted
                                 $binData['blacklistid'] = 1;
                             }
                             //Apply Regexes
                             $regexMatches = [];
                             foreach ($groupRegexes as $groupRegex) {
                                 $regexCheck = $this->performMatch($groupRegex, $subject, $data['From']);
                                 if ($regexCheck !== false) {
                                     $regexMatches = $regexCheck;
                                     $binData['regexid'] = $regexCheck['regexid'];
                                     $binData['categoryid'] = $regexCheck['regcatid'];
                                     $binData['reqid'] = empty($regexCheck['reqid']) ? "null" : $regexCheck['reqid'];
                                     $binData['relname'] = $regexCheck['name'];
                                     break;
                                 }
                             }
                             $binSetData[] = $binData;
                         }
                         //insert 500 bins at a time
                         $binChunks = array_chunk($binSetData, 500);
                         foreach ($binChunks as $binChunk) {
                             foreach ($binChunk as $chunk) {
                                 $binParams[] = sprintf("(%s, %s, FROM_UNIXTIME(%s), %s, %s, %s, %s, %s, %d, %d, now())", $this->pdo->escapeString($chunk['name']), $this->pdo->escapeString($chunk['fromname']), $this->pdo->escapeString($chunk['date']), $this->pdo->escapeString($chunk['binaryhash']), $this->pdo->escapeString($chunk['groupname']), $chunk['regexid'], $chunk['categoryid'], $chunk['reqid'], $chunk['blacklistid'], $chunk['size']);
                             }
                             $binSql = "INSERT IGNORE INTO releaseregextesting (name, fromname, date, binaryhash, groupname, regexid, categoryid, reqid, blacklistid, size, dateadded) VALUES " . implode(', ', $binParams);
                             //echo $binSql;
                             $this->pdo->queryExec($binSql);
                         }
                         $ret[] = "Fetched " . number_format($numarticles) . " articles from " . $group;
                     } else {
                         $ret[] = "No headers found on " . $group;
                         continue;
                     }
                 } else {
                     $ret[] = "Can't get parts from server (msgs not array) on " . $group;
                     continue;
                 }
                 if ($rangeEnd == $groupEnd) {
                     $done = true;
                 }
                 $rangeStart = $rangeEnd + 1;
             }
         }
     }
     $nntp->doQuit();
     return $ret;
 }