/**
* Process bot messages, insert/update PREs.
*
* @access protected
*/
protected function processChannelMessages()
{
if (preg_match('/^(NEW|UPD|NUK): \\[DT: (?P<time>.+?)\\]\\s?\\[TT: (?P<title>.+?)\\]\\s?\\[SC: (?P<source>.+?)\\]\\s?\\[CT: (?P<category>.+?)\\]\\s?\\[RQ: (?P<req>.+?)\\]' . '\\s?\\[SZ: (?P<size>.+?)\\]\\s?\\[FL: (?P<files>.+?)\\]\\s?(\\[FN: (?P<filename>.+?)\\]\\s?)?(\\[(?P<nuked>(UN|MOD|RE|OLD)?NUKED?): (?P<reason>.+?)\\])?$/i', $this->_channelData['message'], $matches)) {
if (isset($this->_ignoredChannels[$matches['source']]) && $this->_ignoredChannels[$matches['source']] === true) {
return;
}
if ($this->_categoryIgnoreRegex !== false && preg_match((string) $this->_categoryIgnoreRegex, $matches['category'])) {
return;
}
if ($this->_titleIgnoreRegex !== false && preg_match((string) $this->_titleIgnoreRegex, $matches['title'])) {
return;
}
$this->_curPre['predate'] = $this->_pdo->from_unixtime(strtotime($matches['time'] . ' UTC'));
$this->_curPre['title'] = $matches['title'];
$this->_curPre['source'] = $matches['source'];
if ($matches['category'] !== 'N/A') {
$this->_curPre['category'] = $matches['category'];
}
if ($matches['req'] !== 'N/A' && preg_match('/^(?P<req>\\d+):(?P<group>.+)$/i', $matches['req'], $matches2)) {
$this->_curPre['reqid'] = $matches2['req'];
$this->_curPre['group_id'] = $this->_getGroupID($matches2['group']);
}
if ($matches['size'] !== 'N/A') {
$this->_curPre['size'] = $matches['size'];
}
if ($matches['files'] !== 'N/A') {
$this->_curPre['files'] = substr($matches['files'], 0, 50);
}
if (isset($matches['filename']) && $matches['filename'] !== 'N/A') {
$this->_curPre['filename'] = $matches['filename'];
}
if (isset($matches['nuked'])) {
switch ($matches['nuked']) {
case 'NUKED':
$this->_curPre['nuked'] = PreDb::PRE_NUKED;
break;
case 'UNNUKED':
$this->_curPre['nuked'] = PreDb::PRE_UNNUKED;
break;
case 'MODNUKED':
$this->_curPre['nuked'] = PreDb::PRE_MODNUKE;
break;
case 'RENUKED':
$this->_curPre['nuked'] = PreDb::PRE_RENUKED;
break;
case 'OLDNUKE':
$this->_curPre['nuked'] = PreDb::PRE_OLDNUKE;
break;
}
$this->_curPre['reason'] = isset($matches['reason']) ? substr($matches['reason'], 0, 255) : '';
}
$this->_checkForDupe();
}
}
protected function _verifyPreData(&$matches)
{
// If the title is too short, don't bother.
if (strlen($matches['title']) < 15) {
return;
}
$matches['title'] = str_replace(array("\r", "\n"), '', $matches['title']);
$duplicateCheck = $this->_db->queryOneRow(sprintf('SELECT * FROM predb WHERE title = %s', $this->_db->escapeString($matches['title'])));
if (!is_numeric($matches['date']) || $matches['date'] < time() - 31536000) {
return;
}
if ($duplicateCheck === false) {
$this->_db->queryExec(sprintf('
INSERT INTO predb (title, size, category, predate, source, requestid, groupid, files, filename, nuked, nukereason, shared)
VALUES (%s, %s, %s, %s, %s, %d, %d, %s, %s, %d, %s, -1)', $this->_db->escapeString($matches['title']), isset($matches['size']) && !empty($matches['size']) ? $this->_db->escapeString($matches['size']) : 'NULL', isset($matches['category']) && !empty($matches['category']) ? $this->_db->escapeString($matches['category']) : 'NULL', $this->_db->from_unixtime($matches['date']), $this->_db->escapeString($matches['source']), isset($matches['requestid']) && is_numeric($matches['requestid']) ? $matches['requestid'] : 0, isset($matches['groupid']) && is_numeric($matches['groupid']) ? $matches['groupid'] : 0, isset($matches['files']) && !empty($matches['files']) ? $this->_db->escapeString($matches['files']) : 'NULL', isset($matches['filename']) ? $this->_db->escapeString($matches['filename']) : $this->_db->escapeString(''), isset($matches['nuked']) && is_numeric($matches['nuked']) ? $matches['nuked'] : 0, isset($matches['reason']) && !empty($matches['nukereason']) ? $this->_db->escapeString($matches['nukereason']) : 'NULL'));
$this->_done++;
} else {
$query = 'UPDATE predb SET ';
$query .= $this->_updateString('size', $duplicateCheck['size'], $matches['size']);
$query .= $this->_updateString('files', $duplicateCheck['files'], $matches['files']);
$query .= $this->_updateString('nukereason', $duplicateCheck['nukereason'], $matches['reason']);
$query .= $this->_updateString('requestid', $duplicateCheck['requestid'], $matches['requestid'], false);
$query .= $this->_updateString('groupid', $duplicateCheck['groupid'], $matches['groupid'], false);
$query .= $this->_updateString('nuked', $duplicateCheck['nuked'], $matches['nuked'], false);
$query .= $this->_updateString('filename', $duplicateCheck['filename'], $matches['filename']);
$query .= $this->_updateString('category', $duplicateCheck['category'], $matches['category']);
if ($query === 'UPDATE predb SET ') {
return;
}
$this->_done++;
$query .= 'predate = ' . $this->_db->from_unixtime($matches['date']) . ', ';
$query .= 'source = ' . $this->_db->escapeString($matches['source']) . ', ';
$query .= 'title = ' . $this->_db->escapeString($matches['title']);
$query .= ', shared = -1';
$query .= ' WHERE title = ' . $this->_db->escapeString($matches['title']);
$this->_db->queryExec($query);
}
}
/**
* Get pre date from wD xH yM zS ago string.
*
* @param $agoString
*
* @access protected
*/
protected function getTimeFromAgo($agoString)
{
$predate = 0;
// Get pre date from this format : 10m 54s
if (preg_match('/((?P<day>\\d+)d)?\\s*((?P<hour>\\d+)h)?\\s*((?P<min>\\d+)m)?\\s*((?P<sec>\\d+)s)?/i', $agoString, $matches)) {
if (!empty($matches['day'])) {
$predate += (int) $matches['day'] * 86400;
}
if (!empty($matches['hour'])) {
$predate += (int) $matches['hour'] * 3600;
}
if (!empty($matches['min'])) {
$predate += (int) $matches['min'] * 60;
}
if (!empty($matches['sec'])) {
$predate += (int) $matches['sec'];
}
if ($predate !== 0) {
$predate = time() - $predate;
}
}
$this->CurPre['predate'] = $predate === 0 ? '' : $this->db->from_unixtime($predate);
}
/**
* Backfill single group.
*
* @param array $groupArr
* @param int $left
* @param int|string $articles
*
* @return void
*/
public function backfillGroup($groupArr, $left, $articles = '')
{
// Start time for this group.
$startGroup = microtime(true);
$groupName = str_replace('alt.binaries', 'a.b', $groupArr['name']);
// If our local oldest article 0, it means we never ran update_binaries on the group.
if ($groupArr['first_record'] <= 0) {
$dMessage = "You need to run update_binaries on " . $groupName . ". Otherwise the group is dead, you must disable it.";
if ($this->_debug) {
$this->_debugging->log('Backfill', "backfillGroup", $dMessage, Logger::LOG_ERROR);
}
if ($this->_echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->error($dMessage));
}
return;
}
// Select group, here, only once
$data = $this->_nntp->selectGroup($groupArr['name']);
if ($this->_nntp->isError($data)) {
$data = $this->_nntp->dataError($this->_nntp, $groupArr['name']);
if ($this->_nntp->isError($data)) {
return;
}
}
if ($this->_echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->primary('Processing ' . $groupName), true);
}
// Check if this is days or post backfill.
$postCheck = $articles === '' ? false : true;
// Get target post based on date or user specified number.
$targetpost = (string) ($postCheck ? round($groupArr['first_record'] - $articles) : $this->_binaries->daytopost($groupArr['backfill_target'], $data));
// Check if target post is smaller than server's oldest, set it to oldest if so.
if ($targetpost < $data['first']) {
$targetpost = $data['first'];
}
// Check if our target post is newer than our oldest post or if our local oldest article is older than the servers oldest.
if ($targetpost >= $groupArr['first_record'] || $groupArr['first_record'] <= $data['first']) {
$dMessage = "We have hit the maximum we can backfill for " . $groupName . ", skipping it, consider disabling backfill on it.";
if ($this->_debug) {
$this->_debugging->log('Backfill', "backfillGroup", $dMessage, Logger::LOG_NOTICE);
}
if ($this->_echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->notice($dMessage), true);
}
return;
}
if ($this->_echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->primary('Group ' . $groupName . "'s oldest article is " . number_format($data['first']) . ', newest is ' . number_format($data['last']) . ".\nOur target article is " . number_format($targetpost) . '. Our oldest article is article ' . number_format($groupArr['first_record']) . '.'));
}
// Set first and last, moving the window by max messages.
$last = (string) ($groupArr['first_record'] - 1);
// Set the initial "chunk".
$first = (string) ($last - $this->_binaries->messageBuffer + 1);
// Just in case this is the last chunk we needed.
if ($targetpost > $first) {
$first = $targetpost;
}
$done = false;
while ($done === false) {
if ($this->_echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->set256('Yellow') . "\nGetting " . number_format($last - $first + 1) . " articles from " . $groupName . ", " . $left . " group(s) left. (" . number_format($first - $targetpost) . " articles in queue)." . $this->pdo->log->rsetColor(), true);
}
flush();
$lastMsg = $this->_binaries->scan($groupArr, $first, $last, $this->_safePartRepair);
// Get the oldest date.
if (isset($lastMsg['firstArticleDate'])) {
// Try to get it from the oldest pulled article.
$newdate = strtotime($lastMsg['firstArticleDate']);
} else {
// If above failed, try to get it with postdate method.
$newdate = $this->_binaries->postdate($first, $data);
}
$this->pdo->queryExec(sprintf('
UPDATE groups
SET first_record_postdate = %s, first_record = %s, last_updated = NOW()
WHERE id = %d', $this->pdo->from_unixtime($newdate), $this->pdo->escapeString($first), $groupArr['id']));
if ($first == $targetpost) {
$done = true;
} else {
// Keep going: set new last, new first, check for last chunk.
$last = (string) ($first - 1);
$first = (string) ($last - $this->_binaries->messageBuffer + 1);
if ($targetpost > $first) {
$first = $targetpost;
}
}
}
if ($this->_echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->primary(PHP_EOL . 'Group ' . $groupName . ' processed in ' . number_format(microtime(true) - $startGroup, 2) . " seconds."), true);
}
}
exit('ERROR! No groups were found with backfill enabled!' . PHP_EOL);
} else {
exit('ERROR! Group (' . $argv[1] . ') not found!' . PHP_EOL);
}
}
$nntp = new NNTP(['Settings' => $pdo]);
$nntp->doConnect() or exit('Could not connect to Usenet!' . PHP_EOL);
$binaries = new Binaries(['NNTP' => $nntp, 'Settings' => $pdo]);
foreach ($groups as $group) {
$groupNNTP = $nntp->selectGroup($group['name']);
if ($nntp->isError($groupNNTP)) {
echo 'ERROR! Could not fetch information from NNTP for group (' . $group['name'] . ')' . PHP_EOL;
continue;
}
$postDate = $pdo->queryOneRow(sprintf('SELECT UNIX_TIMESTAMP(postdate) AS postdate FROM releases WHERE group_id = %d ORDER BY postdate ASC LIMIT 1', $group['id']));
if ($postDate === false) {
echo 'ERROR! Could not find any existing releases for group (' . $group['name'] . ')' . PHP_EOL;
continue;
}
$articleNumber = (int) $binaries->daytopost(round((time() - $postDate['postdate']) / 86400), $groupNNTP);
if ($group['last_record'] != 0 && $articleNumber >= $group['last_record']) {
echo 'ERROR! Could not determine the article number for this date: (' . $postDate['postdate'] . ') on group (' . $group['name'] . ')' . PHP_EOL;
continue;
}
$articleDate = $binaries->postdate($articleNumber, $groupNNTP);
$pdo->queryExec(sprintf('
UPDATE groups
SET first_record = %d, first_record_postdate = %s
WHERE id = %d', $articleNumber, $pdo->from_unixtime($articleDate), $group['id']));
echo 'SUCCESS! Updated group (' . $group['name'] . ')\'s first article number to (' . $articleNumber . ') dated (' . date('r', $articleDate) . ').' . PHP_EOL . 'The previous first article number was: (' . $group['first_record'] . ')' . (empty($group['first_record_postdate']) ? '.' : ' dated (' . date('r', strtotime($group['first_record_postdate'])) . ').') . PHP_EOL;
}