/** * Backfill single group. * * @param array $groupArr * @param int $left * @param int|string $articles * * @return void */ public function backfillGroup($groupArr, $left, $articles = '') { // Start time for this group. $startGroup = microtime(true); $groupName = str_replace('alt.binaries', 'a.b', $groupArr['name']); // If our local oldest article 0, it means we never ran update_binaries on the group. if ($groupArr['first_record'] <= 0) { $dMessage = "You need to run update_binaries on " . $groupName . ". Otherwise the group is dead, you must disable it."; if ($this->_debug) { $this->_debugging->log(get_class(), __FUNCTION__, $dMessage, Logger::LOG_ERROR); } if ($this->_echoCLI) { $this->pdo->log->doEcho($this->pdo->log->error($dMessage)); } return; } // Select group, here, only once $data = $this->_nntp->selectGroup($groupArr['name']); if ($this->_nntp->isError($data)) { $data = $this->_nntp->dataError($this->_nntp, $groupArr['name']); if ($this->_nntp->isError($data)) { return; } } if ($this->_echoCLI) { $this->pdo->log->doEcho($this->pdo->log->primary('Processing ' . $groupName), true); } // Check if this is days or post backfill. $postCheck = $articles === '' ? false : true; // Get target post based on date or user specified number. $targetpost = (string) ($postCheck ? round($groupArr['first_record'] - $articles) : $this->_binaries->daytopost($groupArr['backfill_target'], $data)); // Check if target post is smaller than server's oldest, set it to oldest if so. if ($targetpost < $data['first']) { $targetpost = $data['first']; } // Check if our target post is newer than our oldest post or if our local oldest article is older than the servers oldest. if ($targetpost >= $groupArr['first_record'] || $groupArr['first_record'] <= $data['first']) { $dMessage = "We have hit the maximum we can backfill for " . $groupName . ($this->_disableBackfillGroup ? ", disabling backfill on it." : ", skipping it, consider disabling backfill on it."); if ($this->_debug) { $this->_debugging->log(get_class(), __FUNCTION__, $dMessage, Logger::LOG_NOTICE); } if ($this->_disableBackfillGroup) { $this->pdo->queryExec(sprintf(' UPDATE groups SET backfill = 0 WHERE id = %d', $groupArr['id'])); } if ($this->_echoCLI) { $this->pdo->log->doEcho($this->pdo->log->notice($dMessage), true); } return; } if ($this->_echoCLI) { $this->pdo->log->doEcho($this->pdo->log->primary('Group ' . $groupName . "'s oldest article is " . number_format($data['first']) . ', newest is ' . number_format($data['last']) . ".\nOur target article is " . number_format($targetpost) . '. Our oldest article is article ' . number_format($groupArr['first_record']) . '.')); } // Set first and last, moving the window by max messages. $last = (string) ($groupArr['first_record'] - 1); // Set the initial "chunk". $first = (string) ($last - $this->_binaries->messageBuffer + 1); // Just in case this is the last chunk we needed. if ($targetpost > $first) { $first = $targetpost; } $done = false; while ($done === false) { if ($this->_echoCLI) { $this->pdo->log->doEcho($this->pdo->log->set256('Yellow') . "\nGetting " . number_format($last - $first + 1) . " articles from " . $groupName . ", " . $left . " group(s) left. (" . number_format($first - $targetpost) . " articles in queue)." . $this->pdo->log->rsetColor(), true); } flush(); $lastMsg = $this->_binaries->scan($groupArr, $first, $last, $this->_safePartRepair); // Get the oldest date. if (isset($lastMsg['firstArticleDate'])) { // Try to get it from the oldest pulled article. $newdate = strtotime($lastMsg['firstArticleDate']); } else { // If above failed, try to get it with postdate method. $newdate = $this->_binaries->postdate($first, $data); } $this->pdo->queryExec(sprintf(' UPDATE groups SET first_record_postdate = %s, first_record = %s, last_updated = NOW() WHERE id = %d', $this->pdo->from_unixtime($newdate), $this->pdo->escapeString($first), $groupArr['id'])); if ($first == $targetpost) { $done = true; } else { // Keep going: set new last, new first, check for last chunk. $last = (string) ($first - 1); $first = (string) ($last - $this->_binaries->messageBuffer + 1); if ($targetpost > $first) { $first = $targetpost; } } } if ($this->_echoCLI) { $this->pdo->log->doEcho($this->pdo->log->primary(PHP_EOL . 'Group ' . $groupName . ' processed in ' . number_format(microtime(true) - $startGroup, 2) . " seconds."), true); } }
* $options[4] => (int) First article number in range. * $options[5] => (int) Last article number in range. * $options[6] => (int) Number of threads. */ case 'get_range': $pdo = new Settings(); $nntp = nntp($pdo); $groups = new Groups(['Settings' => $pdo]); $groupMySQL = $groups->getByName($options[3]); if ($nntp->isError($nntp->selectGroup($groupMySQL['name']))) { if ($nntp->isError($nntp->dataError($nntp, $groupMySQL['name']))) { return; } } $binaries = new Binaries(['NNTP' => $nntp, 'Settings' => $pdo, 'Groups' => $groups]); $return = $binaries->scan($groupMySQL, $options[4], $options[5], $pdo->getSetting('safepartrepair') == 1 ? 'update' : 'backfill'); if (empty($return)) { exit; } $columns = []; switch ($options[2]) { case 'binaries': if ($return['lastArticleNumber'] <= $groupMySQL['last_record']) { exit; } $columns[1] = sprintf('last_record_postdate = %s', $pdo->from_unixtime(is_numeric($return['lastArticleDate']) ? $return['lastArticleDate'] : strtotime($return['lastArticleDate']))); $columns[2] = sprintf('last_record = %s', $return['lastArticleNumber']); $query = sprintf('UPDATE groups SET %s, %s, last_updated = NOW() WHERE id = %d AND last_record < %s', $columns[1], $columns[2], $groupMySQL['id'], $return['lastArticleNumber']); break; case 'backfill': if ($return['firstArticleNumber'] >= $groupMySQL['first_record']) {