/**
* Backfill single group.
*
* @param array $groupArr
* @param int $left
* @param int|string $articles
*
* @return void
*/
public function backfillGroup($groupArr, $left, $articles = '')
{
// Start time for this group.
$startGroup = microtime(true);
$groupName = str_replace('alt.binaries', 'a.b', $groupArr['name']);
// If our local oldest article 0, it means we never ran update_binaries on the group.
if ($groupArr['first_record'] <= 0) {
$dMessage = "You need to run update_binaries on " . $groupName . ". Otherwise the group is dead, you must disable it.";
if ($this->_debug) {
$this->_debugging->log('Backfill', "backfillGroup", $dMessage, Logger::LOG_ERROR);
}
if ($this->_echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->error($dMessage));
}
return;
}
// Select group, here, only once
$data = $this->_nntp->selectGroup($groupArr['name']);
if ($this->_nntp->isError($data)) {
$data = $this->_nntp->dataError($this->_nntp, $groupArr['name']);
if ($this->_nntp->isError($data)) {
return;
}
}
if ($this->_echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->primary('Processing ' . $groupName), true);
}
// Check if this is days or post backfill.
$postCheck = $articles === '' ? false : true;
// Get target post based on date or user specified number.
$targetpost = (string) ($postCheck ? round($groupArr['first_record'] - $articles) : $this->_binaries->daytopost($groupArr['backfill_target'], $data));
// Check if target post is smaller than server's oldest, set it to oldest if so.
if ($targetpost < $data['first']) {
$targetpost = $data['first'];
}
// Check if our target post is newer than our oldest post or if our local oldest article is older than the servers oldest.
if ($targetpost >= $groupArr['first_record'] || $groupArr['first_record'] <= $data['first']) {
$dMessage = "We have hit the maximum we can backfill for " . $groupName . ", skipping it, consider disabling backfill on it.";
if ($this->_debug) {
$this->_debugging->log('Backfill', "backfillGroup", $dMessage, Logger::LOG_NOTICE);
}
if ($this->_echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->notice($dMessage), true);
}
return;
}
if ($this->_echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->primary('Group ' . $groupName . "'s oldest article is " . number_format($data['first']) . ', newest is ' . number_format($data['last']) . ".\nOur target article is " . number_format($targetpost) . '. Our oldest article is article ' . number_format($groupArr['first_record']) . '.'));
}
// Set first and last, moving the window by max messages.
$last = (string) ($groupArr['first_record'] - 1);
// Set the initial "chunk".
$first = (string) ($last - $this->_binaries->messageBuffer + 1);
// Just in case this is the last chunk we needed.
if ($targetpost > $first) {
$first = $targetpost;
}
$done = false;
while ($done === false) {
if ($this->_echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->set256('Yellow') . "\nGetting " . number_format($last - $first + 1) . " articles from " . $groupName . ", " . $left . " group(s) left. (" . number_format($first - $targetpost) . " articles in queue)." . $this->pdo->log->rsetColor(), true);
}
flush();
$lastMsg = $this->_binaries->scan($groupArr, $first, $last, $this->_safePartRepair);
// Get the oldest date.
if (isset($lastMsg['firstArticleDate'])) {
// Try to get it from the oldest pulled article.
$newdate = strtotime($lastMsg['firstArticleDate']);
} else {
// If above failed, try to get it with postdate method.
$newdate = $this->_binaries->postdate($first, $data);
}
$this->pdo->queryExec(sprintf('
UPDATE groups
SET first_record_postdate = %s, first_record = %s, last_updated = NOW()
WHERE id = %d', $this->pdo->from_unixtime($newdate), $this->pdo->escapeString($first), $groupArr['id']));
if ($first == $targetpost) {
$done = true;
} else {
// Keep going: set new last, new first, check for last chunk.
$last = (string) ($first - 1);
$first = (string) ($last - $this->_binaries->messageBuffer + 1);
if ($targetpost > $first) {
$first = $targetpost;
}
}
}
if ($this->_echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->primary(PHP_EOL . 'Group ' . $groupName . ' processed in ' . number_format(microtime(true) - $startGroup, 2) . " seconds."), true);
}
}
* $options[4] => (int) First article number in range.
* $options[5] => (int) Last article number in range.
* $options[6] => (int) Number of threads.
*/
case 'get_range':
$pdo = new Settings();
$nntp = nntp($pdo);
$groups = new \Groups();
$groupMySQL = $groups->getByName($options[3]);
if ($nntp->isError($nntp->selectGroup($groupMySQL['name']))) {
if ($nntp->isError($nntp->dataError($nntp, $groupMySQL['name']))) {
return;
}
}
$binaries = new \Binaries(['NNTP' => $nntp, 'Settings' => $pdo, 'Groups' => $groups]);
$return = $binaries->scan($groupMySQL, $options[4], $options[5], $pdo->getSetting('safepartrepair') == 1 ? 'update' : 'backfill');
if (empty($return)) {
exit;
}
$columns = [];
switch ($options[2]) {
case 'binaries':
if ($return['lastArticleNumber'] <= $groupMySQL['last_record']) {
exit;
}
$columns[1] = sprintf('last_record_postdate = %s', $pdo->from_unixtime(is_numeric($return['lastArticleDate']) ? $return['lastArticleDate'] : strtotime($return['lastArticleDate'])));
$columns[2] = sprintf('last_record = %s', $return['lastArticleNumber']);
$query = sprintf('UPDATE groups SET %s, %s, last_updated = NOW() WHERE id = %d AND last_record < %s', $columns[1], $columns[2], $groupMySQL['id'], $return['lastArticleNumber']);
break;
case 'backfill':
if ($return['firstArticleNumber'] >= $groupMySQL['first_record']) {
/**
* Update a group back to a specified date.
*/
function backfillGroup($groupArr, $backfillDate = null, $regexOnly = false)
{
$db = new DB();
$binaries = new Binaries();
$n = $this->n;
if ($regexOnly === true) {
echo "Only inserting binaries which match regex{$n}";
$binaries->onlyProcessRegexBinaries = true;
}
$this->startGroup = microtime(true);
$nntp = new Nntp();
$nntpc = new Nntp();
//Make sure we actually have a connection going before doing anything.
if ($nntp->doConnect(5, false, true)) {
echo 'Processing ' . $groupArr['name'] . $n;
$data = $nntp->selectGroup($groupArr['name']);
if ($nntp->isError($data)) {
echo "Could not select group (bad name?): {$groupArr['name']}{$n}";
return;
}
if ($backfillDate) {
$targetpost = $this->daytopost($nntp, $groupArr['name'], $this->dateToDays($backfillDate), TRUE);
} else {
$targetpost = $this->daytopost($nntp, $groupArr['name'], $groupArr['backfill_target'], TRUE);
}
//get targetpost based on days target
if ($groupArr['first_record'] == 0 || $groupArr['backfill_target'] == 0 && !$backfillDate) {
echo "Group " . $groupArr['name'] . " has invalid numbers. Have you run update on it? Have you set the backfill days amount?{$n}";
return;
}
echo "Group " . $data["group"] . ": server has " . $data['first'] . " - " . $data['last'] . ", or ~";
echo (int) (($this->postdate($nntp, $data['last'], FALSE) - $this->postdate($nntp, $data['first'], FALSE)) / 86400);
echo " days." . $n . "Local first = " . $groupArr['first_record'] . " (";
echo (int) ((date('U') - $this->postdate($nntp, $groupArr['first_record'], FALSE)) / 86400);
echo " days). Backfill target of " . ($backfillDate ? date('Y-m-d', $backfillDate) : $groupArr['backfill_target'] . " days") . " is post {$targetpost}.{$n}";
if ($targetpost >= $groupArr['first_record']) {
echo "Nothing to do, we already have the target post.{$n} {$n}";
return "";
}
//get first and last part numbers from newsgroup
if ($targetpost < $data['first']) {
echo "WARNING: Backfill came back as before server's first. Setting targetpost to server first.{$n}";
echo "Skipping Group {$n}";
return "";
}
echo $binaries->onlyProcessRegexBinaries === true ? "Note: Discarding parts that do not match a regex" . $n : "";
//Done with $nntp for now, close it to avoid timeouts.
$nntp->doQuit();
$nntpc->doConnect();
$datac = $nntpc->selectGroup($groupArr['name']);
if ($nntpc->isError($datac)) {
echo "Could not select group (bad name?): {$groupArr['name']}{$n}";
return;
}
//calculate total number of parts
$total = $groupArr['first_record'] - $targetpost;
$done = false;
//set first and last, moving the window by maxxMssgs
$last = $groupArr['first_record'] - 1;
$first = $last - $binaries->messagebuffer + 1;
//set initial "chunk"
if ($targetpost > $first) {
//just in case this is the last chunk we needed
$first = $targetpost;
}
while ($done === false) {
$binaries->startLoop = microtime(true);
echo "Getting " . ($last - $first + 1) . " parts (" . number_format($first - $targetpost) . " in queue)" . $n;
flush();
$success = $binaries->scan($nntpc, $groupArr, $first, $last, 'backfill');
if (!$success) {
return "";
}
$db->exec(sprintf("update groups SET first_record = %s, last_updated = now() WHERE ID = %d", $db->escapeString($first), $groupArr['ID']));
if ($first == $targetpost) {
$done = true;
} else {
//Keep going: set new last, new first, check for last chunk.
$last = $first - 1;
$first = $last - $binaries->messagebuffer + 1;
if ($targetpost > $first) {
$first = $targetpost;
}
}
}
//Done with $nntpc
$nntpc->doQuit();
//Just need $nntp for a quick check on the first_record_postdate
$nntp->doConnect();
//$nntp->selectGroup($groupArr['name']); // some users report having this in keeps backfill working
$first_record_postdate = $this->postdate($nntp, $first, false);
//All done with NNTP.
$nntp->doQuit();
if ($first_record_postdate != "") {
$db->exec(sprintf("update groups SET first_record_postdate = FROM_UNIXTIME(" . $first_record_postdate . "), last_updated = now() WHERE ID = %d", $groupArr['ID']));
}
//Set group's first postdate
$timeGroup = number_format(microtime(true) - $this->startGroup, 2);
echo "Group processed in {$timeGroup} seconds {$n}";
} else {
echo "Failed to get NNTP connection.{$n}";
}
}
function backfillGroup($nntp, $groupArr, $backfillDate = null)
{
$db = new DB();
$binaries = new Binaries();
$n = $this->n;
$this->startGroup = microtime(true);
echo 'Processing ' . $groupArr['name'] . $n;
$data = $nntp->selectGroup($groupArr['name']);
if (PEAR::isError($data)) {
echo "Could not select group (bad name?): {$groupArr['name']}{$n}";
return;
}
if ($backfillDate) {
$targetpost = $this->daytopost($nntp, $groupArr['name'], dateToDays($backfillDate), TRUE);
} else {
$targetpost = $this->daytopost($nntp, $groupArr['name'], $groupArr['backfill_target'], TRUE);
}
//get targetpost based on days target
if ($groupArr['first_record'] == 0 || $groupArr['backfill_target'] == 0) {
echo "Group " . $groupArr['name'] . " has invalid numbers. Have you run update on it? Have you set the backfill days amount?{$n}";
return;
}
echo "Group " . $data["group"] . ": server has " . $data['first'] . " - " . $data['last'] . ", or ~";
echo (int) (($this->postdate($nntp, $data['last'], FALSE) - $this->postdate($nntp, $data['first'], FALSE)) / 86400);
echo " days." . $n . "Local first = " . $groupArr['first_record'] . " (";
echo (int) ((date('U') - $this->postdate($nntp, $groupArr['first_record'], FALSE)) / 86400);
echo " days). Backfill target of " . $groupArr['backfill_target'] . "days is post {$targetpost}.{$n}";
if ($targetpost >= $groupArr['first_record']) {
echo "Nothing to do, we already have the target post.{$n} {$n}";
return "";
}
//get first and last part numbers from newsgroup
if ($targetpost < $data['first']) {
echo "WARNING: Backfill came back as before server's first. Setting targetpost to server first.{$n}";
echo "Skipping Group {$n}";
return "";
}
//calculate total number of parts
$total = $groupArr['first_record'] - $targetpost;
$done = false;
//set first and last, moving the window by maxxMssgs
$last = $groupArr['first_record'] - 1;
$first = $last - $binaries->messagebuffer + 1;
//set initial "chunk"
if ($targetpost > $first) {
//just in case this is the last chunk we needed
$first = $targetpost;
}
while ($done === false) {
$binaries->startLoop = microtime(true);
echo "Getting " . ($last - $first + 1) . " parts (" . ($first - $targetpost) . " in queue)" . $n;
flush();
$binaries->scan($nntp, $groupArr, $first, $last, 'backfill');
$db->query(sprintf("UPDATE groups SET first_record = %s, last_updated = now() WHERE ID = %d", $db->escapeString($first), $groupArr['ID']));
if ($first == $targetpost) {
$done = true;
} else {
//Keep going: set new last, new first, check for last chunk.
$last = $first - 1;
$first = $last - $binaries->messagebuffer + 1;
if ($targetpost > $first) {
$first = $targetpost;
}
}
}
$first_record_postdate = $this->postdate($nntp, $first, false);
$db->query(sprintf("UPDATE groups SET first_record_postdate = FROM_UNIXTIME(" . $first_record_postdate . "), last_updated = now() WHERE ID = %d", $groupArr['ID']));
//Set group's first postdate
$timeGroup = number_format(microtime(true) - $this->startGroup, 2);
echo "Group processed in {$timeGroup} seconds {$n}";
}
