function daytopost($nntp, $group, $days, $debug = true, $bfcheck = true)
{
global $pdo;
$st = false;
if ($debug && $bfcheck) {
echo $pdo->log->primary('Finding start and end articles for ' . $group . '.');
}
if (!isset($nntp)) {
$nntp = new \NNTP(['Settings' => $pdo]);
if ($nntp->doConnect(false) !== true) {
return;
}
$st = true;
}
$binaries = new \Binaries(['NNTP' => $nntp, 'Settings' => $pdo]);
$data = $nntp->selectGroup($group);
if ($nntp->isError($data)) {
$data = $nntp->dataError($nntp, $group, false);
if ($data === false) {
return;
}
}
// Goal timestamp.
$goaldate = date('U') - 86400 * $days;
$totalnumberofarticles = $data['last'] - $data['first'];
$upperbound = $data['last'];
$lowerbound = $data['first'];
if ($debug && $bfcheck) {
echo $pdo->log->header('Total Articles: ' . number_format($totalnumberofarticles) . ' Newest: ' . number_format($upperbound) . ' Oldest: ' . number_format($lowerbound));
}
if ($data['last'] == PHP_INT_MAX) {
exit($pdo->log->error("Group data is coming back as php's max value. You should not see this since we use a patched Net_NNTP that fixes this bug."));
}
$firstDate = $binaries->postdate($data['first'], $data);
$lastDate = $binaries->postdate($data['last'], $data);
if ($goaldate < $firstDate && $bfcheck) {
if ($st === true) {
$nntp->doQuit();
}
echo $pdo->log->warning("The oldest post indexed from {$days} day(s) ago is older than the first article stored on your news server.\nSetting to First available article of (date('r', {$firstDate}) or daysOld({$firstDate}) days).");
return $data['first'];
} else {
if ($goaldate > $lastDate && $bfcheck) {
if ($st === true) {
$nntp->doQuit();
}
echo $pdo->log->error("ERROR: The oldest post indexed from {$days} day(s) ago is newer than the last article stored on your news server.\nTo backfill this group you need to set Backfill Days to at least ceil(daysOld({$lastDate})+1) days (date('r', {$lastDate}-86400).");
return '';
}
}
if ($debug && $bfcheck) {
echo $pdo->log->primary("Searching for postdates.\nGroup's Firstdate: " . $firstDate . ' (' . (is_int($firstDate) ? date('r', $firstDate) : 'n/a') . ").\nGroup's Lastdate: " . $lastDate . ' (' . date('r', $lastDate) . ").");
}
$interval = floor(($upperbound - $lowerbound) * 0.5);
$templowered = '';
$dateofnextone = $lastDate;
// Match on days not timestamp to speed things up.
while (daysOld($dateofnextone) < $days) {
while (($tmpDate = $binaries->postdate($upperbound - $interval, $data)) > $goaldate) {
$upperbound = $upperbound - $interval;
}
if (!$templowered) {
$interval = ceil($interval / 2);
}
$dateofnextone = $binaries->postdate($upperbound - 1, $data);
while (!$dateofnextone) {
$dateofnextone = $binaries->postdate($upperbound - 1, $data);
}
}
if ($st === true) {
$nntp->doQuit();
}
if ($bfcheck) {
echo $pdo->log->header("\nBackfill article determined to be " . $upperbound . " " . $pdo->log->setColor('Yellow') . "(" . date('r', $dateofnextone) . ")" . $pdo->log->rsetcolor());
} else {
echo $pdo->log->header('Current article determined to be ' . $upperbound . " " . $pdo->log->setColor('Yellow') . "(" . date('r', $dateofnextone) . ")" . $pdo->log->rsetcolor());
}
// which is '.daysOld($dateofnextone)." days old.\n";
return $upperbound;
}
/**
* Download new headers for a single group.
*
* @param array $groupMySQL Array of MySQL results for a single group.
* @param int $maxHeaders (Optional) How many headers to download max.
*
* @return void
*/
public function updateGroup($groupMySQL, $maxHeaders = 0)
{
$startGroup = microtime(true);
// Select the group on the NNTP server, gets the latest info on it.
$groupNNTP = $this->_nntp->selectGroup($groupMySQL['name']);
if ($this->_nntp->isError($groupNNTP)) {
$groupNNTP = $this->_nntp->dataError($this->_nntp, $groupMySQL['name']);
if ($groupNNTP->code == 411) {
$this->_groups->disableIfNotExist($groupMySQL['id']);
}
if ($this->_nntp->isError($groupNNTP)) {
return;
}
}
if ($this->_echoCLI) {
$this->_colorCLI->doEcho($this->_colorCLI->primary('Processing ' . $groupMySQL['name']), true);
}
if ($groupMySQL['regexmatchonly'] == 1) {
$this->onlyProcessRegexBinaries = true;
if ($this->_echoCLI) {
$this->_colorCLI->doEcho($this->_colorCLI->primary('Note: Discarding parts that do not match a regex', true));
}
} else {
$this->onlyProcessRegexBinaries = false;
}
// Attempt to repair any missing parts before grabbing new ones.
if ($groupMySQL['last_record'] != 0) {
if ($this->_partRepair) {
if ($this->_echoCLI) {
$this->_colorCLI->doEcho($this->_colorCLI->primary('Part repair enabled. Checking for missing parts.'), true);
}
$this->partRepair($groupMySQL);
} else {
if ($this->_echoCLI) {
$this->_colorCLI->doEcho($this->_colorCLI->primary('Part repair disabled by user.'), true);
}
}
}
// Generate postdate for first record, for those that upgraded.
if (is_null($groupMySQL['first_record_postdate']) && $groupMySQL['first_record'] != 0) {
$groupMySQL['first_record_postdate'] = $this->postdate($groupMySQL['first_record'], $groupNNTP);
$this->_pdo->queryExec(sprintf('
UPDATE groups
SET first_record_postdate = %s
WHERE id = %d', $this->_pdo->from_unixtime($groupMySQL['first_record_postdate']), $groupMySQL['id']));
}
// Get first article we want aka the oldest.
if ($groupMySQL['last_record'] == 0) {
if ($this->_newGroupScanByDays) {
// For new newsgroups - determine here how far we want to go back using date.
$first = $this->daytopost($this->_newGroupDaysToScan, $groupNNTP);
} else {
if ($groupNNTP['first'] >= $groupNNTP['last'] - ($this->_newGroupMessagesToScan + $this->messageBuffer)) {
// If what we want is lower than the groups first article, set the wanted first to the first.
$first = $groupNNTP['first'];
} else {
// Or else, use the newest article minus how much we should get for new groups.
$first = (string) ($groupNNTP['last'] - ($this->_newGroupMessagesToScan + $this->messageBuffer));
}
}
// We will use this to subtract so we leave articles for the next time (in case the server doesn't have them yet)
$leaveOver = $this->messageBuffer;
// If this is not a new group, go from our newest to the servers newest.
} else {
// Set our oldest wanted to our newest local article.
$first = $groupMySQL['last_record'];
// This is how many articles we will grab. (the servers newest minus our newest).
$totalCount = (string) ($groupNNTP['last'] - $first);
// Check if the server has more articles than our loop limit x 2.
if ($totalCount > $this->messageBuffer * 2) {
// Get the remainder of $totalCount / $this->message buffer
$leaveOver = round($totalCount % $this->messageBuffer, 0, PHP_ROUND_HALF_DOWN) + $this->messageBuffer;
} else {
// Else get half of the available.
$leaveOver = round($totalCount / 2, 0, PHP_ROUND_HALF_DOWN);
}
}
// The last article we want, aka the newest.
$last = $groupLast = (string) ($groupNNTP['last'] - $leaveOver);
// If the newest we want is older than the oldest we want somehow.. set them equal.
if ($last < $first) {
$last = $groupLast = $first;
}
// This is how many articles we are going to get.
$total = (string) ($groupLast - $first);
// This is how many articles are available (without $leaveOver).
$realTotal = (string) ($groupNNTP['last'] - $first);
// Check if we should limit the amount of fetched new headers.
if ($maxHeaders > 0) {
if ($maxHeaders < $groupLast - $first) {
$groupLast = $last = (string) ($maxHeaders + $first);
}
$total = (string) ($groupLast - $first);
}
// If total is bigger than 0 it means we have new parts in the newsgroup.
if ($total > 0) {
if ($this->_echoCLI) {
$this->_colorCLI->doEcho($this->_colorCLI->primary(($groupMySQL['last_record'] == 0 ? 'New group ' . $groupNNTP['group'] . ' starting with ' . ($this->_newGroupScanByDays ? $this->_newGroupDaysToScan . ' days' : number_format($this->_newGroupMessagesToScan) . ' messages') . ' worth.' : 'Group ' . $groupNNTP['group'] . ' has ' . number_format($realTotal) . ' new articles.') . ' Leaving ' . number_format($leaveOver) . " for next pass.\nServer oldest: " . number_format($groupNNTP['first']) . ' Server newest: ' . number_format($groupNNTP['last']) . ' Local newest: ' . number_format($groupMySQL['last_record'])), true);
}
$done = false;
// Get all the parts (in portions of $this->messageBuffer to not use too much memory).
while ($done === false) {
// Increment last until we reach $groupLast (group newest article).
if ($total > $this->messageBuffer) {
$last = (string) ($first + $this->messageBuffer) > $groupLast ? $groupLast : (string) ($first + $this->messageBuffer);
}
// Increment first so we don't get an article we already had.
$first++;
if ($this->_echoCLI) {
$this->_colorCLI->doEcho($this->_colorCLI->header("\nGetting " . number_format($last - $first + 1) . ' articles (' . number_format($first) . ' to ' . number_format($last) . ') from ' . $groupMySQL['name'] . " - (" . number_format($groupLast - $last) . " articles in queue)."));
}
// Get article headers from newsgroup.
$scanSummary = $this->scan($groupMySQL, $first, $last);
// Check if we fetched headers.
if (!empty($scanSummary)) {
// If new group, update first record & postdate
if (is_null($groupMySQL['first_record_postdate']) && $groupMySQL['first_record'] == 0) {
$groupMySQL['first_record'] = $scanSummary['firstArticleNumber'];
if (isset($scanSummary['firstArticleDate'])) {
$groupMySQL['first_record_postdate'] = strtotime($scanSummary['firstArticleDate']);
} else {
$groupMySQL['first_record_postdate'] = $this->postdate($groupMySQL['first_record'], $groupNNTP);
}
$this->_pdo->queryExec(sprintf('
UPDATE groups
SET first_record = %s, first_record_postdate = %s
WHERE id = %d', $scanSummary['firstArticleNumber'], $this->_pdo->from_unixtime($this->_pdo->escapeString($groupMySQL['first_record_postdate'])), $groupMySQL['id']));
}
if (isset($scanSummary['lastArticleDate'])) {
$scanSummary['lastArticleDate'] = strtotime($scanSummary['lastArticleDate']);
} else {
$scanSummary['lastArticleDate'] = $this->postdate($scanSummary['lastArticleNumber'], $groupNNTP);
}
$this->_pdo->queryExec(sprintf('
UPDATE groups
SET last_record = %s, last_record_postdate = %s, last_updated = NOW()
WHERE id = %d', $this->_pdo->escapeString($scanSummary['lastArticleNumber']), $this->_pdo->from_unixtime($scanSummary['lastArticleDate']), $groupMySQL['id']));
} else {
// If we didn't fetch headers, update the record still.
$this->_pdo->queryExec(sprintf('
UPDATE groups
SET last_record = %s, last_updated = NOW()
WHERE id = %d', $this->_pdo->escapeString($last), $groupMySQL['id']));
}
if ($last == $groupLast) {
$done = true;
} else {
$first = $last;
}
}
if ($this->_echoCLI) {
$this->_colorCLI->doEcho($this->_colorCLI->primary(PHP_EOL . 'Group ' . $groupMySQL['name'] . ' processed in ' . number_format(microtime(true) - $startGroup, 2) . ' seconds.'), true);
}
} else {
if ($this->_echoCLI) {
$this->_colorCLI->doEcho($this->_colorCLI->primary('No new articles for ' . $groupMySQL['name'] . ' (first ' . number_format($first) . ', last ' . number_format($last) . ', grouplast ' . number_format($groupMySQL['last_record']) . ', total ' . number_format($total) . ")\n" . 'Server oldest: ' . number_format($groupNNTP['first']) . ' Server newest: ' . number_format($groupNNTP['last']) . ' Local newest: ' . number_format($groupMySQL['last_record'])), true);
}
}
}
/**
* Backfill single group.
*
* @param array $groupArr
* @param int $left
* @param int|string $articles
*
* @return void
*/
public function backfillGroup($groupArr, $left, $articles = '')
{
// Start time for this group.
$startGroup = microtime(true);
$groupName = str_replace('alt.binaries', 'a.b', $groupArr['name']);
// If our local oldest article 0, it means we never ran update_binaries on the group.
if ($groupArr['first_record'] <= 0) {
$dMessage = "You need to run update_binaries on " . $groupName . ". Otherwise the group is dead, you must disable it.";
if ($this->_debug) {
$this->_debugging->log('Backfill', "backfillGroup", $dMessage, Logger::LOG_ERROR);
}
if ($this->_echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->error($dMessage));
}
return;
}
// Select group, here, only once
$data = $this->_nntp->selectGroup($groupArr['name']);
if ($this->_nntp->isError($data)) {
$data = $this->_nntp->dataError($this->_nntp, $groupArr['name']);
if ($this->_nntp->isError($data)) {
return;
}
}
if ($this->_echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->primary('Processing ' . $groupName), true);
}
// Check if this is days or post backfill.
$postCheck = $articles === '' ? false : true;
// Get target post based on date or user specified number.
$targetpost = (string) ($postCheck ? round($groupArr['first_record'] - $articles) : $this->_binaries->daytopost($groupArr['backfill_target'], $data));
// Check if target post is smaller than server's oldest, set it to oldest if so.
if ($targetpost < $data['first']) {
$targetpost = $data['first'];
}
// Check if our target post is newer than our oldest post or if our local oldest article is older than the servers oldest.
if ($targetpost >= $groupArr['first_record'] || $groupArr['first_record'] <= $data['first']) {
$dMessage = "We have hit the maximum we can backfill for " . $groupName . ", skipping it, consider disabling backfill on it.";
if ($this->_debug) {
$this->_debugging->log('Backfill', "backfillGroup", $dMessage, Logger::LOG_NOTICE);
}
if ($this->_echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->notice($dMessage), true);
}
return;
}
if ($this->_echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->primary('Group ' . $groupName . "'s oldest article is " . number_format($data['first']) . ', newest is ' . number_format($data['last']) . ".\nOur target article is " . number_format($targetpost) . '. Our oldest article is article ' . number_format($groupArr['first_record']) . '.'));
}
// Set first and last, moving the window by max messages.
$last = (string) ($groupArr['first_record'] - 1);
// Set the initial "chunk".
$first = (string) ($last - $this->_binaries->messageBuffer + 1);
// Just in case this is the last chunk we needed.
if ($targetpost > $first) {
$first = $targetpost;
}
$done = false;
while ($done === false) {
if ($this->_echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->set256('Yellow') . "\nGetting " . number_format($last - $first + 1) . " articles from " . $groupName . ", " . $left . " group(s) left. (" . number_format($first - $targetpost) . " articles in queue)." . $this->pdo->log->rsetColor(), true);
}
flush();
$lastMsg = $this->_binaries->scan($groupArr, $first, $last, $this->_safePartRepair);
// Get the oldest date.
if (isset($lastMsg['firstArticleDate'])) {
// Try to get it from the oldest pulled article.
$newdate = strtotime($lastMsg['firstArticleDate']);
} else {
// If above failed, try to get it with postdate method.
$newdate = $this->_binaries->postdate($first, $data);
}
$this->pdo->queryExec(sprintf('
UPDATE groups
SET first_record_postdate = %s, first_record = %s, last_updated = NOW()
WHERE id = %d', $this->pdo->from_unixtime($newdate), $this->pdo->escapeString($first), $groupArr['id']));
if ($first == $targetpost) {
$done = true;
} else {
// Keep going: set new last, new first, check for last chunk.
$last = (string) ($first - 1);
$first = (string) ($last - $this->_binaries->messageBuffer + 1);
if ($targetpost > $first) {
$first = $targetpost;
}
}
}
if ($this->_echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->primary(PHP_EOL . 'Group ' . $groupName . ' processed in ' . number_format(microtime(true) - $startGroup, 2) . " seconds."), true);
}
}
