/** * Download new headers for a single group. * * @param array $groupMySQL Array of MySQL results for a single group. * @param int $maxHeaders (Optional) How many headers to download max. * * @return void */ public function updateGroup($groupMySQL, $maxHeaders = 0) { $startGroup = microtime(true); // Select the group on the NNTP server, gets the latest info on it. $groupNNTP = $this->_nntp->selectGroup($groupMySQL['name']); if ($this->_nntp->isError($groupNNTP)) { $groupNNTP = $this->_nntp->dataError($this->_nntp, $groupMySQL['name']); if ($this->_nntp->isError($groupNNTP)) { return; } } if ($this->_echoCLI) { $this->_colorCLI->doEcho($this->_colorCLI->primary('Processing ' . $groupMySQL['name']), true); } // Attempt to repair any missing parts before grabbing new ones. if ($groupMySQL['last_record'] != 0) { if ($this->_partRepair) { if ($this->_echoCLI) { $this->_colorCLI->doEcho($this->_colorCLI->primary('Part repair enabled. Checking for missing parts.'), true); } $this->partRepair($groupMySQL); } else { if ($this->_echoCLI) { $this->_colorCLI->doEcho($this->_colorCLI->primary('Part repair disabled by user.'), true); } } } // Generate postdate for first record, for those that upgraded. if (is_null($groupMySQL['first_record_postdate']) && $groupMySQL['first_record'] != 0) { $groupMySQL['first_record_postdate'] = $this->postdate($groupMySQL['first_record'], $groupNNTP); $this->_pdo->queryExec(sprintf(' UPDATE groups SET first_record_postdate = %s WHERE id = %d', $this->_pdo->from_unixtime($groupMySQL['first_record_postdate']), $groupMySQL['id'])); } // Get first article we want aka the oldest. if ($groupMySQL['last_record'] == 0) { if ($this->_newGroupScanByDays) { // For new newsgroups - determine here how far we want to go back using date. $first = $this->daytopost($this->_newGroupDaysToScan, $groupNNTP); } else { if ($groupNNTP['first'] >= $groupNNTP['last'] - ($this->_newGroupMessagesToScan + $this->messageBuffer)) { // If what we want is lower than the groups first article, set the wanted first to the first. $first = $groupNNTP['first']; } else { // Or else, use the newest article minus how much we should get for new groups. $first = (string) ($groupNNTP['last'] - ($this->_newGroupMessagesToScan + $this->messageBuffer)); } } // We will use this to subtract so we leave articles for the next time (in case the server doesn't have them yet) $leaveOver = $this->messageBuffer; // If this is not a new group, go from our newest to the servers newest. } else { // Set our oldest wanted to our newest local article. $first = $groupMySQL['last_record']; // This is how many articles we will grab. (the servers newest minus our newest). $totalCount = (string) ($groupNNTP['last'] - $first); // Check if the server has more articles than our loop limit x 2. if ($totalCount > $this->messageBuffer * 2) { // Get the remainder of $totalCount / $this->message buffer $leaveOver = round($totalCount % $this->messageBuffer, 0, PHP_ROUND_HALF_DOWN) + $this->messageBuffer; } else { // Else get half of the available. $leaveOver = round($totalCount / 2, 0, PHP_ROUND_HALF_DOWN); } } // The last article we want, aka the newest. $last = $groupLast = (string) ($groupNNTP['last'] - $leaveOver); // If the newest we want is older than the oldest we want somehow.. set them equal. if ($last < $first) { $last = $groupLast = $first; } // This is how many articles we are going to get. $total = (string) ($groupLast - $first); // This is how many articles are available (without $leaveOver). $realTotal = (string) ($groupNNTP['last'] - $first); // Check if we should limit the amount of fetched new headers. if ($maxHeaders > 0) { if ($maxHeaders < $groupLast - $first) { $groupLast = $last = (string) ($first + $maxHeaders); } $total = (string) ($groupLast - $first); } // If total is bigger than 0 it means we have new parts in the newsgroup. if ($total > 0) { if ($this->_echoCLI) { $this->_colorCLI->doEcho($this->_colorCLI->primary(($groupMySQL['last_record'] == 0 ? 'New group ' . $groupNNTP['group'] . ' starting with ' . ($this->_newGroupScanByDays ? $this->_newGroupDaysToScan . ' days' : number_format($this->_newGroupMessagesToScan) . ' messages') . ' worth.' : 'Group ' . $groupNNTP['group'] . ' has ' . number_format($realTotal) . ' new articles.') . ' Leaving ' . number_format($leaveOver) . " for next pass.\nServer oldest: " . number_format($groupNNTP['first']) . ' Server newest: ' . number_format($groupNNTP['last']) . ' Local newest: ' . number_format($groupMySQL['last_record'])), true); } $done = false; // Get all the parts (in portions of $this->messageBuffer to not use too much memory). while ($done === false) { // Increment last until we reach $groupLast (group newest article). if ($total > $this->messageBuffer) { if ((string) ($first + $this->messageBuffer) > $groupLast) { $last = $groupLast; } else { $last = (string) ($first + $this->messageBuffer); } } // Increment first so we don't get an article we already had. $first++; if ($this->_echoCLI) { $this->_colorCLI->doEcho($this->_colorCLI->header("\nGetting " . number_format($last - $first + 1) . ' articles (' . number_format($first) . ' to ' . number_format($last) . ') from ' . $groupMySQL['name'] . " - (" . number_format($groupLast - $last) . " articles in queue).")); } // Get article headers from newsgroup. $scanSummary = $this->scan($groupMySQL, $first, $last); // Check if we fetched headers. if (!empty($scanSummary)) { // If new group, update first record & postdate if (is_null($groupMySQL['first_record_postdate']) && $groupMySQL['first_record'] == 0) { $groupMySQL['first_record'] = $scanSummary['firstArticleNumber']; if (isset($scanSummary['firstArticleDate'])) { $groupMySQL['first_record_postdate'] = strtotime($scanSummary['firstArticleDate']); } else { $groupMySQL['first_record_postdate'] = $this->postdate($groupMySQL['first_record'], $groupNNTP); } $this->_pdo->queryExec(sprintf(' UPDATE groups SET first_record = %s, first_record_postdate = %s WHERE id = %d', $scanSummary['firstArticleNumber'], $this->_pdo->from_unixtime($this->_pdo->escapeString($groupMySQL['first_record_postdate'])), $groupMySQL['id'])); } if (isset($scanSummary['lastArticleDate'])) { $scanSummary['lastArticleDate'] = strtotime($scanSummary['lastArticleDate']); } else { $scanSummary['lastArticleDate'] = $this->postdate($scanSummary['lastArticleNumber'], $groupNNTP); } $this->_pdo->queryExec(sprintf(' UPDATE groups SET last_record = %s, last_record_postdate = %s, last_updated = NOW() WHERE id = %d', $this->_pdo->escapeString($scanSummary['lastArticleNumber']), $this->_pdo->from_unixtime($scanSummary['lastArticleDate']), $groupMySQL['id'])); } else { // If we didn't fetch headers, update the record still. $this->_pdo->queryExec(sprintf(' UPDATE groups SET last_record = %s, last_updated = NOW() WHERE id = %d', $this->_pdo->escapeString($last), $groupMySQL['id'])); } if ($last == $groupLast) { $done = true; } else { $first = $last; } } if ($this->_echoCLI) { $this->_colorCLI->doEcho($this->_colorCLI->primary(PHP_EOL . 'Group ' . $groupMySQL['name'] . ' processed in ' . number_format(microtime(true) - $startGroup, 2) . ' seconds.'), true); } } else { if ($this->_echoCLI) { $this->_colorCLI->doEcho($this->_colorCLI->primary('No new articles for ' . $groupMySQL['name'] . ' (first ' . number_format($first) . ', last ' . number_format($last) . ', grouplast ' . number_format($groupMySQL['last_record']) . ', total ' . number_format($total) . ")\n" . 'Server oldest: ' . number_format($groupNNTP['first']) . ' Server newest: ' . number_format($groupNNTP['last']) . ' Local newest: ' . number_format($groupMySQL['last_record'])), true); } } }