<?php require "../../www/config.php"; require_once WWW_DIR . "/lib/backfill.php"; $nntp = new nntp(); $nntp->doConnect(); $backfill = new Backfill(); print_r($backfill->daytopost($nntp, "alt.binaries.teevee", 100)); $nntp->doQuit();
<?php
require "../../www/config.php";
require_once WWW_DIR . "/lib/backfill.php";
$nntp = new nntp();
$nntp->doConnect();
$backfill = new Backfill();
$data = $nntp->selectGroup("alt.binaries.nintendo.ds");
$output = $backfill->postdate($nntp, "7434768");
echo date('r', $output) . "\n";
print_r($data);
$nntp->doQuit();
DESCRIPTION:
This script is an alternative to backfill.php.
It allows you to backfill based on a specific date, bypassing
the "backfill target" setting for each group, used by backfill.php
PURPOSE:
If you are backfilling many groups over a long span of time, the # of days
set as your backfill target can become quickly outdated, resulting in
potential gaps in your database. In this case, it may be more practical
to specify a date explicity and just let the backfill work from there.
USAGE:
$ php backfill_date.php 2011-05-15
=> Script will backfill ALL active groups from May 15, 2011
$ php backfill_date.php 2011-05-15 alt.binaries.games.xbox
=> Script will backfill ONLY a.b.games.xbox from May 15, 2011
*/
require "config.php";
require_once WWW_DIR . "/lib/backfill_date.php";
if (strtotime($argv[1]) && strtotime($argv[1]) < $strtotime('now')) {
$groupName = isset($argv[2]) ? $argv[2] : '';
$backfill = new Backfill();
$backfill->backfillAllGroups($groupName, strtotime($argv[1]));
} else {
echo "You must provide a backfill date in the format YYYY-MM-DD to use backfill_range.php\n";
echo "example: backfill_range.php 2002-04-27 alt.binaries.games.xbox\n";
echo "This will backfill your index with everything posted to a.b.g.x since April 27, 2002";
echo "If you choose not to provide a groupname, all active groups will be backfilled.\n";
echo "\nIf you do not want to use a date, use the backfill.php script instead.\n";
}
if (isset($argv[1]) && $argv[1] == 'all' && $argv[1] !== 'safe' && $argv[1] !== 'alph' && $argv[1] !== 'date' && !is_numeric($argv[1]) && !isset($argv[2])) {
$backfill = new \Backfill(['NNTP' => $nntp, 'Settings' => $pdo]);
$backfill->backfillAllGroups();
} else {
if (isset($argv[1]) && $argv[1] !== 'all' && $argv[1] !== 'safe' && $argv[1] !== 'alph' && $argv[1] !== 'date' && !is_numeric($argv[1]) && !isset($argv[2])) {
$backfill = new \Backfill(['NNTP' => $nntp, 'Settings' => $pdo]);
$backfill->backfillAllGroups($argv[1]);
} else {
if (isset($argv[1]) && $argv[1] !== 'all' && $argv[1] !== 'safe' && $argv[1] !== 'alph' && $argv[1] !== 'date' && !is_numeric($argv[1]) && isset($argv[2]) && is_numeric($argv[2])) {
$backfill = new \Backfill(['NNTP' => $nntp, 'Settings' => $pdo]);
$backfill->backfillAllGroups($argv[1], $argv[2]);
} else {
if (isset($argv[1]) && $argv[1] !== 'all' && $argv[1] !== 'safe' && $argv[1] == 'alph' && $argv[1] !== 'date' && !is_numeric($argv[1]) && isset($argv[2]) && is_numeric($argv[2])) {
$backfill = new \Backfill(['NNTP' => $nntp, 'Settings' => $pdo]);
$backfill->backfillAllGroups('', $argv[2], 'normal');
} else {
if (isset($argv[1]) && $argv[1] !== 'all' && $argv[1] !== 'safe' && $argv[1] !== 'alph' && $argv[1] == 'date' && !is_numeric($argv[1]) && isset($argv[2]) && is_numeric($argv[2])) {
$backfill = new \Backfill(['NNTP' => $nntp, 'Settings' => $pdo]);
$backfill->backfillAllGroups('', $argv[2], 'date');
} else {
if (isset($argv[1]) && $argv[1] !== 'all' && $argv[1] == 'safe' && $argv[1] !== 'alph' && $argv[1] !== 'date' && !is_numeric($argv[1]) && isset($argv[2]) && is_numeric($argv[2])) {
$backfill = new \Backfill(['NNTP' => $nntp, 'Settings' => $pdo]);
$backfill->safeBackfill($argv[2]);
} else {
exit($pdo->log->error("\nWrong set of arguments.\n" . 'php backfill.php safe 200000 ...: Backfill an active group alphabetically, x articles, the script stops,' . "\n" . ' ...: if the group has reached reached 2012-06-24, the next group will backfill.' . "\n" . 'php backfill.php alph 200000 ...: Backfills all groups (sorted alphabetically) by number of articles' . "\n" . 'php backfill.php date 200000 ...: Backfills all groups (sorted by least backfilled in time) by number of articles' . "\n" . 'php backfill.php alt.binaries.ath 200000 ...: Backfills a group by name by number of articles' . "\n" . 'php backfill.php all ...: Backfills all groups 1 at a time, by date (set in admin-view groups)' . "\n" . 'php backfill.php alt.binaries.ath ...: Backfills a group by name, by date (set in admin-view groups)' . "\n"));
}
}
}
}
}
}
<?php
require "config.php";
require_once WWW_DIR . "/lib/backfill.php";
if (isset($argv[1])) {
$groupName = $argv[1];
} else {
$groupName = '';
}
if (isset($argv[2]) && $argv[2] == true) {
$regexOnly = true;
} else {
$regexOnly = false;
}
$backfill = new Backfill();
$backfill->backfillAllGroups($groupName, null, $regexOnly);
/**
* Process headers and store in database for a group.
*/
function updateGroup($nntp = null, $groupArr)
{
$blnDoDisconnect = false;
if ($nntp == null) {
$nntp = new Nntp();
if (!$nntp->doConnect()) {
echo "Failed to get NNTP connection.";
return;
}
$this->message = array();
$blnDoDisconnect = true;
}
$db = new DB();
$backfill = new Backfill();
$n = $this->n;
$this->startGroup = microtime(true);
$this->startLoop = microtime(true);
echo 'Processing ' . $groupArr['name'] . $n;
// Connect to server
$data = $nntp->selectGroup($groupArr['name']);
if ($nntp->isError($data)) {
echo "Could not select group (bad name?): {$groupArr['name']}{$n} {$n}";
return;
}
if ($groupArr['regexmatchonly'] == 1) {
$this->onlyProcessRegexBinaries = true;
echo "Note: Discarding parts that do not match a regex" . $n;
} else {
$this->onlyProcessRegexBinaries = false;
}
//Attempt to repair any missing parts before grabbing new ones
$this->partRepair($nntp, $groupArr);
//Get first and last part numbers from newsgroup
$last = $grouplast = $data['last'];
// For new newsgroups - determine here how far you want to go back.
if ($groupArr['last_record'] == 0) {
if ($this->NewGroupScanByDays) {
$first = $backfill->daytopost($nntp, $groupArr['name'], $this->NewGroupDaysToScan, true);
if ($first == '') {
echo "Skipping group: {$groupArr['name']}{$n}";
return;
}
} else {
if ($data['first'] > $data['last'] - $this->NewGroupMsgsToScan) {
$first = $data['first'];
} else {
$first = $data['last'] - $this->NewGroupMsgsToScan;
}
}
$first_record_postdate = $backfill->postdate($nntp, $first, false);
if ($first_record_postdate != "") {
$db->exec(sprintf("update groups SET first_record = %s, first_record_postdate = FROM_UNIXTIME(" . $first_record_postdate . ") WHERE ID = %d", $db->escapeString($first), $groupArr['ID']));
}
} else {
if ($data['last'] < $groupArr['last_record']) {
echo "Warning: Server's last num {$data['last']} is lower than the local last num {$groupArr['last_record']}" . $n;
return;
}
$first = $groupArr['last_record'] + 1;
}
// Generate postdates for first and last records, for those that upgraded
if ((is_null($groupArr['first_record_postdate']) || is_null($groupArr['last_record_postdate'])) && ($groupArr['last_record'] != "0" && $groupArr['first_record'] != "0")) {
$db->exec(sprintf("update groups SET first_record_postdate = FROM_UNIXTIME(" . $backfill->postdate($nntp, $groupArr['first_record'], false) . "), last_record_postdate = FROM_UNIXTIME(" . $backfill->postdate($nntp, $groupArr['last_record'], false) . ") WHERE ID = %d", $groupArr['ID']));
}
// Deactivate empty groups
if ($data['last'] - $data['first'] <= 5) {
$db->exec(sprintf("update groups SET active = %s, last_updated = now() WHERE ID = %d", $db->escapeString('0'), $groupArr['ID']));
}
// Calculate total number of parts
$total = $grouplast - $first + 1;
// If total is bigger than 0 it means we have new parts in the newsgroup
if ($total > 0) {
echo "Group " . $data["group"] . " has " . number_format($total) . " new parts." . $n;
if ($total > $this->MaxMsgsPerRun) {
$grouplast = $first + $this->MaxMsgsPerRun;
echo "NOTICE: Only processing first " . number_format($this->MaxMsgsPerRun) . " parts." . $n;
}
echo "First: " . $data['first'] . " Last: " . $data['last'] . " Local last: " . $groupArr['last_record'] . $n;
if ($groupArr['last_record'] == 0) {
echo "New group starting with " . ($this->NewGroupScanByDays ? $this->NewGroupDaysToScan . " days" : $this->NewGroupMsgsToScan . " messages") . " worth." . $n;
}
$done = false;
// Get all the parts (in portions of $this->messagebuffer to not use too much memory)
while ($done === false) {
$this->startLoop = microtime(true);
if ($total > $this->messagebuffer) {
if ($first + $this->messagebuffer > $grouplast) {
$last = $grouplast;
} else {
$last = $first + $this->messagebuffer;
}
}
echo "Getting " . number_format($last - $first + 1) . " parts (" . $first . " to " . $last . ") - " . number_format($grouplast - $last) . " in queue" . $n;
flush();
//get headers from newsgroup
$lastId = $this->scan($nntp, $groupArr, $first, $last);
if ($lastId === false) {
//scan failed - skip group
return;
}
$db->exec(sprintf("update groups SET last_record = %s, last_updated = now() WHERE ID = %d", $db->escapeString($lastId), $groupArr['ID']));
if ($last == $grouplast) {
$done = true;
} else {
$last = $lastId;
$first = $last + 1;
}
}
$last_record_postdate = $backfill->postdate($nntp, $last, false);
if ($last_record_postdate != "") {
$db->exec(sprintf("update groups SET last_record_postdate = FROM_UNIXTIME(" . $last_record_postdate . "), last_updated = now() WHERE ID = %d", $groupArr['ID']));
//Set group's last postdate
}
$timeGroup = number_format(microtime(true) - $this->startGroup, 2);
echo "Group processed in {$timeGroup} seconds {$n} {$n}";
} else {
echo "No new records for " . $data["group"] . " (first {$first} last {$last} total {$total}) grouplast " . $groupArr['last_record'] . $n . $n;
}
if ($blnDoDisconnect) {
$nntp->doQuit();
}
}
break;
// Do a single group (update_binaries/backFill/update_releases/postprocess).
// $options[2] => (int)groupid, group to work on
// Do a single group (update_binaries/backFill/update_releases/postprocess).
// $options[2] => (int)groupid, group to work on
case 'update_per_group':
if (is_numeric($options[2])) {
$pdo = new Settings();
// Get the group info from MySQL.
$groupMySQL = $pdo->queryOneRow(sprintf('SELECT * FROM groups WHERE id = %d', $options[2]));
if ($groupMySQL === false) {
exit('ERROR: Group not found with id ' . $options[2] . PHP_EOL);
}
// Connect to NNTP.
$nntp = nntp($pdo);
$backFill = new \Backfill();
// Update the group for new binaries.
(new \Binaries())->updateGroup($groupMySQL);
// BackFill the group with 20k articles.
$backFill->backfillAllGroups($groupMySQL['name'], 20000, 'normal');
// Create releases.
processReleases(new \Releases(['Settings' => $pdo]), $options[2]);
// Post process the releases.
(new ProcessAdditional(['Echo' => true, 'NNTP' => $nntp, 'Settings' => $pdo]))->start($options[2]);
(new \Info(['Echo' => true, 'Settings' => $pdo]))->processNfoFiles($nntp, $options[2]);
}
break;
// Post process additional and NFO.
// $options[2] => (char)Letter or number a-f 0-9, first character of release guid.
// Post process additional and NFO.
// $options[2] => (char)Letter or number a-f 0-9, first character of release guid.
<?php
require "config.php";
require_once WWW_DIR . "/lib/backfill.php";
if (isset($argv[1])) {
$groupName = $argv[1];
} else {
$groupName = '';
}
$backfill = new Backfill();
$backfill->backfillAllGroups($groupName);
to specify a date explicity and just let the backfill work from there.
USAGE:
$ php backfill_date.php 2011-05-15
=> Script will backfill ALL active groups from May 15, 2011
$ php backfill_date.php 2011-05-15 alt.binaries.games.xbox
=> Script will backfill ONLY a.b.games.xbox from May 15, 2011
*/
require "config.php";
require_once WWW_DIR . "/lib/backfill.php";
$time = 0;
if (isset($argv[1])) {
$time = strtotime($argv[1]);
}
if ($time > 1 && $time < time()) {
$groupName = isset($argv[2]) ? $argv[2] : '';
if (isset($argv[3]) && $argv[3] == true) {
$regexOnly = true;
} else {
$regexOnly = false;
}
$backfill = new Backfill();
$backfill->backfillAllGroups($groupName, strtotime($argv[1]), $regexOnly);
} else {
echo "You must provide a backfill date in the format YYYY-MM-DD to use backfill_date.php\n";
echo "example: backfill_date.php 2002-04-27 alt.binaries.games.xbox true\n";
echo "This will backfill your index with everything posted to a.b.g.x since April 27, 2002 that matches system regex\n";
echo "If you choose not to provide a groupname, all active groups will be backfilled.\n";
echo "\nIf you do not want to use a date, use the backfill.php script instead.\n";
}