Esempio n. 1
0
<?php

require "../../www/config.php";
require_once WWW_DIR . "/lib/backfill.php";
$nntp = new nntp();
$nntp->doConnect();
$backfill = new Backfill();
print_r($backfill->daytopost($nntp, "alt.binaries.teevee", 100));
$nntp->doQuit();
Esempio n. 2
0
<?php

require "../../www/config.php";
require_once WWW_DIR . "/lib/backfill.php";
$nntp = new nntp();
$nntp->doConnect();
$backfill = new Backfill();
$data = $nntp->selectGroup("alt.binaries.nintendo.ds");
$output = $backfill->postdate($nntp, "7434768");
echo date('r', $output) . "\n";
print_r($data);
$nntp->doQuit();
Esempio n. 3
0
  DESCRIPTION:
    This script is an alternative to backfill.php.
    It allows you to backfill based on a specific date, bypassing
    the "backfill target" setting for each group, used by backfill.php
  
  PURPOSE:
    If you are backfilling many groups over a long span of time, the # of days
    set as your backfill target can become quickly outdated, resulting in
    potential gaps in your database. In this case, it may be more practical
    to specify a date explicity and just let the backfill work from there.
  
  USAGE:
    $ php backfill_date.php 2011-05-15
    => Script will backfill ALL active groups from May 15, 2011
    
    $ php backfill_date.php 2011-05-15 alt.binaries.games.xbox
    => Script will backfill ONLY a.b.games.xbox from May 15, 2011
*/
require "config.php";
require_once WWW_DIR . "/lib/backfill_date.php";
if (strtotime($argv[1]) && strtotime($argv[1]) < $strtotime('now')) {
    $groupName = isset($argv[2]) ? $argv[2] : '';
    $backfill = new Backfill();
    $backfill->backfillAllGroups($groupName, strtotime($argv[1]));
} else {
    echo "You must provide a backfill date in the format YYYY-MM-DD to use backfill_range.php\n";
    echo "example: backfill_range.php 2002-04-27 alt.binaries.games.xbox\n";
    echo "This will backfill your index with everything posted to a.b.g.x since April 27, 2002";
    echo "If you choose not to provide a groupname, all active groups will be backfilled.\n";
    echo "\nIf you do not want to use a date, use the backfill.php script instead.\n";
}
Esempio n. 4
0
if (isset($argv[1]) && $argv[1] == 'all' && $argv[1] !== 'safe' && $argv[1] !== 'alph' && $argv[1] !== 'date' && !is_numeric($argv[1]) && !isset($argv[2])) {
    $backfill = new \Backfill(['NNTP' => $nntp, 'Settings' => $pdo]);
    $backfill->backfillAllGroups();
} else {
    if (isset($argv[1]) && $argv[1] !== 'all' && $argv[1] !== 'safe' && $argv[1] !== 'alph' && $argv[1] !== 'date' && !is_numeric($argv[1]) && !isset($argv[2])) {
        $backfill = new \Backfill(['NNTP' => $nntp, 'Settings' => $pdo]);
        $backfill->backfillAllGroups($argv[1]);
    } else {
        if (isset($argv[1]) && $argv[1] !== 'all' && $argv[1] !== 'safe' && $argv[1] !== 'alph' && $argv[1] !== 'date' && !is_numeric($argv[1]) && isset($argv[2]) && is_numeric($argv[2])) {
            $backfill = new \Backfill(['NNTP' => $nntp, 'Settings' => $pdo]);
            $backfill->backfillAllGroups($argv[1], $argv[2]);
        } else {
            if (isset($argv[1]) && $argv[1] !== 'all' && $argv[1] !== 'safe' && $argv[1] == 'alph' && $argv[1] !== 'date' && !is_numeric($argv[1]) && isset($argv[2]) && is_numeric($argv[2])) {
                $backfill = new \Backfill(['NNTP' => $nntp, 'Settings' => $pdo]);
                $backfill->backfillAllGroups('', $argv[2], 'normal');
            } else {
                if (isset($argv[1]) && $argv[1] !== 'all' && $argv[1] !== 'safe' && $argv[1] !== 'alph' && $argv[1] == 'date' && !is_numeric($argv[1]) && isset($argv[2]) && is_numeric($argv[2])) {
                    $backfill = new \Backfill(['NNTP' => $nntp, 'Settings' => $pdo]);
                    $backfill->backfillAllGroups('', $argv[2], 'date');
                } else {
                    if (isset($argv[1]) && $argv[1] !== 'all' && $argv[1] == 'safe' && $argv[1] !== 'alph' && $argv[1] !== 'date' && !is_numeric($argv[1]) && isset($argv[2]) && is_numeric($argv[2])) {
                        $backfill = new \Backfill(['NNTP' => $nntp, 'Settings' => $pdo]);
                        $backfill->safeBackfill($argv[2]);
                    } else {
                        exit($pdo->log->error("\nWrong set of arguments.\n" . 'php backfill.php safe 200000		 ...: Backfill an active group alphabetically, x articles, the script stops,' . "\n" . '					 ...: if the group has reached reached 2012-06-24, the next group will backfill.' . "\n" . 'php backfill.php alph 200000 		 ...: Backfills all groups (sorted alphabetically) by number of articles' . "\n" . 'php backfill.php date 200000 		 ...: Backfills all groups (sorted by least backfilled in time) by number of articles' . "\n" . 'php backfill.php alt.binaries.ath 200000 ...: Backfills a group by name by number of articles' . "\n" . 'php backfill.php all			 ...: Backfills all groups 1 at a time, by date (set in admin-view groups)' . "\n" . 'php backfill.php alt.binaries.ath	 ...: Backfills a group by name, by date (set in admin-view groups)' . "\n"));
                    }
                }
            }
        }
    }
}
<?php

require "config.php";
require_once WWW_DIR . "/lib/backfill.php";
if (isset($argv[1])) {
    $groupName = $argv[1];
} else {
    $groupName = '';
}
if (isset($argv[2]) && $argv[2] == true) {
    $regexOnly = true;
} else {
    $regexOnly = false;
}
$backfill = new Backfill();
$backfill->backfillAllGroups($groupName, null, $regexOnly);
 /**
  * Process headers and store in database for a group.
  */
 function updateGroup($nntp = null, $groupArr)
 {
     $blnDoDisconnect = false;
     if ($nntp == null) {
         $nntp = new Nntp();
         if (!$nntp->doConnect()) {
             echo "Failed to get NNTP connection.";
             return;
         }
         $this->message = array();
         $blnDoDisconnect = true;
     }
     $db = new DB();
     $backfill = new Backfill();
     $n = $this->n;
     $this->startGroup = microtime(true);
     $this->startLoop = microtime(true);
     echo 'Processing ' . $groupArr['name'] . $n;
     // Connect to server
     $data = $nntp->selectGroup($groupArr['name']);
     if ($nntp->isError($data)) {
         echo "Could not select group (bad name?): {$groupArr['name']}{$n} {$n}";
         return;
     }
     if ($groupArr['regexmatchonly'] == 1) {
         $this->onlyProcessRegexBinaries = true;
         echo "Note: Discarding parts that do not match a regex" . $n;
     } else {
         $this->onlyProcessRegexBinaries = false;
     }
     //Attempt to repair any missing parts before grabbing new ones
     $this->partRepair($nntp, $groupArr);
     //Get first and last part numbers from newsgroup
     $last = $grouplast = $data['last'];
     // For new newsgroups - determine here how far you want to go back.
     if ($groupArr['last_record'] == 0) {
         if ($this->NewGroupScanByDays) {
             $first = $backfill->daytopost($nntp, $groupArr['name'], $this->NewGroupDaysToScan, true);
             if ($first == '') {
                 echo "Skipping group: {$groupArr['name']}{$n}";
                 return;
             }
         } else {
             if ($data['first'] > $data['last'] - $this->NewGroupMsgsToScan) {
                 $first = $data['first'];
             } else {
                 $first = $data['last'] - $this->NewGroupMsgsToScan;
             }
         }
         $first_record_postdate = $backfill->postdate($nntp, $first, false);
         if ($first_record_postdate != "") {
             $db->exec(sprintf("update groups SET first_record = %s, first_record_postdate = FROM_UNIXTIME(" . $first_record_postdate . ") WHERE ID = %d", $db->escapeString($first), $groupArr['ID']));
         }
     } else {
         if ($data['last'] < $groupArr['last_record']) {
             echo "Warning: Server's last num {$data['last']} is lower than the local last num {$groupArr['last_record']}" . $n;
             return;
         }
         $first = $groupArr['last_record'] + 1;
     }
     // Generate postdates for first and last records, for those that upgraded
     if ((is_null($groupArr['first_record_postdate']) || is_null($groupArr['last_record_postdate'])) && ($groupArr['last_record'] != "0" && $groupArr['first_record'] != "0")) {
         $db->exec(sprintf("update groups SET first_record_postdate = FROM_UNIXTIME(" . $backfill->postdate($nntp, $groupArr['first_record'], false) . "), last_record_postdate = FROM_UNIXTIME(" . $backfill->postdate($nntp, $groupArr['last_record'], false) . ") WHERE ID = %d", $groupArr['ID']));
     }
     // Deactivate empty groups
     if ($data['last'] - $data['first'] <= 5) {
         $db->exec(sprintf("update groups SET active = %s, last_updated = now() WHERE ID = %d", $db->escapeString('0'), $groupArr['ID']));
     }
     // Calculate total number of parts
     $total = $grouplast - $first + 1;
     // If total is bigger than 0 it means we have new parts in the newsgroup
     if ($total > 0) {
         echo "Group " . $data["group"] . " has " . number_format($total) . " new parts." . $n;
         if ($total > $this->MaxMsgsPerRun) {
             $grouplast = $first + $this->MaxMsgsPerRun;
             echo "NOTICE: Only processing first " . number_format($this->MaxMsgsPerRun) . " parts." . $n;
         }
         echo "First: " . $data['first'] . " Last: " . $data['last'] . " Local last: " . $groupArr['last_record'] . $n;
         if ($groupArr['last_record'] == 0) {
             echo "New group starting with " . ($this->NewGroupScanByDays ? $this->NewGroupDaysToScan . " days" : $this->NewGroupMsgsToScan . " messages") . " worth." . $n;
         }
         $done = false;
         // Get all the parts (in portions of $this->messagebuffer to not use too much memory)
         while ($done === false) {
             $this->startLoop = microtime(true);
             if ($total > $this->messagebuffer) {
                 if ($first + $this->messagebuffer > $grouplast) {
                     $last = $grouplast;
                 } else {
                     $last = $first + $this->messagebuffer;
                 }
             }
             echo "Getting " . number_format($last - $first + 1) . " parts (" . $first . " to " . $last . ") - " . number_format($grouplast - $last) . " in queue" . $n;
             flush();
             //get headers from newsgroup
             $lastId = $this->scan($nntp, $groupArr, $first, $last);
             if ($lastId === false) {
                 //scan failed - skip group
                 return;
             }
             $db->exec(sprintf("update groups SET last_record = %s, last_updated = now() WHERE ID = %d", $db->escapeString($lastId), $groupArr['ID']));
             if ($last == $grouplast) {
                 $done = true;
             } else {
                 $last = $lastId;
                 $first = $last + 1;
             }
         }
         $last_record_postdate = $backfill->postdate($nntp, $last, false);
         if ($last_record_postdate != "") {
             $db->exec(sprintf("update groups SET last_record_postdate = FROM_UNIXTIME(" . $last_record_postdate . "), last_updated = now() WHERE ID = %d", $groupArr['ID']));
             //Set group's last postdate
         }
         $timeGroup = number_format(microtime(true) - $this->startGroup, 2);
         echo "Group processed in {$timeGroup} seconds {$n} {$n}";
     } else {
         echo "No new records for " . $data["group"] . " (first {$first} last {$last} total {$total}) grouplast " . $groupArr['last_record'] . $n . $n;
     }
     if ($blnDoDisconnect) {
         $nntp->doQuit();
     }
 }
Esempio n. 7
0
     break;
     // Do a single group (update_binaries/backFill/update_releases/postprocess).
     // $options[2] => (int)groupid, group to work on
 // Do a single group (update_binaries/backFill/update_releases/postprocess).
 // $options[2] => (int)groupid, group to work on
 case 'update_per_group':
     if (is_numeric($options[2])) {
         $pdo = new Settings();
         // Get the group info from MySQL.
         $groupMySQL = $pdo->queryOneRow(sprintf('SELECT * FROM groups WHERE id = %d', $options[2]));
         if ($groupMySQL === false) {
             exit('ERROR: Group not found with id ' . $options[2] . PHP_EOL);
         }
         // Connect to NNTP.
         $nntp = nntp($pdo);
         $backFill = new \Backfill();
         // Update the group for new binaries.
         (new \Binaries())->updateGroup($groupMySQL);
         // BackFill the group with 20k articles.
         $backFill->backfillAllGroups($groupMySQL['name'], 20000, 'normal');
         // Create releases.
         processReleases(new \Releases(['Settings' => $pdo]), $options[2]);
         // Post process the releases.
         (new ProcessAdditional(['Echo' => true, 'NNTP' => $nntp, 'Settings' => $pdo]))->start($options[2]);
         (new \Info(['Echo' => true, 'Settings' => $pdo]))->processNfoFiles($nntp, $options[2]);
     }
     break;
     // Post process additional and NFO.
     // $options[2] => (char)Letter or number a-f 0-9, first character of release guid.
 // Post process additional and NFO.
 // $options[2] => (char)Letter or number a-f 0-9, first character of release guid.
Esempio n. 8
0
<?php

require "config.php";
require_once WWW_DIR . "/lib/backfill.php";
if (isset($argv[1])) {
    $groupName = $argv[1];
} else {
    $groupName = '';
}
$backfill = new Backfill();
$backfill->backfillAllGroups($groupName);
    to specify a date explicity and just let the backfill work from there.
  
  USAGE:
    $ php backfill_date.php 2011-05-15
    => Script will backfill ALL active groups from May 15, 2011
    
    $ php backfill_date.php 2011-05-15 alt.binaries.games.xbox
    => Script will backfill ONLY a.b.games.xbox from May 15, 2011
*/
require "config.php";
require_once WWW_DIR . "/lib/backfill.php";
$time = 0;
if (isset($argv[1])) {
    $time = strtotime($argv[1]);
}
if ($time > 1 && $time < time()) {
    $groupName = isset($argv[2]) ? $argv[2] : '';
    if (isset($argv[3]) && $argv[3] == true) {
        $regexOnly = true;
    } else {
        $regexOnly = false;
    }
    $backfill = new Backfill();
    $backfill->backfillAllGroups($groupName, strtotime($argv[1]), $regexOnly);
} else {
    echo "You must provide a backfill date in the format YYYY-MM-DD to use backfill_date.php\n";
    echo "example: backfill_date.php 2002-04-27 alt.binaries.games.xbox true\n";
    echo "This will backfill your index with everything posted to a.b.g.x since April 27, 2002 that matches system regex\n";
    echo "If you choose not to provide a groupname, all active groups will be backfilled.\n";
    echo "\nIf you do not want to use a date, use the backfill.php script instead.\n";
}