Ejemplo n.º 1
0
    /**
     * Fetch a comment and insert it.
     *
     * @param string $messageID Message-ID for the article.
     * @param string $siteID    ID of the site.
     *
     * @return bool
     *
     * @access protected
     */
    protected function insertNewComment(&$messageID, &$siteID)
    {
        // Get the article body.
        $body = $this->nntp->getMessages(self::group, $messageID);
        // Check if there's an error.
        if ($this->nntp->isError($body)) {
            return false;
        }
        // Decompress the body.
        $body = @gzinflate($body);
        if ($body === false) {
            return false;
        }
        // JSON Decode the body.
        $body = json_decode($body, true);
        if ($body === false) {
            return false;
        }
        // Just in case.
        if (!isset($body['USER']) || !isset($body['SID']) || !isset($body['RID']) || !isset($body['TIME']) | !isset($body['BODY'])) {
            return false;
        }
        // Insert the comment.
        if ($this->pdo->queryExec(sprintf('
				INSERT INTO release_comments
				(text, createddate, shareid, nzb_guid, siteid, username, user_id, releaseid, shared, host)
				VALUES (%s, %s, %s, UNHEX(%s), %s, %s, 0, 0, 2, "")', $this->pdo->escapeString($body['BODY']), $this->pdo->from_unixtime($body['TIME'] > time() ? time() : $body['TIME']), $this->pdo->escapeString($body['SID']), $this->pdo->escapeString($body['RID']), $this->pdo->escapeString($siteID), $this->pdo->escapeString(substr($body['USER'], 0, 3) === 'sn-' ? 'SH_ANON' : 'SH_' . $body['USER'])))) {
            return true;
        }
        return false;
    }
Ejemplo n.º 2
0
 /**
  * Look for an .nfo file in the NZB, return the NFO message id.
  * Gets the NZB completion.
  * Looks for PAR2 files in the NZB.
  *
  * @param string $guid
  * @param string $relID
  * @param int    $groupID
  * @param string $groupName
  *
  * @return bool
  *
  * @access public
  */
 public function getNfoFromNZB($guid, $relID, $groupID, $groupName)
 {
     $fetchedBinary = false;
     $messageID = $this->parseNZB($guid, $relID, $groupID, true);
     if ($messageID !== false) {
         $fetchedBinary = $this->nntp->getMessages($groupName, $messageID['ID'], $this->alternateNNTP);
         if ($this->nntp->isError($fetchedBinary)) {
             // NFO download failed, increment attempts.
             $this->pdo->queryExec(sprintf('UPDATE releases SET nfostatus = nfostatus - 1 WHERE id = %d', $relID));
             if ($this->echooutput) {
                 echo 'f';
             }
             return false;
         }
         if ($this->nfo->isNFO($fetchedBinary, $guid) === true) {
             if ($this->echooutput) {
                 echo $messageID['hidden'] === false ? '+' : '*';
             }
         } else {
             if ($this->echooutput) {
                 echo '-';
             }
             $this->pdo->queryExec(sprintf('UPDATE releases SET nfostatus = %d WHERE id = %d', Nfo::NFO_NONFO, $relID));
             $fetchedBinary = false;
         }
     } else {
         if ($this->echooutput) {
             echo '-';
         }
         $this->pdo->queryExec(sprintf('UPDATE releases SET nfostatus = %d WHERE id = %d', Nfo::NFO_NONFO, $relID));
     }
     return $fetchedBinary;
 }
Ejemplo n.º 3
0
use nzedb\NNTP;
use nzedb\db\Settings;
use nzedb\processing\PostProcess;
$pdo = new Settings();
/**
  Array with possible arguments for run and
  whether or not those methods of operation require NNTP
**/
$args = array('additional' => true, 'all' => true, 'allinf' => true, 'amazon' => false, 'anime' => false, 'book' => false, 'console' => false, 'games' => false, 'movies' => false, 'music' => false, 'nfo' => true, 'pre' => true, 'sharing' => true, 'tv' => false, 'xxx' => false);
$bool = array('true', 'false');
if (!isset($argv[1]) || !in_array($argv[1], $args) || !isset($argv[2]) || !in_array($argv[2], $bool)) {
    exit($pdo->log->error("\nIncorrect arguments.\n" . "The second argument (true/false) determines wether to echo or not.\n\n" . "php postprocess.php all true         ...: Does all the types of post processing.\n" . "php postprocess.php pre true         ...: Processes all Predb sites.\n" . "php postprocess.php nfo true         ...: Processes NFO files.\n" . "php postprocess.php movies true      ...: Processes movies.\n" . "php postprocess.php music true       ...: Processes music.\n" . "php postprocess.php console true     ...: Processes console games.\n" . "php postprocess.php games true       ...: Processes games.\n" . "php postprocess.php book true        ...: Processes books.\n" . "php postprocess.php anime true       ...: Processes anime.\n" . "php postprocess.php tv true          ...: Processes tv.\n" . "php postprocess.php xxx true         ...: Processes xxx.\n" . "php postprocess.php additional true  ...: Processes previews/mediainfo/etc...\n" . "php postprocess.php sharing true     ...: Processes uploading/downloading comments.\n" . "php postprocess.php allinf true      ...: Does all the types of post processing on a loop, sleeping 15 seconds between.\n" . "php postprocess.php amazon true      ...: Does all the amazon (books/console/games/music/xxx).\n"));
}
$nntp = null;
if ($args[$argv[1]] === true) {
    $nntp = new NNTP(['Settings' => $pdo]);
    if (($pdo->getSetting('alternate_nntp') == 1 ? $nntp->doConnect(true, true) : $nntp->doConnect()) !== true) {
        exit($pdo->log->error("Unable to connect to usenet." . PHP_EOL));
    }
}
$postProcess = new PostProcess(['Settings' => $pdo, 'Echo' => $argv[2] === 'true' ? true : false]);
$charArray = ['a', 'b', 'c', 'd', 'e', 'f', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9'];
switch ($argv[1]) {
    case 'all':
        $postProcess->processAll($nntp);
        break;
    case 'allinf':
        $i = 1;
        while ($i = 1) {
            $postProcess->processAll($nntp);
            sleep(15);
Ejemplo n.º 4
0
    /**
     * Backfill single group.
     *
     * @param array $groupArr
     * @param int $left
     * @param int|string $articles
     *
     * @return void
     */
    public function backfillGroup($groupArr, $left, $articles = '')
    {
        // Start time for this group.
        $startGroup = microtime(true);
        $groupName = str_replace('alt.binaries', 'a.b', $groupArr['name']);
        // If our local oldest article 0, it means we never ran update_binaries on the group.
        if ($groupArr['first_record'] <= 0) {
            $dMessage = "You need to run update_binaries on " . $groupName . ". Otherwise the group is dead, you must disable it.";
            if ($this->_debug) {
                $this->_debugging->log(get_class(), __FUNCTION__, $dMessage, Logger::LOG_ERROR);
            }
            if ($this->_echoCLI) {
                $this->pdo->log->doEcho($this->pdo->log->error($dMessage));
            }
            return;
        }
        // Select group, here, only once
        $data = $this->_nntp->selectGroup($groupArr['name']);
        if ($this->_nntp->isError($data)) {
            $data = $this->_nntp->dataError($this->_nntp, $groupArr['name']);
            if ($this->_nntp->isError($data)) {
                return;
            }
        }
        if ($this->_echoCLI) {
            $this->pdo->log->doEcho($this->pdo->log->primary('Processing ' . $groupName), true);
        }
        // Check if this is days or post backfill.
        $postCheck = $articles === '' ? false : true;
        // Get target post based on date or user specified number.
        $targetpost = (string) ($postCheck ? round($groupArr['first_record'] - $articles) : $this->_binaries->daytopost($groupArr['backfill_target'], $data));
        // Check if target post is smaller than server's oldest, set it to oldest if so.
        if ($targetpost < $data['first']) {
            $targetpost = $data['first'];
        }
        // Check if our target post is newer than our oldest post or if our local oldest article is older than the servers oldest.
        if ($targetpost >= $groupArr['first_record'] || $groupArr['first_record'] <= $data['first']) {
            $dMessage = "We have hit the maximum we can backfill for " . $groupName . ($this->_disableBackfillGroup ? ", disabling backfill on it." : ", skipping it, consider disabling backfill on it.");
            if ($this->_debug) {
                $this->_debugging->log(get_class(), __FUNCTION__, $dMessage, Logger::LOG_NOTICE);
            }
            if ($this->_disableBackfillGroup) {
                $this->pdo->queryExec(sprintf('
					UPDATE groups
					SET backfill = 0
					WHERE id = %d', $groupArr['id']));
            }
            if ($this->_echoCLI) {
                $this->pdo->log->doEcho($this->pdo->log->notice($dMessage), true);
            }
            return;
        }
        if ($this->_echoCLI) {
            $this->pdo->log->doEcho($this->pdo->log->primary('Group ' . $groupName . "'s oldest article is " . number_format($data['first']) . ', newest is ' . number_format($data['last']) . ".\nOur target article is " . number_format($targetpost) . '. Our oldest article is article ' . number_format($groupArr['first_record']) . '.'));
        }
        // Set first and last, moving the window by max messages.
        $last = (string) ($groupArr['first_record'] - 1);
        // Set the initial "chunk".
        $first = (string) ($last - $this->_binaries->messageBuffer + 1);
        // Just in case this is the last chunk we needed.
        if ($targetpost > $first) {
            $first = $targetpost;
        }
        $done = false;
        while ($done === false) {
            if ($this->_echoCLI) {
                $this->pdo->log->doEcho($this->pdo->log->set256('Yellow') . "\nGetting " . number_format($last - $first + 1) . " articles from " . $groupName . ", " . $left . " group(s) left. (" . number_format($first - $targetpost) . " articles in queue)." . $this->pdo->log->rsetColor(), true);
            }
            flush();
            $lastMsg = $this->_binaries->scan($groupArr, $first, $last, $this->_safePartRepair);
            // Get the oldest date.
            if (isset($lastMsg['firstArticleDate'])) {
                // Try to get it from the oldest pulled article.
                $newdate = strtotime($lastMsg['firstArticleDate']);
            } else {
                // If above failed, try to get it with postdate method.
                $newdate = $this->_binaries->postdate($first, $data);
            }
            $this->pdo->queryExec(sprintf('
					UPDATE groups
					SET first_record_postdate = %s, first_record = %s, last_updated = NOW()
					WHERE id = %d', $this->pdo->from_unixtime($newdate), $this->pdo->escapeString($first), $groupArr['id']));
            if ($first == $targetpost) {
                $done = true;
            } else {
                // Keep going: set new last, new first, check for last chunk.
                $last = (string) ($first - 1);
                $first = (string) ($last - $this->_binaries->messageBuffer + 1);
                if ($targetpost > $first) {
                    $first = $targetpost;
                }
            }
        }
        if ($this->_echoCLI) {
            $this->pdo->log->doEcho($this->pdo->log->primary(PHP_EOL . 'Group ' . $groupName . ' processed in ' . number_format(microtime(true) - $startGroup, 2) . " seconds."), true);
        }
    }
Ejemplo n.º 5
0
/**
 * Connect to usenet, return NNTP object.
 *
 * @param Settings $pdo
 * @param bool               $alternate Use alternate NNTP provider.
 *
 * @return NNTP
 */
function &nntp(&$pdo, $alternate = false)
{
    $nntp = new NNTP(['Settings' => $pdo]);
    if (($alternate && $pdo->getSetting('alternate_nntp') == 1 ? $nntp->doConnect(true, true) : $nntp->doConnect()) !== true) {
        exit("ERROR: Unable to connect to usenet." . PHP_EOL);
    }
    return $nntp;
}
Ejemplo n.º 6
0
<?php

require_once realpath(dirname(dirname(__DIR__)) . DIRECTORY_SEPARATOR . 'indexer.php');
use nzedb\ConsoleTools;
use nzedb\NNTP;
use nzedb\db\Settings;
use nzedb\processing\ProcessReleases;
$pdo = new Settings();
if (isset($argv[2]) && $argv[2] === 'true') {
    // Create the connection here and pass
    $nntp = new NNTP(['Settings' => $pdo]);
    if ($nntp->doConnect() !== true) {
        exit($pdo->log->error("Unable to connect to usenet."));
    }
}
if ($pdo->getSetting('tablepergroup') === 1) {
    exit($pdo->log->error("You are using 'tablepergroup', you must use .../misc/update/nix/multiprocessing/releases.php"));
}
$groupName = isset($argv[3]) ? $argv[3] : '';
if (isset($argv[1]) && isset($argv[2])) {
    $consoletools = new ConsoleTools(['ColorCLI' => $pdo->log]);
    $releases = new ProcessReleases(['Settings' => $pdo, 'ConsoleTools' => $consoletools]);
    if ($argv[1] == 1 && $argv[2] == 'true') {
        $releases->processReleases(1, 1, $groupName, $nntp, true);
    } else {
        if ($argv[1] == 1 && $argv[2] == 'false') {
            $releases->processReleases(1, 2, $groupName, $nntp, true);
        } else {
            if ($argv[1] == 2 && $argv[2] == 'true') {
                $releases->processReleases(2, 1, $groupName, $nntp, true);
            } else {
Ejemplo n.º 7
0
     if (($pdo->getSetting('alternate_nntp') == 1 ? $nntp->doConnect(true, true) : $nntp->doConnect()) !== true) {
         exit($pdo->log->error("Unable to connect to usenet."));
     }
     $relID = $pieces[1];
     $guid = $pieces[2];
     $groupID = $pieces[3];
     $nzbcontents = new NZBContents(array('Echo' => true, 'NNTP' => $nntp, 'Settings' => $pdo, 'Nfo' => new Nfo(['Settings' => $pdo, 'Echo' => true]), 'PostProcess' => new PostProcess(['Settings' => $pdo, 'NameFixer' => $namefixer])));
     //echo " " . microtime();
     $res = $nzbcontents->checkPAR2($guid, $relID, $groupID, 1, 1);
     //echo " " . microtime() . " ";
     if ($res === false) {
         echo '.';
     }
 } else {
     if (isset($pieces[1]) && $pieces[0] == 'miscsorter') {
         $nntp = new NNTP(['Settings' => $pdo]);
         if (($pdo->getSetting('alternate_nntp') == 1 ? $nntp->doConnect(true, true) : $nntp->doConnect()) !== true) {
             exit($pdo->log->error("Unable to connect to usenet."));
         }
         $sorter = new MiscSorter(true);
         $relID = $pieces[1];
         $res = $sorter->nfosorter(null, $relID, $nntp);
         if ($res != true) {
             $pdo->queryExec(sprintf('UPDATE releases SET proc_sorter = 1 WHERE id = %d', $relID));
             echo '.';
         }
     } else {
         if (isset($pieces[1]) && $pieces[0] == 'predbft') {
             $pre = $pieces[1];
             if ($res = $pdo->queryOneRow(sprintf('SELECT id AS preid, title, source, searched FROM predb ' . 'WHERE id = %d', $pre))) {
                 $namefixer->done = $namefixer->matched = false;
Ejemplo n.º 8
0
 /**
  * Process sharing.
  *
  * @return bool
  */
 private function processSharing()
 {
     $sharing = $this->pdo->queryOneRow('SELECT enabled FROM sharing');
     if ($sharing !== false && $sharing['enabled'] == 1) {
         $nntp = new NNTP(['Settings' => $this->pdo]);
         if (($this->pdo->getSetting('alternate_nntp') == 1 ? $nntp->doConnect(true, true) : $nntp->doConnect()) === true) {
             (new PostProcess(['Settings' => $this->pdo, 'ColorCLI' => $this->_colorCLI]))->processSharing($nntp);
         }
         return true;
     }
     return false;
 }
Ejemplo n.º 9
0
<?php

// To troubleshoot what's actually on usenet.
require_once realpath(dirname(dirname(dirname(__DIR__))) . DIRECTORY_SEPARATOR . 'indexer.php');
use nzedb\Binaries;
use nzedb\ColorCLI;
use nzedb\NNTP;
$cli = new ColorCLI();
if (!isset($argv[2]) || !is_numeric($argv[2])) {
    exit($cli->error("\nTest your nntp connection, get group information and postdate for specific article.\n\n" . "php {$argv['0']} alt.binaries.teevee 595751142    ...: To test nntp on alt.binaries.teevee with artivle 595751142.\n"));
}
$nntp = new NNTP();
if ($nntp->doConnect() !== true) {
    exit;
}
$first = $argv[2];
$group = $argv[1];
// Select a group.
$groupArr = $nntp->selectGroup($group);
print_r($groupArr);
// Insert actual local part numbers here.
$msg = $nntp->getXOVER($first . '-' . $first);
// Print out the array of headers.
print_r($msg);
// get postdate for an article
$binaries = new Binaries(['NNTP' => $nntp]);
$newdate = $binaries->postdate($first, $groupArr);
echo $cli->primary("The posted date for " . $group . ", article " . $first . " is " . date('Y-m-d H:i:s', $newdate));
Ejemplo n.º 10
0
function daytopost($nntp, $group, $days, $debug = true, $bfcheck = true)
{
    global $pdo;
    $st = false;
    if ($debug && $bfcheck) {
        echo $pdo->log->primary('Finding start and end articles for ' . $group . '.');
    }
    if (!isset($nntp)) {
        $nntp = new NNTP(['Settings' => $pdo]);
        if ($nntp->doConnect(false) !== true) {
            return;
        }
        $st = true;
    }
    $binaries = new Binaries(['NNTP' => $nntp, 'Settings' => $pdo]);
    $data = $nntp->selectGroup($group);
    if ($nntp->isError($data)) {
        $data = $nntp->dataError($nntp, $group, false);
        if ($data === false) {
            return;
        }
    }
    // Goal timestamp.
    $goaldate = date('U') - 86400 * $days;
    $totalnumberofarticles = $data['last'] - $data['first'];
    $upperbound = $data['last'];
    $lowerbound = $data['first'];
    if ($debug && $bfcheck) {
        echo $pdo->log->header('Total Articles: ' . number_format($totalnumberofarticles) . ' Newest: ' . number_format($upperbound) . ' Oldest: ' . number_format($lowerbound));
    }
    if ($data['last'] == PHP_INT_MAX) {
        exit($pdo->log->error("Group data is coming back as php's max value. You should not see this since we use a patched Net_NNTP that fixes this bug."));
    }
    $firstDate = $binaries->postdate($data['first'], $data);
    $lastDate = $binaries->postdate($data['last'], $data);
    if ($goaldate < $firstDate && $bfcheck) {
        if ($st === true) {
            $nntp->doQuit();
        }
        echo $pdo->log->warning("The oldest post indexed from {$days} day(s) ago is older than the first article stored on your news server.\nSetting to First available article of (date('r', {$firstDate}) or daysOld({$firstDate}) days).");
        return $data['first'];
    } else {
        if ($goaldate > $lastDate && $bfcheck) {
            if ($st === true) {
                $nntp->doQuit();
            }
            echo $pdo->log->error("ERROR: The oldest post indexed from {$days} day(s) ago is newer than the last article stored on your news server.\nTo backfill this group you need to set Backfill Days to at least ceil(daysOld({$lastDate})+1) days (date('r', {$lastDate}-86400).");
            return '';
        }
    }
    if ($debug && $bfcheck) {
        echo $pdo->log->primary("Searching for postdates.\nGroup's Firstdate: " . $firstDate . ' (' . (is_int($firstDate) ? date('r', $firstDate) : 'n/a') . ").\nGroup's Lastdate: " . $lastDate . ' (' . date('r', $lastDate) . ").");
    }
    $interval = floor(($upperbound - $lowerbound) * 0.5);
    $templowered = '';
    $dateofnextone = $lastDate;
    // Match on days not timestamp to speed things up.
    while (daysOld($dateofnextone) < $days) {
        while (($tmpDate = $binaries->postdate($upperbound - $interval, $data)) > $goaldate) {
            $upperbound = $upperbound - $interval;
        }
        if (!$templowered) {
            $interval = ceil($interval / 2);
        }
        $dateofnextone = $binaries->postdate($upperbound - 1, $data);
        while (!$dateofnextone) {
            $dateofnextone = $binaries->postdate($upperbound - 1, $data);
        }
    }
    if ($st === true) {
        $nntp->doQuit();
    }
    if ($bfcheck) {
        echo $pdo->log->header("\nBackfill article determined to be " . $upperbound . " " . $pdo->log->setColor('Yellow') . "(" . date('r', $dateofnextone) . ")" . $pdo->log->rsetcolor());
    } else {
        echo $pdo->log->header('Current article determined to be ' . $upperbound . " " . $pdo->log->setColor('Yellow') . "(" . date('r', $dateofnextone) . ")" . $pdo->log->rsetcolor());
    }
    // which is '.daysOld($dateofnextone)." days old.\n";
    return $upperbound;
}
Ejemplo n.º 11
0
use nzedb\NNTP;
use nzedb\db\DB;
$pdo = new DB();
if ($argv[1] === 'all') {
    $groups = $pdo->query('SELECT * FROM groups WHERE backfill = 1');
} else {
    $groups = $pdo->query(sprintf('SELECT * FROM groups WHERE name = %s', $pdo->escapeString($argv[1])));
}
if (count($groups) === 0) {
    if ($argv[1] === 'all') {
        exit('ERROR! No groups were found with backfill enabled!' . PHP_EOL);
    } else {
        exit('ERROR! Group (' . $argv[1] . ') not found!' . PHP_EOL);
    }
}
$nntp = new NNTP(['Settings' => $pdo]);
$nntp->doConnect() or exit('Could not connect to Usenet!' . PHP_EOL);
$binaries = new Binaries(['NNTP' => $nntp, 'Settings' => $pdo]);
foreach ($groups as $group) {
    $groupNNTP = $nntp->selectGroup($group['name']);
    if ($nntp->isError($groupNNTP)) {
        echo 'ERROR! Could not fetch information from NNTP for group (' . $group['name'] . ')' . PHP_EOL;
        continue;
    }
    $postDate = $pdo->queryOneRow(sprintf('SELECT UNIX_TIMESTAMP(postdate) AS postdate FROM releases WHERE group_id = %d ORDER BY postdate ASC LIMIT 1', $group['id']));
    if ($postDate === false) {
        echo 'ERROR! Could not find any existing releases for group (' . $group['name'] . ')' . PHP_EOL;
        continue;
    }
    $articleNumber = (int) $binaries->daytopost(round((time() - $postDate['postdate']) / 86400), $groupNNTP);
    if ($group['last_record'] != 0 && $articleNumber >= $group['last_record']) {
Ejemplo n.º 12
0
<?php

require dirname(__FILE__) . '/config.php';
use nzedb\Backfill;
use nzedb\NNTP;
use nzedb\db\Settings;
$pdo = new Settings();
// Create the connection here and pass
$nntp = new NNTP(['Settings' => $pdo]);
if ($nntp->doConnect() !== true) {
    exit($pdo->log->error("Unable to connect to usenet."));
}
if (isset($argv[1]) && $argv[1] == 'all' && $argv[1] !== 'safe' && $argv[1] !== 'alph' && $argv[1] !== 'date' && !is_numeric($argv[1]) && !isset($argv[2])) {
    $backfill = new Backfill(['NNTP' => $nntp, 'Settings' => $pdo]);
    $backfill->backfillAllGroups();
} else {
    if (isset($argv[1]) && $argv[1] !== 'all' && $argv[1] !== 'safe' && $argv[1] !== 'alph' && $argv[1] !== 'date' && !is_numeric($argv[1]) && !isset($argv[2])) {
        $backfill = new Backfill(['NNTP' => $nntp, 'Settings' => $pdo]);
        $backfill->backfillAllGroups($argv[1]);
    } else {
        if (isset($argv[1]) && $argv[1] !== 'all' && $argv[1] !== 'safe' && $argv[1] !== 'alph' && $argv[1] !== 'date' && !is_numeric($argv[1]) && isset($argv[2]) && is_numeric($argv[2])) {
            $backfill = new Backfill(['NNTP' => $nntp, 'Settings' => $pdo]);
            $backfill->backfillAllGroups($argv[1], $argv[2]);
        } else {
            if (isset($argv[1]) && $argv[1] !== 'all' && $argv[1] !== 'safe' && $argv[1] == 'alph' && $argv[1] !== 'date' && !is_numeric($argv[1]) && isset($argv[2]) && is_numeric($argv[2])) {
                $backfill = new Backfill(['NNTP' => $nntp, 'Settings' => $pdo]);
                $backfill->backfillAllGroups('', $argv[2], 'normal');
            } else {
                if (isset($argv[1]) && $argv[1] !== 'all' && $argv[1] !== 'safe' && $argv[1] !== 'alph' && $argv[1] == 'date' && !is_numeric($argv[1]) && isset($argv[2]) && is_numeric($argv[2])) {
                    $backfill = new Backfill(['NNTP' => $nntp, 'Settings' => $pdo]);
                    $backfill->backfillAllGroups('', $argv[2], 'date');
Ejemplo n.º 13
0
 public function __destruct()
 {
     parent::__destruct();
 }
Ejemplo n.º 14
0
    /**
     * Download and process binaries for JPG pictures.
     *
     * @void
     * @access protected
     */
    protected function _processJPGMessageIDs()
    {
        // Download JPG file.
        if ($this->_foundJPGSample === false && !empty($this->_JPGMessageIDs)) {
            // Try to download it.
            $jpgBinary = $this->_nntp->getMessages($this->_releaseGroupName, $this->_JPGMessageIDs, $this->_alternateNNTP);
            if ($this->_nntp->isError($jpgBinary)) {
                $jpgBinary = false;
            }
            if ($jpgBinary !== false) {
                if ($this->_echoCLI) {
                    $this->_echo('(jB)', 'primaryOver', false);
                }
                // Try to create a file with it.
                @file_put_contents($this->tmpPath . 'samplepicture.jpg', $jpgBinary);
                // Try to resize and move it.
                $this->_foundJPGSample = $this->_releaseImage->saveImage($this->_release['guid'] . '_thumb', $this->tmpPath . 'samplepicture.jpg', $this->_releaseImage->jpgSavePath, 650, 650) === 1 ? true : false;
                if ($this->_foundJPGSample !== false) {
                    // Update the DB to say we got it.
                    $this->pdo->queryExec(sprintf('
							UPDATE releases
							SET jpgstatus = %d
							WHERE id = %d', 1, $this->_release['id']));
                    if ($this->_echoCLI) {
                        $this->_echo('j', 'primaryOver', false);
                    }
                }
                @unlink($this->tmpPath . 'samplepicture.jpg');
            } else {
                if ($this->_echoCLI) {
                    $this->_echo('f', 'warningOver', false);
                }
            }
        }
    }
Ejemplo n.º 15
0
    /**
     * Returns unix time for an article number.
     *
     * @param int    $post      The article number to get the time from.
     * @param array  $groupData Usenet group info from NNTP selectGroup method.
     *
     * @return int	Timestamp.
     */
    public function postdate($post, array $groupData)
    {
        // Set table names
        $groupID = $this->_groups->getIDByName($groupData['group']);
        $group = [];
        if ($groupID !== '') {
            $group = $this->_groups->getCBPTableNames($this->_tablePerGroup, $groupID);
        }
        $currentPost = $post;
        $attempts = $date = 0;
        do {
            // Try to get the article date locally first.
            if ($groupID !== '') {
                // Try to get locally.
                $local = $this->_pdo->queryOneRow(sprintf('
						SELECT c.date AS date
						FROM %s c
						INNER JOIN %s b ON(c.id=b.collection_id)
						INNER JOIN %s p ON(b.id=p.binaryid)
						WHERE p.number = %s
						%s LIMIT 1', $group['cname'], $group['bname'], $group['pname'], $currentPost, $this->_tablePerGroup === false ? sprintf('AND c.group_id = %d', $groupID) : ''));
                if ($local !== false) {
                    $date = $local['date'];
                    break;
                }
            }
            // If we could not find it locally, try usenet.
            $header = $this->_nntp->getXOVER($currentPost);
            if (!$this->_nntp->isError($header)) {
                // Check if the date is set.
                if (isset($header[0]['Date']) && strlen($header[0]['Date']) > 0) {
                    $date = $header[0]['Date'];
                    break;
                }
            }
            // Try to get a different article number.
            if (abs($currentPost - $groupData['first']) > abs($groupData['last'] - $currentPost)) {
                $tempPost = round($currentPost / (mt_rand(1005, 1012) / 1000), 0, PHP_ROUND_HALF_UP);
                if ($tempPost < $groupData['first']) {
                    $tempPost = $groupData['first'];
                }
            } else {
                $tempPost = round(mt_rand(1005, 1012) / 1000 * $currentPost, 0, PHP_ROUND_HALF_UP);
                if ($tempPost > $groupData['last']) {
                    $tempPost = $groupData['last'];
                }
            }
            // If we got the same article number as last time, give up.
            if ($tempPost === $currentPost) {
                break;
            }
            $currentPost = $tempPost;
            if ($this->_debug) {
                $this->_colorCLI->doEcho($this->_colorCLI->debug('Postdate retried ' . $attempts . " time(s)."));
            }
        } while ($attempts++ <= 20);
        // If we didn't get a date, set it to now.
        if (!$date) {
            $date = time();
        } else {
            $date = strtotime($date);
        }
        if ($this->_debug) {
            $this->_debugging->log(get_class(), __FUNCTION__, 'Article (' . $post . "'s) date is (" . $date . ') (' . $this->daysOld($date) . " days old)", Logger::LOG_INFO);
        }
        return $date;
    }
Ejemplo n.º 16
0
 /**
  * Update the list of newsgroups and return an array of messages.
  *
  * @param string $groupList
  * @param int    $active
  * @param int    $backfill
  *
  * @return array
  */
 public function addBulk($groupList, $active = 1, $backfill = 1)
 {
     if (preg_match('/^\\s*$/m', $groupList)) {
         $ret = "No group list provided.";
     } else {
         $nntp = new NNTP(['Echo' => false]);
         if ($nntp->doConnect() !== true) {
             return 'Problem connecting to usenet.';
         }
         $groups = $nntp->getGroups();
         $nntp->doQuit();
         if ($nntp->isError($groups)) {
             return 'Problem fetching groups from usenet.';
         }
         $regFilter = '/' . $groupList . '/i';
         $ret = [];
         foreach ($groups as $group) {
             if (preg_match($regFilter, $group['group']) > 0) {
                 $res = $this->pdo->queryOneRow(sprintf('SELECT id FROM groups WHERE name = %s', $this->pdo->escapeString($group['group'])));
                 if ($res === false) {
                     $this->pdo->queryInsert(sprintf('INSERT INTO groups (name, active, backfill) VALUES (%s, %d, %d)', $this->pdo->escapeString($group['group']), $active, $backfill));
                     $ret[] = ['group' => $group['group'], 'msg' => 'Created'];
                 }
             }
         }
         if (count($ret) === 0) {
             $ret = 'No groups found with your regex, try again!';
         }
     }
     return $ret;
 }
Ejemplo n.º 17
0
 /**
  * Restart the NNTP connection if an error occurs in the selectGroup
  * function, if it does not restart display the error.
  *
  * @param NNTP   $nntp  Instance of class NNTP.
  * @param string $group Name of the group.
  * @param bool   $comp Use compression or not?
  *
  * @return mixed On success : (array)  The group summary.
  *               On Failure : (object) PEAR_Error.
  *
  * @access public
  */
 public function dataError($nntp, $group, $comp = true)
 {
     // Disconnect.
     $nntp->doQuit();
     // Try reconnecting. This uses another round of max retries.
     if ($nntp->doConnect($comp) !== true) {
         if ($this->_debugBool) {
             $this->_debugging->log(get_class(), __FUNCTION__, 'Unable to reconnect to usenet!', Logger::LOG_NOTICE);
         }
         return $this->throwError('Unable to reconnect to usenet!');
     }
     // Try re-selecting the group.
     $data = $nntp->selectGroup($group);
     if ($this->isError($data)) {
         $message = "Code {$data->code}: {$data->message}\nSkipping group: {$group}";
         if ($this->_debugBool) {
             $this->_debugging->log(get_class(), __FUNCTION__, $message, Logger::LOG_NOTICE);
         }
         if ($this->_echo) {
             $this->pdo->log->doEcho($this->pdo->log->error($message), true);
         }
         $nntp->doQuit();
     }
     return $data;
 }
Ejemplo n.º 18
0
 $other = 1;
 if ($argv[3] === 'all') {
     $other = 2;
 } else {
     if ($argv[3] === 'preid') {
         $other = 3;
     }
 }
 $setStatus = $argv[4] == "yes" ? 1 : 2;
 $show = 2;
 if (isset($argv[5]) && $argv[5] === 'show') {
     $show = 1;
 }
 $nntp = null;
 if ($argv[1] == 7 || $argv[1] == 8) {
     $nntp = new NNTP(['Settings' => $pdo]);
     if (($pdo->getSetting('alternate_nntp') == '1' ? $nntp->doConnect(true, true) : $nntp->doConnect()) !== true) {
         echo $pdo->log->error("Unable to connect to usenet.\n");
         return;
     }
 }
 switch ($argv[1]) {
     case 1:
         $predb->parseTitles(1, $update, $other, $setStatus, $show);
         break;
     case 2:
         $predb->parseTitles(2, $update, $other, $setStatus, $show);
         break;
     case 3:
         $namefixer->fixNamesWithNfo(1, $update, $other, $setStatus, $show);
         break;
Ejemplo n.º 19
0
    /**
     * Attempt to get a better name from a par2 file and categorize the release.
     *
     * @note Called from NZBContents.php
     *
     * @param string $messageID MessageID from NZB file.
     * @param int    $relID     ID of the release.
     * @param int    $groupID   Group ID of the release.
     * @param \nzedb\NNTP   $nntp      Class NNTP
     * @param int    $show      Only show result or apply iy.
     *
     * @return bool
     */
    public function parsePAR2($messageID, $relID, $groupID, &$nntp, $show)
    {
        if ($messageID === '') {
            return false;
        }
        $query = $this->pdo->queryOneRow(sprintf('
				SELECT id, group_id, categoryid, name, searchname, UNIX_TIMESTAMP(postdate) AS post_date, id AS releaseid
				FROM releases
				WHERE isrenamed = 0
				AND id = %d', $relID));
        if ($query === false) {
            return false;
        }
        // Only get a new name if the category is OTHER.
        $foundName = true;
        if (!in_array((int) $query['categoryid'], [Category::CAT_BOOKS_OTHER, Category::CAT_GAME_OTHER, Category::CAT_MOVIE_OTHER, Category::CAT_MUSIC_OTHER, Category::CAT_PC_PHONE_OTHER, Category::CAT_TV_OTHER, Category::CAT_OTHER_HASHED, Category::CAT_XXX_OTHER, Category::CAT_MISC])) {
            $foundName = false;
        }
        // Get the PAR2 file.
        $par2 = $nntp->getMessages($this->groups->getByNameByID($groupID), $messageID, $this->alternateNNTP);
        if ($nntp->isError($par2)) {
            return false;
        }
        // Put the PAR2 into Par2Info, check if there's an error.
        $this->_par2Info->setData($par2);
        if ($this->_par2Info->error) {
            return false;
        }
        // Get the file list from Par2Info.
        $files = $this->_par2Info->getFileList();
        if ($files !== false && count($files) > 0) {
            $filesAdded = 0;
            // Loop through the files.
            foreach ($files as $file) {
                if (!isset($file['name'])) {
                    continue;
                }
                // If we found a name and added 10 files, stop.
                if ($foundName === true && $filesAdded > 10) {
                    break;
                }
                if ($this->addpar2) {
                    // Add to release files.
                    if ($filesAdded < 11 && $this->pdo->queryOneRow(sprintf('
								SELECT id
								FROM release_files
								WHERE releaseid = %d
								AND name = %s', $relID, $this->pdo->escapeString($file['name']))) === false) {
                        // Try to add the files to the DB.
                        if ($this->releaseFiles->add($relID, $file['name'], $file['size'], $query['post_date'], 0)) {
                            $filesAdded++;
                        }
                    }
                } else {
                    $filesAdded++;
                }
                // Try to get a new name.
                if ($foundName === false) {
                    $query['textstring'] = $file['name'];
                    if ($this->nameFixer->checkName($query, 1, 'PAR2, ', 1, $show) === true) {
                        $foundName = true;
                    }
                }
            }
            // If we found some files.
            if ($filesAdded > 0) {
                $this->debugging->log('PostProcess', 'parsePAR2', 'Added ' . $filesAdded . ' release_files from PAR2 for ' . $query['searchname'], Logger::LOG_INFO);
                // Update the file count with the new file count + old file count.
                $this->pdo->queryExec(sprintf('
						UPDATE releases
						SET rarinnerfilecount = rarinnerfilecount + %d
						WHERE id = %d', $filesAdded, $relID));
            }
            if ($foundName === true) {
                return true;
            }
        }
        return false;
    }
Ejemplo n.º 20
0
                        }
                    }
                }
                break;
            case $pieces[0] === 'par2' && isset($guidChar) && isset($maxperrun) && is_numeric($maxperrun):
                $releases = $pdo->queryDirect(sprintf('
								SELECT r.id AS releaseid, r.guid, r.group_id
								FROM releases r
								WHERE r.guid %s
								AND r.nzbstatus = 1
								AND r.proc_par2 = 0
								AND r.preid = 0
								ORDER BY r.postdate ASC
								LIMIT %s', $pdo->likeString($guidChar, false, true), $maxperrun));
                if ($releases instanceof \Traversable) {
                    $nntp = new NNTP(['Settings' => $pdo]);
                    if (($pdo->getSetting('alternate_nntp') == '1' ? $nntp->doConnect(true, true) : $nntp->doConnect()) !== true) {
                        exit($pdo->log->error("Unable to connect to usenet."));
                    }
                    $Nfo = new Nfo(['Settings' => $pdo, 'Echo' => true]);
                    $nzbcontents = new NZBContents(['Echo' => true, 'NNTP' => $nntp, 'Nfo' => $Nfo, 'Settings' => $pdo, 'PostProcess' => new PostProcess(['Settings' => $pdo, 'Nfo' => $Nfo, 'NameFixer' => $namefixer])]);
                    foreach ($releases as $release) {
                        $res = $nzbcontents->checkPAR2($release['guid'], $release['releaseid'], $release['group_id'], 1, 1);
                        if ($res === false) {
                            echo '.';
                        }
                    }
                }
                break;
            case $pieces[0] === 'miscsorter' && isset($guidChar) && isset($maxperrun) && is_numeric($maxperrun):
                $releases = $pdo->queryDirect(sprintf('
Ejemplo n.º 21
0
<?php

require_once realpath(dirname(dirname(dirname(dirname(dirname(__DIR__))))) . DIRECTORY_SEPARATOR . 'indexer.php');
use nzedb\ConsoleTools;
use nzedb\NNTP;
use nzedb\db\Settings;
$start = TIME();
$pdo = new Settings();
$consoleTools = new ConsoleTools(['ColorCLI' => $pdo->log]);
// Create the connection here and pass
$nntp = new NNTP(['Settings' => $pdo]);
if ($nntp->doConnect() !== true) {
    exit($pdo->log->error("Unable to connect to usenet."));
}
echo $pdo->log->header("Getting first/last for all your active groups.");
$data = $nntp->getGroups();
if ($nntp->isError($data)) {
    exit($pdo->log->error("Failed to getGroups() from nntp server."));
}
echo $pdo->log->header("Inserting new values into short_groups table.");
$pdo->queryExec('TRUNCATE TABLE short_groups');
// Put into an array all active groups
$res = $pdo->query('SELECT name FROM groups WHERE active = 1 OR backfill = 1');
foreach ($data as $newgroup) {
    if (myInArray($res, $newgroup['group'], 'name')) {
        $pdo->queryInsert(sprintf('INSERT INTO short_groups (name, first_record, last_record, updated) VALUES (%s, %s, %s, NOW())', $pdo->escapeString($newgroup['group']), $pdo->escapeString($newgroup['first']), $pdo->escapeString($newgroup['last'])));
        echo $pdo->log->primary('Updated ' . $newgroup['group']);
    }
}
echo $pdo->log->header('Running time: ' . $consoleTools->convertTimer(TIME() - $start));
function myInArray($array, $value, $key)