/**
* @param int|string $groupID (optional)
*/
public function applyRegex($groupID)
{
//
// Get all regexes for all groups which are to be applied to new binaries
// in order of how they should be applied
//
$group = $this->groups->getCBPTableNames($this->tablePerGroup, $groupID);
$activeGroups = $this->groups->getActive();
$this->releaseRegex->get();
$this->pdo->log->doEcho($this->pdo->log->primary('Applying regex to binaries'), true);
foreach ($activeGroups as $groupArr) {
//check if regexes have already been applied during update binaries
if ($groupArr['regexmatchonly'] == 1) {
continue;
}
$groupRegexes = $this->releaseRegex->getForGroup($groupArr['name']);
$this->pdo->log->doEcho($this->pdo->log->primary('Applying ' . sizeof($groupRegexes) . ' regexes to group ' . $groupArr['name']), true);
// Get out all binaries of STAGE0 for current group
$newUnmatchedBinaries = array();
$ressql = sprintf('SELECT id, name, date, totalparts, procstat, fromname FROM %s b
WHERE groupid = %d AND procstat IN (%d, %d) AND regexid IS NULL ORDER BY b.date ASC', $group['bname'], $groupArr['id'], Releases::PROCSTAT_NEW, Releases::PROCSTAT_TITLENOTMATCHED);
$resbin = $this->pdo->queryDirect($ressql);
$matchedbins = 0;
while ($rowbin = $this->pdo->getAssocArray($resbin)) {
$regexMatches = array();
foreach ($groupRegexes as $groupRegex) {
$regexCheck = $this->releaseRegex->performMatch($groupRegex, $rowbin['name']);
if ($regexCheck !== false) {
$regexMatches = $regexCheck;
break;
}
}
if (!empty($regexMatches)) {
$matchedbins++;
$relparts = explode("/", $regexMatches['parts']);
$this->pdo->queryExec(sprintf("UPDATE %s SET relname = replace(%s, '_', ' '), relpart = %d, reltotalpart = %d, procstat=%d, categoryid=%s, regexid=%d, reqid=%s WHERE id = %d", $group['bname'], $this->pdo->escapeString($regexMatches['name']), $relparts[0], $relparts[1], Releases::PROCSTAT_TITLEMATCHED, $regexMatches['regcatid'], $regexMatches['regexid'], $this->pdo->escapeString($regexMatches['reqid']), $rowbin["id"]));
} else {
if ($rowbin['procstat'] == Releases::PROCSTAT_NEW) {
$newUnmatchedBinaries[] = $rowbin['id'];
}
}
}
//mark as not matched
if (!empty($newUnmatchedBinaries)) {
$this->pdo->queryExec(sprintf("UPDATE %s SET procstat=%d WHERE id IN (%s)", $group['bname'], Releases::PROCSTAT_TITLENOTMATCHED, implode(',', $newUnmatchedBinaries)));
}
}
}
/**
* Returns unix time for an article number.
*
* @param int $post The article number to get the time from.
* @param array $groupData Usenet group info from NNTP selectGroup method.
*
* @return bool|int
*/
public function postdate($post, array $groupData)
{
// Set table names
$groupID = $this->_groups->getIDByName($groupData['group']);
$group = [];
if ($groupID !== '') {
$group = $this->_groups->getCBPTableNames($this->_tablePerGroup, $groupID);
}
$currentPost = $post;
$attempts = $date = 0;
do {
// Try to get the article date locally first.
if ($groupID !== '') {
// Try to get locally.
$local = $this->_pdo->queryOneRow(sprintf('
SELECT b.date AS date
FROM %s b, %s p
WHERE b.id = p.binaryid
AND b.groupid = %s
AND p.number = %s LIMIT 1', $group['bname'], $group['pname'], $groupID, $currentPost));
if ($local !== false) {
$date = $local['date'];
break;
}
}
// If we could not find it locally, try usenet.
$header = $this->_nntp->getXOVER($currentPost);
if (!$this->_nntp->isError($header)) {
// Check if the date is set.
if (isset($header[0]['Date']) && strlen($header[0]['Date']) > 0) {
$date = $header[0]['Date'];
break;
}
}
// Try to get a different article number.
if (abs($currentPost - $groupData['first']) > abs($groupData['last'] - $currentPost)) {
$tempPost = round($currentPost / (mt_rand(1005, 1012) / 1000), 0, PHP_ROUND_HALF_UP);
if ($tempPost < $groupData['first']) {
$tempPost = $groupData['first'];
}
} else {
$tempPost = round(mt_rand(1005, 1012) / 1000 * $currentPost, 0, PHP_ROUND_HALF_UP);
if ($tempPost > $groupData['last']) {
$tempPost = $groupData['last'];
}
}
// If we got the same article number as last time, give up.
if ($tempPost === $currentPost) {
break;
}
$currentPost = $tempPost;
if ($this->_debug) {
$this->_colorCLI->doEcho($this->_colorCLI->debug('Postdate retried ' . $attempts . " time(s)."));
}
} while ($attempts++ <= 20);
// If we didn't get a date, set it to now.
if (!$date) {
$date = time();
} else {
$date = strtotime($date);
}
if ($this->_debug) {
$this->_debugging->log('Binaries', "postdate", 'Article (' . $post . "'s) date is (" . $date . ') (' . $this->daysOld($date) . " days old)", \Logger::LOG_INFO);
}
return $date;
}
public function testRegex($regex, $groupname, $poster, $ignorematched, $matchagainstbins)
{
$cat = new Categorize();
$groups = new Groups();
$groupID = $groups->getByNameByID($groupname);
$group = $groups->getCBPTableNames($this->tablePerGroup, $groupID);
$catList = $cat->getForSelect();
$matches = [];
if ($groupname === 0) {
$groupname = '.*';
}
if ($matchagainstbins !== '') {
$sql = sprintf("select b.*, '0' as size, '0' as blacklistID, g.name as groupname from %s b left join groups g on g.id = b.groupid where b.groupid IN (select g.id from groups g where g.name REGEXP %s) order by b.date desc", $group['bname'], $this->pdo->escapeString('^' . $groupname . '$'));
} else {
$sql = sprintf("select rrt.* from releaseregextesting rrt where rrt.groupname REGEXP %s order by rrt.date desc", $this->pdo->escapeString('^' . $groupname . '$'));
}
$resbin = $this->pdo->queryDirect($sql);
while ($rowbin = $this->pdo->getAssocArray($resbin)) {
if ($ignorematched !== '' && ($rowbin['regexid'] != '' || $rowbin['blacklistid'] == 1)) {
continue;
}
$regexarr = array("id" => "", 'regex' => $regex, 'poster' => $poster, "categoryid" => "");
$regexCheck = $this->performMatch($regexarr, $rowbin['name'], $rowbin['fromname']);
if ($regexCheck !== false) {
$relname = $regexCheck['name'];
$relparts = explode("/", $regexCheck['parts']);
$matches[$relname]['name'] = $relname;
$matches[$relname]['parts'] = $regexCheck['parts'];
$matches[$relname]['bincount'] = isset($matches[$relname]['bincount']) ? $matches[$relname]['bincount'] + 1 : 1;
$matches[$relname]['bininfo'][] = $rowbin;
$matches[$relname]['binsize'][] = $rowbin['size'];
$matches[$relname]['totalsize'] = array_sum($matches[$relname]['binsize']);
$matches[$relname]['relparts'][$relparts[1]] = $relparts[1];
$matches[$relname]['reltotalparts'] = array_sum($matches[$relname]['relparts']);
$matches[$relname]['regexid'] = $regexCheck['regexid'];
if (ctype_digit($regexCheck['regcatid'])) {
$matches[$relname]['catname'] = $catList[$regexCheck['regcatid']];
} else {
$matches[$relname]['catname'] = $catList[$cat->determineCategory($groupname, $relname)];
}
}
}
//echo '<pre>';
//print_r(array_pop($matches));
//echo '</pre>';
return $matches;
}
/**
* Delete collections (complete/incomplete/old/etc).
*
* @param int|string $groupID (optional)
*
* @void
* @access public
*/
public function deleteCollections($groupID)
{
$startTime = time();
$group = $this->groups->getCBPTableNames($this->tablePerGroup, $groupID);
$deletedCount = 0;
if ($this->echoCLI) {
echo $this->pdo->log->header("Process Releases -> Delete finished collections." . PHP_EOL) . $this->pdo->log->primary('Deleting old collections/binaries/parts.');
}
$deleted = 0;
// CBP older than retention.
$deleteQuery = $this->pdo->queryExec(sprintf('DELETE FROM %s WHERE dateadded < (NOW() - INTERVAL %d HOUR) %s', $group['cname'], $this->pdo->getSetting('partretentionhours'), !empty($groupID) && $this->tablePerGroup === false ? ' AND group_id = ' . $groupID : ''));
if ($deleteQuery !== false) {
$deleted = $deleteQuery->rowCount();
$deletedCount += $deleted;
}
$firstQuery = time();
if ($this->echoCLI) {
echo $this->pdo->log->primary('Finished deleting ' . $deleted . ' old collections/binaries/parts in ' . ($firstQuery - $startTime) . ' seconds.' . PHP_EOL . 'Deleting binaries/parts with no collections.');
}
$deleted = 0;
// Binaries/parts that somehow have no collection.
$deleteQuery = $this->pdo->queryExec(sprintf('DELETE %s, %s FROM %s, %s WHERE %s.collectionid = 0 AND %s.id = %s.binaryid', $group['bname'], $group['pname'], $group['bname'], $group['pname'], $group['bname'], $group['bname'], $group['pname']));
if ($deleteQuery !== false) {
$deleted = $deleteQuery->rowCount();
$deletedCount += $deleted;
}
$secondQuery = time();
if ($this->echoCLI) {
echo $this->pdo->log->primary('Finished deleting ' . $deleted . ' binaries/parts with no collections in ' . ($secondQuery - $firstQuery) . ' seconds.' . PHP_EOL . 'Deleting parts with no binaries.');
}
$deleted = 0;
// Parts that somehow have no binaries. Don't delete parts currently inserting, by checking the max ID.
if (mt_rand(0, 100) <= 5) {
$deleteQuery = $this->pdo->queryExec(sprintf('DELETE FROM %s WHERE binaryid NOT IN (SELECT id FROM %s) %s', $group['pname'], $group['bname'], $this->minMaxQueryFormulator($group['pname'], 40000)));
if ($deleteQuery !== false) {
$deleted = $deleteQuery->rowCount();
$deletedCount += $deleted;
}
}
$thirdQuery = time();
if ($this->echoCLI) {
echo $this->pdo->log->primary('Finished deleting ' . $deleted . ' parts with no binaries in ' . ($thirdQuery - $secondQuery) . ' seconds.' . PHP_EOL . 'Deleting binaries with no collections.');
}
$deleted = 0;
// Binaries that somehow have no collection. Don't delete currently inserting binaries by checking the max id.
$deleteQuery = $this->pdo->queryExec(sprintf('DELETE FROM %s WHERE collectionid NOT IN (SELECT id FROM %s) %s', $group['bname'], $group['cname'], $this->minMaxQueryFormulator($group['bname'], 20000)));
if ($deleteQuery !== false) {
$deleted = $deleteQuery->rowCount();
$deletedCount += $deleted;
}
$fourthQuery = time();
if ($this->echoCLI) {
echo $this->pdo->log->primary('Finished deleting ' . $deleted . ' binaries with no collections in ' . ($fourthQuery - $thirdQuery) . ' seconds.' . PHP_EOL . 'Deleting collections with no binaries.');
}
$deleted = 0;
// Collections that somehow have no binaries.
$collectionIDs = $this->pdo->queryDirect(sprintf('SELECT id FROM %s WHERE id NOT IN (SELECT collectionid FROM %s) %s', $group['cname'], $group['bname'], $this->minMaxQueryFormulator($group['cname'], 10000)));
if ($collectionIDs instanceof \Traversable) {
foreach ($collectionIDs as $collectionID) {
$deleted++;
$this->pdo->queryExec(sprintf('DELETE FROM %s WHERE id = %d', $group['cname'], $collectionID['id']));
}
$deletedCount += $deleted;
}
$fifthQuery = time();
if ($this->echoCLI) {
echo $this->pdo->log->primary('Finished deleting ' . $deleted . ' collections with no binaries in ' . ($fifthQuery - $fourthQuery) . ' seconds.' . PHP_EOL . 'Deleting collections that were missed after NZB creation.');
}
$deleted = 0;
// Collections that were missing on NZB creation.
$collections = $this->pdo->queryDirect(sprintf('
SELECT c.id
FROM %s c
INNER JOIN releases r ON r.id = c.releaseid
WHERE r.nzbstatus = 1', $group['cname']));
if ($collections instanceof \Traversable) {
foreach ($collections as $collection) {
$deleted++;
$this->pdo->queryExec(sprintf('
DELETE FROM %s WHERE id = %d', $group['cname'], $collection['id']));
}
$deletedCount += $deleted;
}
if ($this->echoCLI) {
$this->pdo->log->doEcho($this->pdo->log->primary('Finished deleting ' . $deleted . ' collections missed after NZB creation in ' . (time() - $fifthQuery) . ' seconds.' . PHP_EOL . 'Removed ' . number_format($deletedCount) . ' parts/binaries/collection rows in ' . $this->consoleTools->convertTime($fifthQuery - $startTime) . PHP_EOL));
}
}
