/** * @param string $message * @param bool $reset */ public function overWritePrimary($message, $reset = False) { if ($reset) { $this->lastMessageLength = 0; } echo str_repeat(chr(8), $this->lastMessageLength); echo str_repeat(" ", $this->lastMessageLength); echo str_repeat(chr(8), $this->lastMessageLength); $this->lastMessageLength = strlen($message); echo $this->cli->primaryOver($message); }
/** * Download a range of usenet messages. Store binaries with subjects matching a * specific pattern in the database. * * @param $groupArr * @param $first * @param $last * @param string $type * * @return array */ function scan($groupArr, $first, $last, $type = 'update') { $db = new Settings(); $releaseRegex = new ReleaseRegex(); $n = $this->n; // Check if MySQL tables exist, create if they do not, get their names at the same time. $tableNames = $this->_groups->getCBPTableNames($this->_tablePerGroup, $groupArr['id']); $partRepair = $type === 'partrepair'; $returnArray = []; // Download the headers. if ($partRepair === true) { // This is slower but possibly is better with missing headers. $msgs = $this->_nntp->getOverview($first . '-' . $last, true, false); } else { $msgs = $this->_nntp->getXOVER($first . '-' . $last); } // If there was an error, try to reconnect. if ($this->_nntp->isError($msgs)) { // Increment if part repair and return false. if ($partRepair === true) { $this->_pdo->queryExec(sprintf('UPDATE %s SET attempts = attempts + 1 WHERE groupid = %d AND numberid %s', $tableNames['prname'], $groupArr['id'], $first == $last ? '= ' . $first : 'IN (' . implode(',', range($first, $last)) . ')')); return $returnArray; } // This is usually a compression error, so try disabling compression. $this->_nntp->doQuit(); if ($this->_nntp->doConnect(false) !== true) { return $returnArray; } // Re-select group, download headers again without compression and re-enable compression. $this->_nntp->selectGroup($groupArr['name']); $msgs = $this->_nntp->getXOVER($first . '-' . $last); $this->_nntp->enableCompression(); // Check if the non-compression headers have an error. if ($this->_nntp->isError($msgs)) { $this->log("Code {$msgs->code}: {$msgs->message}\nSkipping group: {$groupArr['name']}", 'scan', \Logger::LOG_WARNING, 'error'); return $returnArray; } } $rangerequested = range($first, $last); $msgsreceived = array(); $msgsblacklisted = array(); $msgsignored = array(); $msgsinserted = array(); $msgsnotinserted = array(); $timeHeaders = number_format(microtime(true) - $this->startHeaders, 2); if ($this->_nntp->isError($msgs)) { echo "Error {$msgs->code}: {$msgs->message}{$n}"; echo "Skipping group{$n}"; return $returnArray; } // Check if we got headers. $msgCount = count($msgs); if ($msgCount < 1) { return $returnArray; } // Get highest and lowest article numbers/dates. $iterator1 = 0; $iterator2 = $msgCount - 1; while (true) { if (!isset($returnArray['firstArticleNumber']) && isset($msgs[$iterator1]['Number'])) { $returnArray['firstArticleNumber'] = $msgs[$iterator1]['Number']; $returnArray['firstArticleDate'] = $msgs[$iterator1]['Date']; } if (!isset($returnArray['lastArticleNumber']) && isset($msgs[$iterator2]['Number'])) { $returnArray['lastArticleNumber'] = $msgs[$iterator2]['Number']; $returnArray['lastArticleDate'] = $msgs[$iterator2]['Date']; } // Break if we found non empty articles. if (isset($returnArray['firstArticleNumber']) && isset($returnArray['lastArticleNumber'])) { break; } // Break out if we couldn't find anything. if ($iterator1++ >= $msgCount - 1 || $iterator2-- <= 0) { break; } } if (is_array($msgs)) { //loop headers, figure out parts foreach ($msgs as $msg) { if (!isset($msg['Number'])) { continue; } $msgsreceived[] = $msg['Number']; $msgPart = $msgTotalParts = 0; $pattern = '|\\((\\d+)[\\/](\\d+)\\)|i'; preg_match_all($pattern, $msg['Subject'], $matches, PREG_PATTERN_ORDER); $matchcnt = sizeof($matches[0]); for ($i = 0; $i < $matchcnt; $i++) { $msgPart = $matches[1][$i]; $msgTotalParts = $matches[2][$i]; } if (!isset($msg['Subject']) || $matchcnt == 0) { $msgsignored[] = $msg['Number']; continue; } if ((int) $msgPart > 0 && (int) $msgTotalParts > 0) { $subject = utf8_encode(trim(preg_replace('|\\(' . $msgPart . '[\\/]' . $msgTotalParts . '\\)|i', '', $msg['Subject']))); if (!isset($this->message[$subject])) { $this->message[$subject] = $msg; $this->message[$subject]['MaxParts'] = (int) $msgTotalParts; $this->message[$subject]['Date'] = strtotime($this->message[$subject]['Date']); } if ((int) $msgPart > 0) { $this->message[$subject]['Parts'][(int) $msgPart] = array('Message-ID' => substr($msg['Message-ID'], 1, -1), 'number' => $msg['Number'], 'part' => (int) $msgPart, 'size' => $msg['Bytes']); $this->message[$subject]['PartNumbers'][(int) $msgPart] = $msg['Number']; } } } unset($msg); unset($msgs); $count = 0; $updatecount = 0; $partcount = 0; $rangenotreceived = array_diff($rangerequested, $msgsreceived); if ($type != 'partrepair') { echo "Received " . sizeof($msgsreceived) . " articles of " . ($last - $first + 1) . " requested, " . sizeof($msgsignored) . " not binaries {$n}"; } if ($type == 'update' && sizeof($msgsreceived) == 0) { echo "Error: Server did not return any articles.{$n}"; echo "Skipping group{$n}"; return $returnArray; } if (sizeof($rangenotreceived) > 0) { switch ($type) { case 'backfill': //don't add missing articles break; case 'partrepair': case 'update': default: $this->addMissingParts($rangenotreceived, $tableNames['prname'], $groupArr['id']); break; } echo "Server did not return " . count($rangenotreceived) . " article(s).{$n}"; } if (isset($this->message) && count($this->message)) { $groupRegexes = $releaseRegex->getForGroup($groupArr['name']); //insert binaries and parts into database. when binary already exists; only insert new parts foreach ($this->message as $subject => $data) { //Filter binaries based on black/white list if ($this->isBlackListed($data, $groupArr['name'])) { $msgsblacklisted[] = count($data['Parts']); if ($type == 'partrepair') { $partIds = array(); foreach ($data['Parts'] as $partdata) { $partIds[] = $partdata['number']; } $db->queryExec(sprintf("DELETE FROM %s WHERE numberid IN (%s) AND groupid=%d", $tableNames['prname'], implode(',', $partIds), $groupArr['id'])); } continue; } if (isset($data['Parts']) && count($data['Parts']) > 0 && $subject != '') { //Check for existing binary $binaryID = 0; $binaryHash = md5($subject . $data['From'] . $groupArr['id']); $res = $db->queryOneRow(sprintf("SELECT id FROM %s WHERE binaryhash = %s", $tableNames['bname'], $db->escapeString($binaryHash))); if (!$res) { //Apply Regexes $regexMatches = array(); foreach ($groupRegexes as $groupRegex) { $regexCheck = $releaseRegex->performMatch($groupRegex, $subject); if ($regexCheck !== false) { $regexMatches = $regexCheck; break; } } $sql = ''; if (!empty($regexMatches)) { $relparts = explode("/", $regexMatches['parts']); $sql = sprintf('INSERT INTO %s (name, fromname, date, xref, totalparts, groupid, procstat, categoryid, regexid, reqid, relpart, reltotalpart, binaryhash, relname, dateadded) VALUES (%s, %s, FROM_UNIXTIME(%s), %s, %s, %d, %d, %s, %d, %s, %d, %d, %s, %s, now())', $tableNames['bname'], $db->escapeString($subject), $db->escapeString(utf8_encode($data['From'])), $db->escapeString($data['Date']), $db->escapeString($data['Xref']), $db->escapeString($data['MaxParts']), $groupArr['id'], Releases::PROCSTAT_TITLEMATCHED, $regexMatches['regcatid'], $regexMatches['regexid'], $db->escapeString($regexMatches['reqid']), $relparts[0], $relparts[1], $db->escapeString($binaryHash), $db->escapeString(str_replace('_', ' ', $regexMatches['name']))); } elseif ($this->onlyProcessRegexBinaries === false) { $sql = sprintf('INSERT INTO %s (name, fromname, date, xref, totalparts, groupid, binaryhash, dateadded) VALUES (%s, %s, FROM_UNIXTIME(%s), %s, %s, %d, %s, now())', $tableNames['bname'], $db->escapeString($subject), $db->escapeString(utf8_encode($data['From'])), $db->escapeString($data['Date']), $db->escapeString($data['Xref']), $db->escapeString($data['MaxParts']), $groupArr['id'], $db->escapeString($binaryHash)); } elseif ($type == 'partrepair') { $partIds = array(); foreach ($data['Parts'] as $partdata) { $partIds[] = $partdata['number']; } $db->queryExec(sprintf('DELETE FROM %s WHERE numberid IN (%s) AND groupid = %d', $tableNames['prname'], implode(',', $partIds), $groupArr['id'])); continue; } if ($sql != '') { $binaryID = $db->queryInsert($sql); $count++; //if ($count % 500 == 0) echo "$count bin adds..."; } } else { $binaryID = $res["id"]; $updatecount++; //if ($updatecount % 500 == 0) echo "$updatecount bin updates..."; } if ($binaryID != 0) { $partParams = array(); $partNumbers = array(); foreach ($data['Parts'] as $partdata) { $partcount++; $partParams[] = sprintf('(%d, %s, %s, %s, %s)', $binaryID, $db->escapeString($partdata['Message-ID']), $db->escapeString($partdata['number']), $db->escapeString(round($partdata['part'])), $db->escapeString($partdata['size'])); $partNumbers[] = $partdata['number']; } $partSql = 'INSERT INTO ' . $tableNames['pname'] . ' (binaryid, messageid, number, partnumber, size) VALUES ' . implode(', ', $partParams); $pidata = $db->queryInsert($partSql); if (!$pidata) { $msgsnotinserted = array_merge($msgsnotinserted, $partNumbers); } else { $msgsinserted = array_merge($msgsinserted, $partNumbers); } } } } if (!empty($this->_binaryBlacklistIdsToUpdate)) { $this->_pdo->queryExec(sprintf('UPDATE binaryblacklist SET last_activity = NOW() WHERE id IN (%s)', implode(',', $this->_binaryBlacklistIdsToUpdate))); $this->_binaryBlacklistIdsToUpdate = []; } //TODO: determine whether to add to missing articles if insert failed if (sizeof($msgsnotinserted) > 0) { echo 'WARNING: ' . count($msgsnotinserted) . ' Parts failed to insert' . $n; $this->addMissingParts($msgsnotinserted, $tableNames['prname'], $groupArr['id']); } if ($count >= 500 || $updatecount >= 500) { echo $n; } //line break for bin adds output } $timeUpdate = number_format(microtime(true) - $this->startUpdate, 2); $timeLoop = number_format(microtime(true) - $this->startLoop, 2); if (sizeof($msgsblacklisted) > 0) { echo "Blacklisted " . array_sum($msgsblacklisted) . " parts in " . sizeof($msgsblacklisted) . " binaries" . $n; } if ($type != 'partrepair') { if ($this->_echoCLI) { $this->_colorCLI->doEcho($this->_colorCLI->alternateOver(number_format($count)) . $this->_colorCLI->primaryOver(' new, ') . $this->_colorCLI->alternateOver(number_format($updatecount)) . $this->_colorCLI->primaryOver(' updated, ') . $this->_colorCLI->alternateOver(number_format($partcount)) . $this->_colorCLI->primaryOver(' parts, ') . $this->_colorCLI->alternateOver($timeHeaders . 's') . $this->_colorCLI->primaryOver(' to download articles, ') . $this->_colorCLI->alternateOver($timeUpdate . 's') . $this->_colorCLI->primaryOver(' to insert binaries/parts, ') . $this->_colorCLI->alternateOver($timeLoop . 's') . $this->_colorCLI->primary(' total.')); } } unset($this->message); unset($data); return $returnArray; } else { echo "Error: Can't get parts from server (msgs not array) {$n}"; echo "Skipping group{$n}"; return $returnArray; } }
/** * Echo a updated release name to CLI. * * @param array $data * array( * 'new_name' => (string) The new release search name. * 'old_name' => (string) The old release search name. * 'new_category' => (string) The new category name or id for the release. * 'old_category' => (string) The old category name or id for the release. * 'group' => (string) The group name or id of the release. * 'release_id' => (int) The id of the release. * 'method' => (string) The method used to rename the release. * ) * * @access public * @static * @void */ public static function echoChangedReleaseName(array $data = array('new_name' => '', 'old_name' => '', 'new_category' => '', 'old_category' => '', 'group' => '', 'release_id' => 0, 'method' => '')) { echo PHP_EOL . ColorCLI::headerOver('New name: ') . ColorCLI::primaryOver($data['new_name']) . PHP_EOL . ColorCLI::headerOver('Old name: ') . ColorCLI::primaryOver($data['old_name']) . PHP_EOL . ColorCLI::headerOver('New category: ') . ColorCLI::primaryOver($data['new_category']) . PHP_EOL . ColorCLI::headerOver('Old category: ') . ColorCLI::primaryOver($data['old_category']) . PHP_EOL . ColorCLI::headerOver('Group: ') . ColorCLI::primaryOver($data['group']) . PHP_EOL . ColorCLI::headerOver('Release id: ') . ColorCLI::primaryOver($data['release_id']) . PHP_EOL . ColorCLI::headerOver('Method: ') . ColorCLI::primaryOver($data['method']) . PHP_EOL; }
/** * Work out the category based on the name, resets to null if no category matched. */ private function determineCategory($rel, &$foundName, &$methodused) { $categoryID = null; $category = new Categorize(); $categoryID = $category->determineCategory($rel['groupname'], $foundName); if ($methodused == 'a.b.hdtv.x264' && $rel['groupname'] == 'alt.binaries.hdtv.x264') { $categoryID = Category::CAT_MOVIE_HD; } if ($categoryID == $rel['categoryid'] || $categoryID == '7900' || ($foundName == $rel['name'] || $foundName == $rel['searchname'])) { $foundName = null; $methodused = null; } else { $foundName = str_replace(''', '', trim(html_entity_decode($foundName))); $name = str_replace(' ', '_', $foundName); $searchname = str_replace('_', ' ', $foundName); echo PHP_EOL . ColorCLI::headerOver('ReleaseID: ') . ColorCLI::primaryOver($rel['rid']) . PHP_EOL . ColorCLI::headerOver(' Group: ') . ColorCLI::primaryOver($rel['groupname']) . PHP_EOL . ColorCLI::headerOver(' Old Name: ') . ColorCLI::primaryOver($rel['name']) . PHP_EOL . ColorCLI::headerOver(' Old SearchName: ') . ColorCLI::primaryOver($rel['searchname']) . PHP_EOL . ColorCLI::headerOver(' New Name: ') . ColorCLI::primaryOver($name) . PHP_EOL . ColorCLI::headerOver(' New SearchName: ') . ColorCLI::primaryOver($searchname) . PHP_EOL . ColorCLI::headerOver(' Old Cat: ') . ColorCLI::primaryOver($rel['categoryid']) . PHP_EOL . ColorCLI::headerOver(' New Cat: ') . ColorCLI::primaryOver($categoryID) . PHP_EOL . ColorCLI::headerOver(' Method: ') . ColorCLI::primaryOver($methodused) . PHP_EOL; if (!$this->echoonly) { $this->pdo->queryExec(sprintf("update releases SET name = %s, searchname = %s, categoryid = %d, imdbid = NULL, rageid = -1, bookinfoid = NULL, musicinfoid = NULL, consoleinfoid = NULL WHERE releases.id = %d", $this->pdo->escapeString($name), $this->pdo->escapeString($searchname), $categoryID, $rel['rid'])); } $this->numupdated++; } }
/** * Loop over range of wanted headers, insert headers into DB. * * @param array $groupMySQL The group info from mysql. * @param int $first The oldest wanted header. * @param int $last The newest wanted header. * @param string $type Is this partrepair or update or backfill? * @param null|array $missingParts If we are running in partrepair, the list of missing article numbers. * * @return array Empty on failure. */ public function scan($groupMySQL, $first, $last, $type = 'update', $missingParts = null) { // Start time of scan method and of fetching headers. $startLoop = microtime(true); // Check if MySQL tables exist, create if they do not, get their names at the same time. $tableNames = $this->_groups->getCBPTableNames($this->_tablePerGroup, $groupMySQL['id']); $returnArray = []; $partRepair = $type === 'partrepair'; $addToPartRepair = $type === 'update' && $this->_partRepair; // Download the headers. if ($partRepair === true) { // This is slower but possibly is better with missing headers. $headers = $this->_nntp->getOverview($first . '-' . $last, true, false); } else { $headers = $this->_nntp->getXOVER($first . '-' . $last); } // If there was an error, try to reconnect. if ($this->_nntp->isError($headers)) { // Increment if part repair and return false. if ($partRepair === true) { $this->_pdo->queryExec(sprintf('UPDATE partrepair SET attempts = attempts + 1 WHERE group_id = %d AND numberid %s', $groupMySQL['id'], $first == $last ? '= ' . $first : 'IN (' . implode(',', range($first, $last)) . ')')); return $returnArray; } // This is usually a compression error, so try disabling compression. $this->_nntp->doQuit(); if ($this->_nntp->doConnect(false) !== true) { return $returnArray; } // Re-select group, download headers again without compression and re-enable compression. $this->_nntp->selectGroup($groupMySQL['name']); $headers = $this->_nntp->getXOVER($first . '-' . $last); $this->_nntp->enableCompression(); // Check if the non-compression headers have an error. if ($this->_nntp->isError($headers)) { $this->log("Code {$headers->code}: {$headers->message}\nSkipping group: {${$groupMySQL['name']}}", 'scan', Logger::LOG_WARNING, 'error'); return $returnArray; } } // Start of processing headers. $startCleaning = microtime(true); // End of the getting data from usenet. $timeHeaders = number_format($startCleaning - $startLoop, 2); // Check if we got headers. $msgCount = count($headers); if ($msgCount < 1) { return $returnArray; } // Get highest and lowest article numbers/dates. $iterator1 = 0; $iterator2 = $msgCount - 1; while (true) { if (!isset($returnArray['firstArticleNumber']) && isset($headers[$iterator1]['Number'])) { $returnArray['firstArticleNumber'] = $headers[$iterator1]['Number']; $returnArray['firstArticleDate'] = $headers[$iterator1]['Date']; } if (!isset($returnArray['lastArticleNumber']) && isset($headers[$iterator2]['Number'])) { $returnArray['lastArticleNumber'] = $headers[$iterator2]['Number']; $returnArray['lastArticleDate'] = $headers[$iterator2]['Date']; } // Break if we found non empty articles. if (isset($returnArray['firstArticleNumber']) && isset($returnArray['lastArticleNumber'])) { break; } // Break out if we couldn't find anything. if ($iterator1++ >= $msgCount - 1 || $iterator2-- <= 0) { break; } } $headersRepaired = $articles = $rangeNotReceived = $collectionIDs = $binariesUpdate = $headersReceived = $headersNotInserted = []; $notYEnc = $headersBlackListed = 0; $partsQuery = $partsCheck = sprintf('INSERT INTO %s (binaryid, number, messageid, partnumber, size, collection_id) VALUES ', $tableNames['pname']); $this->_pdo->beginTransaction(); // Loop articles, figure out files/parts. foreach ($headers as $header) { // Check if we got the article or not. if (isset($header['Number'])) { $headersReceived[] = $header['Number']; } else { if ($addToPartRepair) { $rangeNotReceived[] = $header['Number']; } continue; } // If set we are running in partRepair mode. if ($partRepair === true && !is_null($missingParts)) { if (!in_array($header['Number'], $missingParts)) { // If article isn't one that is missing skip it. continue; } else { // We got the part this time. Remove article from part repair. $headersRepaired[] = $header['Number']; } } /* * Find part / total parts. Ignore if no part count found. * * \s* Trims the leading space. * (?!"Usenet Index Post) ignores these types of articles, they are useless. * (.+) Fetches the subject. * \s+ Trims trailing space after the subject. * \((\d+)\/(\d+)\) Gets the part count. * No ending ($) as there are cases of subjects with extra data after the part count. */ if (preg_match('/^\\s*(?!"Usenet Index Post)(.+)\\s+\\((\\d+)\\/(\\d+)\\)/', $header['Subject'], $matches)) { // Add yEnc to subjects that do not have them, but have the part number at the end of the header. if (!stristr($header['Subject'], 'yEnc')) { $matches[1] .= ' yEnc'; } } else { if ($this->_showDroppedYEncParts === true && strpos($header['Subject'], '"Usenet Index Post') !== 0) { file_put_contents(nZEDb_LOGS . 'not_yenc' . $groupMySQL['name'] . '.dropped.log', $header['Subject'] . PHP_EOL, FILE_APPEND); } $notYEnc++; continue; } // Filter subject based on black/white list. if ($this->_blackListEmpty === false && $this->isBlackListed($header, $groupMySQL['name'])) { $headersBlackListed++; continue; } if (!isset($header['Bytes'])) { $header['Bytes'] = isset($header[':bytes']) ? $header[':bytes'] : 0; } $header['Bytes'] = (int) $header['Bytes']; // Set up the info for inserting into parts/binaries/collections tables. if (!isset($articles[$matches[1]])) { // Attempt to find the file count. If it is not found, set it to 0. if (!preg_match('/[[(\\s](\\d{1,5})(\\/|[\\s_]of[\\s_]|-)(\\d{1,5})[])\\s$:]/i', $matches[1], $fileCount)) { $fileCount[1] = $fileCount[3] = 0; if ($this->_showDroppedYEncParts === true) { file_put_contents(nZEDb_LOGS . 'no_files' . $groupMySQL['name'] . '.log', $header['Subject'] . PHP_EOL, FILE_APPEND); } } // Used to group articles together when forming the release/nzb. $header['CollectionKey'] = $this->_collectionsCleaning->collectionsCleaner($matches[1], $groupMySQL['name']) . $header['From'] . $groupMySQL['id'] . $fileCount[3]; if (!isset($collectionIDs[$header['CollectionKey']])) { /* Date from header should be a string this format: * 31 Mar 2014 15:36:04 GMT or 6 Oct 1998 04:38:40 -0500 * Still make sure it's not unix time, convert it to unix time if it is. */ $header['Date'] = is_numeric($header['Date']) ? $header['Date'] : strtotime($header['Date']); // Get the current unixtime from PHP. $now = time(); $collectionID = $this->_pdo->queryInsert(sprintf("\n\t\t\t\t\t\t\tINSERT INTO %s (subject, fromname, date, xref, group_id,\n\t\t\t\t\t\t\t\ttotalfiles, collectionhash, dateadded)\n\t\t\t\t\t\t\tVALUES (%s, %s, FROM_UNIXTIME(%s), %s, %d, %d, '%s', NOW())\n\t\t\t\t\t\t\tON DUPLICATE KEY UPDATE dateadded = NOW()", $tableNames['cname'], $this->_pdo->escapeString(substr(utf8_encode($matches[1]), 0, 255)), $this->_pdo->escapeString(utf8_encode($header['From'])), is_numeric($header['Date']) ? $header['Date'] > $now ? $now : $header['Date'] : $now, $this->_pdo->escapeString(substr($header['Xref'], 0, 255)), $groupMySQL['id'], $fileCount[3], sha1($header['CollectionKey']))); if ($collectionID === false) { if ($addToPartRepair) { $headersNotInserted[] = $header['Number']; } $this->_pdo->Rollback(); $this->_pdo->beginTransaction(); continue; } $collectionIDs[$header['CollectionKey']] = $collectionID; } else { $collectionID = $collectionIDs[$header['CollectionKey']]; } $binaryID = $this->_pdo->queryInsert(sprintf("\n\t\t\t\t\t\tINSERT INTO %s (binaryhash, name, collectionid, totalparts, currentparts, filenumber, partsize)\n\t\t\t\t\t\tVALUES ('%s', %s, %d, %d, 1, %d, %d)\n\t\t\t\t\t\tON DUPLICATE KEY UPDATE currentparts = currentparts + 1, partsize = partsize + %d", $tableNames['bname'], md5($matches[1] . $header['From'] . $groupMySQL['id']), $this->_pdo->escapeString(utf8_encode($matches[1])), $collectionID, $matches[3], $fileCount[1], $header['Bytes'], $header['Bytes'])); if ($binaryID === false) { if ($addToPartRepair) { $headersNotInserted[] = $header['Number']; } $this->_pdo->Rollback(); $this->_pdo->beginTransaction(); continue; } $binariesUpdate[$binaryID]['Size'] = 0; $binariesUpdate[$binaryID]['Parts'] = 0; $articles[$matches[1]]['CollectionID'] = $collectionID; $articles[$matches[1]]['BinaryID'] = $binaryID; } else { $binaryID = $articles[$matches[1]]['BinaryID']; $collectionID = $articles[$matches[1]]['CollectionID']; $binariesUpdate[$binaryID]['Size'] += $header['Bytes']; $binariesUpdate[$binaryID]['Parts']++; } // Strip the < and >, saves space in DB. $header['Message-ID'][0] = "'"; $partsQuery .= '(' . $binaryID . ',' . $header['Number'] . ',' . rtrim($header['Message-ID'], '>') . "'," . $matches[2] . ',' . $header['Bytes'] . ',' . $collectionID . '),'; } unset($headers); // Reclaim memory. // Start of inserting into SQL. $startUpdate = microtime(true); // End of processing headers. $timeCleaning = number_format($startUpdate - $startCleaning, 2); $binariesQuery = $binariesCheck = sprintf('INSERT INTO %s (id, partsize, currentparts) VALUES ', $tableNames['bname']); foreach ($binariesUpdate as $binaryID => $binary) { $binariesQuery .= '(' . $binaryID . ',' . $binary['Size'] . ',' . $binary['Parts'] . '),'; } $binariesEnd = ' ON DUPLICATE KEY UPDATE partsize = VALUES(partsize) + partsize, currentparts = VALUES(currentparts) + currentparts'; $binariesQuery = rtrim($binariesQuery, ',') . $binariesEnd; // Check if we got any binaries. If we did, try to insert them. if (strlen($binariesCheck . $binariesEnd) === strlen($binariesQuery) ? true : $this->_pdo->queryExec($binariesQuery)) { if ($this->_debug) { $this->_colorCLI->doEcho($this->_colorCLI->debug('Sending ' . round(strlen($partsQuery) / 1024, 2) . ' KB of parts to MySQL')); } if (strlen($partsQuery) === strlen($partsCheck) ? true : $this->_pdo->queryExec(rtrim($partsQuery, ','))) { $this->_pdo->Commit(); } else { if ($addToPartRepair) { $headersNotInserted += $headersReceived; } $this->_pdo->Rollback(); } } else { if ($addToPartRepair) { $headersNotInserted += $headersReceived; } $this->_pdo->Rollback(); } if ($this->_echoCLI && $partRepair === false) { $this->_colorCLI->doEcho($this->_colorCLI->primary('Received ' . count($headersReceived) . ' articles of ' . number_format($last - $first + 1) . ' requested, ' . $headersBlackListed . ' blacklisted, ' . $notYEnc . ' not yEnc.')); } // Start of part repair. $startPR = microtime(true); // End of inserting. $timeInsert = number_format($startPR - $startUpdate, 2); if ($partRepair && count($headersRepaired) > 0) { $this->removeRepairedParts($headersRepaired, $tableNames['prname'], $groupMySQL['id']); } if ($addToPartRepair) { $notInsertedCount = count($headersNotInserted); if ($notInsertedCount > 0) { $this->addMissingParts($headersNotInserted, $tableNames['prname'], $groupMySQL['id']); $this->log($notInsertedCount . ' articles failed to insert!', 'scan', Logger::LOG_WARNING, 'warning'); } // Check if we have any missing headers. if ($last - $first - $notYEnc - $headersBlackListed + 1 > count($headersReceived)) { $rangeNotReceived = array_merge($rangeNotReceived, array_diff(range($first, $last), $headersReceived)); } $notReceivedCount = count($rangeNotReceived); if ($notReceivedCount > 0) { $this->addMissingParts($rangeNotReceived, $tableNames['prname'], $groupMySQL['id']); if ($this->_echoCLI) { $this->_colorCLI->doEcho($this->_colorCLI->alternate('Server did not return ' . $notReceivedCount . ' articles from ' . $groupMySQL['name'] . '.'), true); } } } $currentMicroTime = microtime(true); if ($this->_echoCLI) { $this->_colorCLI->doEcho($this->_colorCLI->alternateOver($timeHeaders . 's') . $this->_colorCLI->primaryOver(' to download articles, ') . $this->_colorCLI->alternateOver($timeCleaning . 's') . $this->_colorCLI->primaryOver(' to process collections, ') . $this->_colorCLI->alternateOver($timeInsert . 's') . $this->_colorCLI->primaryOver(' to insert binaries/parts, ') . $this->_colorCLI->alternateOver(number_format($currentMicroTime - $startPR, 2) . 's') . $this->_colorCLI->primaryOver(' for part repair, ') . $this->_colorCLI->alternateOver(number_format($currentMicroTime - $startLoop, 2) . 's') . $this->_colorCLI->primary(' total.')); } return $returnArray; }