/** * Initialise the tokenizer. * * Pre-checks the content to see if it looks minified. * * @param string $content The content to tokenize, * @param \PHP_CodeSniffer\Config $config The config data for the run. * @param string $eolChar The EOL char used in the content. * * @return void * @throws TokenizerException If the file appears to be minified. */ public function __construct($content, Config $config, $eolChar = '\\n') { if ($this->isMinifiedContent($content, $eolChar) === true) { throw new TokenizerException('File appears to be minified and cannot be processed'); } return parent::__construct($content, $config, $eolChar); }
/** * Tokenizes the file and prepares it for the test run. * * @return void */ public function parse() { if (empty($this->tokens) === false) { // File has already been parsed. return; } try { $tokenizerClass = 'PHP_CodeSniffer\\Tokenizers\\' . $this->tokenizerType; $this->tokenizer = new $tokenizerClass($this->content, $this->config, $this->eolChar); $this->tokens = $this->tokenizer->getTokens(); } catch (TokenizerException $e) { $this->addWarning($e->getMessage(), null, 'Internal.Tokenizer.Exception'); if (PHP_CODESNIFFER_VERBOSITY > 0) { echo "[{$this->tokenizerType} => tokenizer error]... "; if (PHP_CODESNIFFER_VERBOSITY > 1) { echo PHP_EOL; } } return; } $this->numTokens = count($this->tokens); // Check for mixed line endings as these can cause tokenizer errors and we // should let the user know that the results they get may be incorrect. // This is done by removing all backslashes, removing the newline char we // detected, then converting newlines chars into text. If any backslashes // are left at the end, we have additional newline chars in use. $contents = str_replace('\\', '', $this->content); $contents = str_replace($this->eolChar, '', $contents); $contents = str_replace("\n", '\\n', $contents); $contents = str_replace("\r", '\\r', $contents); if (strpos($contents, '\\') !== false) { $error = 'File has mixed line endings; this may cause incorrect results'; $this->addWarningOnLine($error, 1, 'Internal.LineEndings.Mixed'); } if (PHP_CODESNIFFER_VERBOSITY > 0) { if ($this->numTokens === 0) { $numLines = 0; } else { $numLines = $this->tokens[$this->numTokens - 1]['line']; } echo "[{$this->tokenizerType} => {$this->numTokens} tokens in {$numLines} lines]... "; if (PHP_CODESNIFFER_VERBOSITY > 1) { echo PHP_EOL; } } }