/**
* Initialise the tokenizer.
*
* Pre-checks the content to see if it looks minified.
*
* @param string $content The content to tokenize,
* @param \PHP_CodeSniffer\Config $config The config data for the run.
* @param string $eolChar The EOL char used in the content.
*
* @return void
* @throws TokenizerException If the file appears to be minified.
*/
public function __construct($content, Config $config, $eolChar = '\\n')
{
if ($this->isMinifiedContent($content, $eolChar) === true) {
throw new TokenizerException('File appears to be minified and cannot be processed');
}
return parent::__construct($content, $config, $eolChar);
}
/**
* Tokenizes the file and prepares it for the test run.
*
* @return void
*/
public function parse()
{
if (empty($this->tokens) === false) {
// File has already been parsed.
return;
}
try {
$tokenizerClass = 'PHP_CodeSniffer\\Tokenizers\\' . $this->tokenizerType;
$this->tokenizer = new $tokenizerClass($this->content, $this->config, $this->eolChar);
$this->tokens = $this->tokenizer->getTokens();
} catch (TokenizerException $e) {
$this->addWarning($e->getMessage(), null, 'Internal.Tokenizer.Exception');
if (PHP_CODESNIFFER_VERBOSITY > 0) {
echo "[{$this->tokenizerType} => tokenizer error]... ";
if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo PHP_EOL;
}
}
return;
}
$this->numTokens = count($this->tokens);
// Check for mixed line endings as these can cause tokenizer errors and we
// should let the user know that the results they get may be incorrect.
// This is done by removing all backslashes, removing the newline char we
// detected, then converting newlines chars into text. If any backslashes
// are left at the end, we have additional newline chars in use.
$contents = str_replace('\\', '', $this->content);
$contents = str_replace($this->eolChar, '', $contents);
$contents = str_replace("\n", '\\n', $contents);
$contents = str_replace("\r", '\\r', $contents);
if (strpos($contents, '\\') !== false) {
$error = 'File has mixed line endings; this may cause incorrect results';
$this->addWarningOnLine($error, 1, 'Internal.LineEndings.Mixed');
}
if (PHP_CODESNIFFER_VERBOSITY > 0) {
if ($this->numTokens === 0) {
$numLines = 0;
} else {
$numLines = $this->tokens[$this->numTokens - 1]['line'];
}
echo "[{$this->tokenizerType} => {$this->numTokens} tokens in {$numLines} lines]... ";
if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo PHP_EOL;
}
}
}
