diff --git a/lib/Tokenizer.php b/lib/Tokenizer.php index 4a72a6e..a70e6c7 100644 --- a/lib/Tokenizer.php +++ b/lib/Tokenizer.php @@ -84,10 +84,15 @@ class Tokenizer { 'text' => substr($line, $this->offset, $lineLength - $this->offset) . ((!$this->data->lastLine) ? "\n" : '') ]; } elseif (!$this->data->lastLine) { - $tokens[] = [ - 'scopes' => $this->scopeStack, - 'text' => "\n" - ]; + // Some matches can grab the new line, so check to see if it's already present + // before appending another token. + $lastToken = end($tokens); + if ($lastToken === false || substr($lastToken['text'], -1) !== "\n") { + $tokens[] = [ + 'scopes' => $this->scopeStack, + 'text' => "\n" + ]; + } } assert($this->debugTokens($tokens));