Browse Source

Now tokenizes properly when matches grab the newline

main
Dustin Wilson 3 years ago
parent
commit
774658603d
  1. 5
      lib/Tokenizer.php

5
lib/Tokenizer.php

@ -84,11 +84,16 @@ class Tokenizer {
'text' => substr($line, $this->offset, $lineLength - $this->offset) . ((!$this->data->lastLine) ? "\n" : '')
];
} elseif (!$this->data->lastLine) {
// Some matches can grab the new line, so check to see if it's already present
// before appending another token.
$lastToken = end($tokens);
if ($lastToken === false || substr($lastToken['text'], -1) !== "\n") {
$tokens[] = [
'scopes' => $this->scopeStack,
'text' => "\n"
];
}
}
assert($this->debugTokens($tokens));
yield $lineNumber => $tokens;

Loading…
Cancel
Save