Browse Source

Now tokenizes properly when matches grab the newline

main
Dustin Wilson 3 years ago
parent
commit
774658603d
  1. 5
      lib/Tokenizer.php

5
lib/Tokenizer.php

@ -84,11 +84,16 @@ class Tokenizer {
'text' => substr($line, $this->offset, $lineLength - $this->offset) . ((!$this->data->lastLine) ? "\n" : '') 'text' => substr($line, $this->offset, $lineLength - $this->offset) . ((!$this->data->lastLine) ? "\n" : '')
]; ];
} elseif (!$this->data->lastLine) { } elseif (!$this->data->lastLine) {
// Some matches can grab the new line, so check to see if it's already present
// before appending another token.
$lastToken = end($tokens);
if ($lastToken === false || substr($lastToken['text'], -1) !== "\n") {
$tokens[] = [ $tokens[] = [
'scopes' => $this->scopeStack, 'scopes' => $this->scopeStack,
'text' => "\n" 'text' => "\n"
]; ];
} }
}
assert($this->debugTokens($tokens)); assert($this->debugTokens($tokens));
yield $lineNumber => $tokens; yield $lineNumber => $tokens;

Loading…
Cancel
Save