Browse Source

Added support for spaces when tokenizing Grammar scope names

main
Dustin Wilson 3 years ago
parent
commit
21f614213e
  1. 7
      lib/Tokenizer.php

7
lib/Tokenizer.php

@ -53,7 +53,12 @@ class Tokenizer {
$this->data = $data;
$this->grammar = $grammar;
$this->ruleStack = [ $this->grammar ];
$this->scopeStack = [ $this->grammar->scopeName ];
// Grammar scope names can contain spaces (eg: source.js.rails
// source.js.jquery), meaning more than one needs to be added to the stack. It
// doesn't look like anything does it elsewhere, so spaces in scope names only
// work here.
$this->scopeStack = preg_split('/\s+/', $this->grammar->scopeName);
}
/** Receives lines from the Data object and yields an array of tokens */

Loading…
Cancel
Save