diff --git a/lib/Tokenizer.php b/lib/Tokenizer.php index a70e6c7..221fc6a 100644 --- a/lib/Tokenizer.php +++ b/lib/Tokenizer.php @@ -53,7 +53,12 @@ class Tokenizer { $this->data = $data; $this->grammar = $grammar; $this->ruleStack = [ $this->grammar ]; - $this->scopeStack = [ $this->grammar->scopeName ]; + + // Grammar scope names can contain spaces (eg: source.js.rails + // source.js.jquery), meaning more than one needs to be added to the stack. It + // doesn't look like anything does it elsewhere, so spaces in scope names only + // work here. + $this->scopeStack = preg_split('/\s+/', $this->grammar->scopeName); } /** Receives lines from the Data object and yields an array of tokens */