From 0dbbc67f1af2f0e5dc59a1ee2f07c2381837725a Mon Sep 17 00:00:00 2001
From: Dustin Wilson <dustin@dustinwilson.com>
Date: Tue, 17 Aug 2021 22:05:45 -0500
Subject: [PATCH] More minor tweaks/fixes

---
 .gitignore        |  2 +-
 lib/Tokenizer.php | 32 +++++++++-----------------------
 2 files changed, 10 insertions(+), 24 deletions(-)

diff --git a/.gitignore b/.gitignore
index cfee353..c03567f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,7 +2,7 @@
 /deps/
 /node_modules/
 /vendor/
-test*.php
+test*.*
 *-old.php
 yarn-error.log
 
diff --git a/lib/Tokenizer.php b/lib/Tokenizer.php
index bb5afa8..4a5935e 100644
--- a/lib/Tokenizer.php
+++ b/lib/Tokenizer.php
@@ -43,12 +43,10 @@ class Tokenizer {
             // If after tokenizing the line the entire line still hasn't been tokenized then
             // create a token of the rest of the line.
             $lineLength = strlen($line);
-            if ($this->offset < $lineLength) {
-                $tokens[] = new Token(
-                    $this->scopeStack,
-                    substr($line, $this->offset, $lineLength)
-                );
-            }
+            $tokens[] = new Token(
+                $this->scopeStack,
+                ($this->offset < $lineLength) ? substr($line, $this->offset, $lineLength) . "\n" : "\n"
+            );
 
             yield $lineNumber => $tokens;
         }
@@ -140,29 +138,17 @@ class Tokenizer {
                                     $this->offset = $m[1];
                                 }
 
-                                // If the capture rule has patterns of its own then
-                                // those must be matched, too.
+                                if ($rule->captures[$k]->name !== null) {
+                                    $this->scopeStack[] = $this->resolveScopeName($rule->captures[$k]->name, $match);
+                                }
+
                                 if ($rule->captures[$k]->patterns !== null) {
                                     $this->ruleStack[] = $rule->captures[$k];
-
-                                    if ($rule->captures[$k]->name !== null) {
-                                        $this->scopeStack[] = $this->resolveScopeName($rule->captures[$k]->name, $match);
-                                    }
-
                                     $tokens = [ ...$tokens, ...$this->tokenizeLine($line) ];
-
                                     array_pop($this->ruleStack);
                                 } else {
-                                    // If it's not the 0 capture and a capture without any patterns add the name
-                                    // and content names if they exist to the token's scope stack but not to the
-                                    // global one.
-                                    $scopeStack = $this->scopeStack;
-                                    if ($rule->captures[$k]->name !== null) {
-                                        $scopeStack[] = $this->resolveScopeName($rule->captures[$k]->name, $match);
-                                    }
-
                                     $tokens[] = new Token(
-                                        $scopeStack,
+                                        $this->scopeStack,
                                         $m[0]
                                     );
                                 }