From 730f10ead343db0c9d5a57b90597947dd0f904bd Mon Sep 17 00:00:00 2001
From: Dustin Wilson <dustin@dustinwilson.com>
Date: Tue, 24 Aug 2021 16:23:50 -0500
Subject: [PATCH] Minor changes

---
 lib/Highlight.php |  2 +-
 lib/Tokenizer.php | 23 +++++------------------
 2 files changed, 6 insertions(+), 19 deletions(-)

diff --git a/lib/Highlight.php b/lib/Highlight.php
index a1abe88..56efb6c 100644
--- a/lib/Highlight.php
+++ b/lib/Highlight.php
@@ -24,7 +24,7 @@ class Highlight {
         $tokenList = $tokenizer->tokenize();
 
         foreach ($tokenList as $lineNumber => $tokens) {
-            if ($lineNumber === 24) {
+            if ($lineNumber === 26) {
                 var_export($tokens);
                 echo "\n";
                 die();
diff --git a/lib/Tokenizer.php b/lib/Tokenizer.php
index 9d6365f..d1ea8c3 100644
--- a/lib/Tokenizer.php
+++ b/lib/Tokenizer.php
@@ -24,8 +24,6 @@ class Tokenizer {
     protected ?Pattern $activeInjection = null;
     protected array $ruleStack;
     protected array $scopeStack;
-    protected int $debug = 0;
-    protected int $debugCount = 0;
 
     protected const SCOPE_RESOLVE_REGEX = '/\$(\d+)|\${(\d+):\/(downcase|upcase)}/S';
     protected const ANCHOR_CHECK_REGEX = '/(?<!\\\)\\\([AGzZ])/S';
@@ -41,8 +39,6 @@ class Tokenizer {
 
     public function tokenize(): \Generator {
         foreach ($this->data->get() as $lineNumber => $line) {
-            $this->debug = $lineNumber;
-            $this->debugCount = 0;
             $this->offset = 0;
 
             $lineLength = strlen($line);
@@ -57,13 +53,11 @@ class Tokenizer {
                     'scopes' => $this->scopeStack,
                     'text' => substr($line, $this->offset, $lineLength - $this->offset) . ((!$this->data->lastLine) ? "\n" : '')
                 ];
-                $this->debugCount++;
             } elseif (!$this->data->lastLine) {
                 $tokens[] = [
                     'scopes' => $this->scopeStack,
                     'text' => "\n"
                 ];
-                $this->debugCount++;
             }
 
             yield $lineNumber => $tokens;
@@ -190,7 +184,6 @@ class Tokenizer {
                         'scopes' => $this->scopeStack,
                         'text' => substr($line, $this->offset, $match[0][1] - $this->offset)
                     ];
-                    $this->debugCount++;
                     $this->offset = $match[0][1];
                 }
 
@@ -217,7 +210,6 @@ class Tokenizer {
                                 'scopes' => $this->scopeStack,
                                 'text' => substr($line, $this->offset, $m[1] - $this->offset)
                             ];
-                            $this->debugCount++;
                             $this->offset = $m[1];
                         }
 
@@ -235,19 +227,17 @@ class Tokenizer {
 
                             $this->ruleStack[] = $pattern->captures[$k];
                             // Only tokenize the part of the line that's contains the match.
-                            $captureLength = $m[1] + strlen($m[0]);
-                            $tokens = [ ...$tokens, ...$this->tokenizeLine($line, $captureLength) ];
+                            $captureEndOffset = $m[1] + strlen($m[0]);
+                            $tokens = [ ...$tokens, ...$this->tokenizeLine($line, $captureEndOffset) ];
 
                             // If the offset is before the end of the capture then create a token from the
                             // bits of the capture from the offset until the end of the capture.
-                            $endOffset = $captureLength;
-                            if ($endOffset > $this->offset) {
+                            if ($captureEndOffset > $this->offset) {
                                 $tokens[] = [
                                     'scopes' => $this->scopeStack,
-                                    'text' => substr($line, $this->offset, $endOffset - $this->offset)
+                                    'text' => substr($line, $this->offset, $captureEndOffset - $this->offset)
                                 ];
-                                $this->debugCount++;
-                                $this->offset = $endOffset;
+                                $this->offset = $captureEndOffset;
                             }
 
                             array_pop($this->ruleStack);
@@ -315,7 +305,6 @@ class Tokenizer {
                                     'scopes' => $this->scopeStack,
                                     'text' => $m[0]
                                 ];
-                                $this->debugCount++;
                                 $this->offset = $m[1] + strlen($m[0]);
                             }
                         }
@@ -335,7 +324,6 @@ class Tokenizer {
                     ];
 
                     $this->offset = $match[0][1] + strlen($match[0][0]);
-                    $this->debugCount++;
                 }
 
                 // If the pattern is a begin pattern and has a content name then add that to the
@@ -364,7 +352,6 @@ class Tokenizer {
                         'scopes' => $this->scopeStack,
                         'text' => substr($line, $this->offset, $endOffset - $this->offset)
                     ];
-                    $this->debugCount++;
                     $this->offset = $endOffset;
                 }