From eb9c34a02474d62895500ee71dceb7de50753bb3 Mon Sep 17 00:00:00 2001
From: Dustin Wilson <dustin@dustinwilson.com>
Date: Sun, 22 Aug 2021 09:29:51 -0500
Subject: [PATCH] Minor nit picking

---
 lib/Tokenizer.php | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/lib/Tokenizer.php b/lib/Tokenizer.php
index 819ae63..5939397 100644
--- a/lib/Tokenizer.php
+++ b/lib/Tokenizer.php
@@ -56,15 +56,15 @@ class Tokenizer {
                     $this->scopeStack,
                     substr($line, $this->offset, $lineLength - $this->offset) . ((!$this->data->lastLine) ? "\n" : '')
                 );
+                $this->debugCount++;
             } elseif (!$this->data->lastLine) {
                 $tokens[] = new Token(
                     $this->scopeStack,
                     "\n"
                 );
+                $this->debugCount++;
             }
 
-            $this->debugCount++;
-
             yield $lineNumber => $tokens;
         }
     }
@@ -112,9 +112,15 @@ class Tokenizer {
                 while (true) {
                     $rule = $currentRules[$i];
 
+                    if ($this->debug === 19 && $this->debugCount === 3) {
+                        $rule->get();
+                        die(var_export($rule->ownerGrammar));
+                    }
+
                     // If the rule is a Pattern
                     if ($rule instanceof Pattern) {
-                        // Throw out pattern regexes with anchors that cannot match the current line.
+                        // Throw out pattern regexes with anchors that should match the current line.
+                        // This is necessary because the tokenizer is fed data line by line.
                         if (preg_match(self::ANCHOR_CHECK_REGEX, $rule->match, $validRegexMatch) === 1) {
                             if (
                                 // \A anchors match the beginning of the whole string, not just this line