From 774658603dde74e7d29e11878d08f277630f5919 Mon Sep 17 00:00:00 2001
From: Dustin Wilson <dustin@dustinwilson.com>
Date: Wed, 15 Sep 2021 14:24:57 -0500
Subject: [PATCH] Now tokenizes properly when matches grab the newline

---
 lib/Tokenizer.php | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/lib/Tokenizer.php b/lib/Tokenizer.php
index 4a72a6e..a70e6c7 100644
--- a/lib/Tokenizer.php
+++ b/lib/Tokenizer.php
@@ -84,10 +84,15 @@ class Tokenizer {
                     'text' => substr($line, $this->offset, $lineLength - $this->offset) . ((!$this->data->lastLine) ? "\n" : '')
                 ];
             } elseif (!$this->data->lastLine) {
-                $tokens[] = [
-                    'scopes' => $this->scopeStack,
-                    'text' => "\n"
-                ];
+                // Some matches can grab the new line, so check to see if it's already present
+                // before appending another token.
+                $lastToken = end($tokens);
+                if ($lastToken === false || substr($lastToken['text'], -1) !== "\n") {
+                    $tokens[] = [
+                        'scopes' => $this->scopeStack,
+                        'text' => "\n"
+                    ];
+                }
             }
 
             assert($this->debugTokens($tokens));