From d93fe25e58b69715e7f55d0e57090494846fcc5d Mon Sep 17 00:00:00 2001
From: "J. King" <jking@jkingweb.ca>
Date: Thu, 12 Dec 2019 10:48:11 -0500
Subject: [PATCH] Combine character tokens in test harness

---
 tests/cases/TestTokenizer.php |  3 ++-
 tests/lib/StandardTest.php    | 24 ++++++++++++++++++++++++
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/tests/cases/TestTokenizer.php b/tests/cases/TestTokenizer.php
index 5f605c7..3ddd0e5 100644
--- a/tests/cases/TestTokenizer.php
+++ b/tests/cases/TestTokenizer.php
@@ -31,7 +31,8 @@ class TestTokenizer extends \dW\HTML5\Test\StandardTest {
             $actual[] = $t;
         } while (!($t instanceof EOFToken));
         array_pop($actual);
-        $this->assertEquals($expected, $actual);       
+        $actual = $this->combineCharacterTokens($actual);
+        $this->assertEquals($expected, $actual);
     }
 
     public function provideStandardTokenizerTests() {
diff --git a/tests/lib/StandardTest.php b/tests/lib/StandardTest.php
index 749bb4f..40b5e58 100644
--- a/tests/lib/StandardTest.php
+++ b/tests/lib/StandardTest.php
@@ -30,6 +30,30 @@ class StandardTest extends \PHPUnit\Framework\TestCase {
         return $str;
     }
 
+    protected function combineCharacterTokens(array $tokens) : array {
+        $out = [];
+        $pending = null;
+        foreach ($tokens as $t) {
+            if ($t instanceof CharacterToken) {
+                if (!$pending) {
+                    $pending = $t;
+                } else {
+                    $pending->data .= $t->data;
+                }
+            } else {
+                if ($pending) {
+                    $out[] = $pending;
+                    $pending = null;
+                }
+                $out[] = $t;
+            }
+        }
+        if ($pending) {
+            $out[] = $pending;
+        }
+        return $out;
+    }
+
     protected function makeTokenTests(string ...$file): iterable {
         foreach ($file as $path) {
             $f = basename($path);