Basic skeleton of test suite

.gitignore

@@ -67,3 +67,6 @@ $RECYCLE.BIN/
 .nfs*
 
 /vendor/
+/vendor-bin/*/vendor
+/tests/html5lib-tests
+/tests/.phpunit.result.cache

RoboFile.php

@@ -0,0 +1,129 @@
+<?php
+
+use Robo\Result;
+
+const BASE = __DIR__.\DIRECTORY_SEPARATOR;
+const BASE_TEST = BASE."tests".\DIRECTORY_SEPARATOR;
+define("IS_WIN", defined("PHP_WINDOWS_VERSION_MAJOR"));
+define("IS_MAC", php_uname("s") === "Darwin");
+
+function norm(string $path): string {
+    $out = realpath($path);
+    if (!$out) {
+        $out = str_replace(["/", "\\"], \DIRECTORY_SEPARATOR, $path);
+    }
+    return $out;
+}
+
+class RoboFile extends \Robo\Tasks {
+    /** Runs the typical test suite
+     *
+     * Arguments passed to the task are passed on to PHPUnit. Thus one may, for
+     * example, run the following command and get the expected results:
+     *
+     * ./robo test --testsuite Tokenizer --exclude-group slow --testdox
+     *
+     * Please see the PHPUnit documentation for available options.
+    */
+    public function test(array $args): Result {
+        return $this->runTests(escapeshellarg(\PHP_BINARY), "typical", $args);
+    }
+
+    /** Runs the full test suite
+     *
+     * This includes pedantic tests which may help to identify problems.
+     * See help for the "test" task for more details.
+    */
+    public function testFull(array $args): Result {
+        return $this->runTests(escapeshellarg(\PHP_BINARY), "full", $args);
+    }
+
+    /**
+     * Runs a quick subset of the test suite
+     *
+     * See help for the "test" task for more details.
+    */
+    public function testQuick(array $args): Result {
+        return $this->runTests(escapeshellarg(\PHP_BINARY), "quick", $args);
+    }
+
+    /** Produces a code coverage report
+     *
+     * By default this task produces an HTML-format coverage report in
+     * tests/coverage/. Additional reports may be produced by passing
+     * arguments to this task as one would to PHPUnit.
+    */
+    public function coverage(array $args): Result {
+        // run tests with code coverage reporting enabled
+        $exec = $this->findCoverageEngine();
+        return $this->runTests($exec, "coverage", array_merge(["--coverage-html", BASE_TEST."coverage"], $args));
+    }
+
+    /** Produces a code coverage report, with redundant tests
+     *
+     * Depending on the environment, some tests that normally provide
+     * coverage may be skipped, while working alternatives are normally
+     * suppressed for reasons of time. This coverage report will try to
+     * run all tests which may cover code.
+     *
+     * See also help for the "coverage" task for more details.
+    */
+    public function coverageFull(array $args): Result {
+        // run tests with code coverage reporting enabled
+        $exec = $this->findCoverageEngine();
+        return $this->runTests($exec, "typical", array_merge(["--coverage-html", BASE_TEST."coverage"], $args));
+    }
+
+    /** Runs the coding standards fixer */
+    public function clean($opts = ['demo|d' => false]): Result {
+        $t = $this->taskExec(norm(BASE."vendor/bin/php-cs-fixer"));
+        $t->arg("fix");
+        if ($opts['demo']) {
+            $t->args("--dry-run", "--diff")->option("--diff-format", "udiff");
+        }
+        return $t->run();
+    }
+
+    protected function findCoverageEngine(): string {
+        if (IS_WIN) {
+            $dbg = dirname(\PHP_BINARY)."\\phpdbg.exe";
+            $dbg = file_exists($dbg) ? $dbg : "";
+        } else {
+            $dbg = trim(`which phpdbg 2>/dev/null`);
+        }
+        if ($dbg) {
+            return escapeshellarg($dbg)." -qrr";
+        } else {
+            $ext = IS_WIN ? "dll" : (IS_MAC ? "dylib" : "so");
+            return escapeshellarg(\PHP_BINARY)." -d zend_extension=xdebug.$ext";
+        }
+    }
+
+    protected function blackhole(bool $all = false): string {
+        $hole = IS_WIN ? "nul" : "/dev/null";
+        return $all ? ">$hole 2>&1" : "2>$hole";
+    }
+
+    protected function runTests(string $executor, string $set, array $args) : Result {
+        switch ($set) {
+            case "typical":
+                $set = ["--exclude-group", "optional"];
+                break;
+            case "quick":
+                $set = ["--exclude-group", "optional,slow"];
+                break;
+            case "coverage":
+                $set = ["--exclude-group", "optional,coverageOptional"];
+                break;
+            case "full":
+                $set = [];
+                break;
+            default:
+                throw new \Exception;
+        }
+        $execpath = norm(BASE."vendor-bin/phpunit/vendor/phpunit/phpunit/phpunit");
+        $confpath = realpath(BASE_TEST."phpunit.dist.xml") ?: norm(BASE_TEST."phpunit.xml");
+        //$this->taskServer(8000)->host("localhost")->dir(BASE_TEST."docroot")->rawArg("-n")->arg(BASE_TEST."server.php")->rawArg($this->blackhole())->background()->run();
+        return $this->taskExec($executor)->option("-d", "zend.assertions=1")->arg($execpath)->option("-c", $confpath)->args(array_merge($set, $args))->run();
+    }
+}

composer.json

@@ -8,12 +8,21 @@
         "ext-hash": "*",
         "mensbeam/intl": "*"
     },
+    "scripts": {
+        "post-install-cmd": ["@composer bin all install"],
+        "post-update-cmd": ["@composer bin all update"]
+    },
     "license": "MIT",
     "authors": [
         {
             "name": "Dustin Wilson",
             "email": "dustin@dustinwilson.com",
             "homepage": "https://dustinwilson.com/"
+        },
+        {
+            "name": "J. King",
+            "email": "jking@jkingweb.ca",
+            "homepage": "https://jkingweb.ca/"
         }
     ],
     "autoload": {
@@ -27,6 +36,13 @@
         "classmap": ["lib/Token.php"]
     },
     "autoload-dev": {
-        "files": ["lib/Token.php"]
+        "files": ["lib/Token.php"],
+        "psr-4": {
+            "dW\\HTML5\\Test\\": "tests/lib/",
+            "dW\\HTML5\\TestCase\\": "tests/cases/"
+        }
+    },
+    "require-dev": {
+        "bamarni/composer-bin-plugin": "^1.3"
     }
 }

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "530885138877b3d0251bb2a794f03ed6",
+    "content-hash": "7f73d57a54597e507dfcbd1a6d40b779",
     "packages": [
         {
             "name": "mensbeam/intl",
@@ -52,7 +52,47 @@
             "time": "2018-09-15T23:52:16+00:00"
         }
     ],
-    "packages-dev": [],
+    "packages-dev": [
+        {
+            "name": "bamarni/composer-bin-plugin",
+            "version": "v1.3.0",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/bamarni/composer-bin-plugin.git",
+                "reference": "67f9d314dc7ecf7245b8637906e151ccc62b8d24"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/bamarni/composer-bin-plugin/zipball/67f9d314dc7ecf7245b8637906e151ccc62b8d24",
+                "reference": "67f9d314dc7ecf7245b8637906e151ccc62b8d24",
+                "shasum": ""
+            },
+            "require": {
+                "composer-plugin-api": "^1.0"
+            },
+            "require-dev": {
+                "composer/composer": "dev-master",
+                "symfony/console": "^2.5 || ^3.0 || ^4.0"
+            },
+            "type": "composer-plugin",
+            "extra": {
+                "class": "Bamarni\\Composer\\Bin\\Plugin",
+                "branch-alias": {
+                    "dev-master": "1.1-dev"
+                }
+            },
+            "autoload": {
+                "psr-4": {
+                    "Bamarni\\Composer\\Bin\\": "src"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "time": "2019-03-17T12:38:04+00:00"
+        }
+    ],
     "aliases": [],
     "minimum-stability": "stable",
     "stability-flags": [],

robo

@@ -0,0 +1,14 @@
+#! /bin/sh
+base=`dirname "$0"`
+roboCommand="$1"
+if [ $# -eq 0 ]; then
+    "$base/vendor/bin/robo"
+else
+    shift
+    ulimit -n 2048
+    if [ "$1" = "clean" ]; then
+        "$base/vendor/bin/robo" "$roboCommand" "$@"
+    else
+        "$base/vendor/bin/robo" "$roboCommand" -- "$@"
+    fi
+fi

robo.bat

@@ -0,0 +1,21 @@
+@echo off
+setlocal
+set base=%~dp0
+set roboCommand=%1
+
+rem get all arguments except the first
+shift
+set "args="
+:parse
+if "%~1" neq "" (
+  set args=%args% %1
+  shift
+  goto :parse
+)
+if defined args set args=%args:~1%
+
+if "%1"=="clean" (
+    call "%base%vendor\bin\robo" "%roboCommand%" %args%
+) else (
+    call "%base%vendor\bin\robo" "%roboCommand%" -- %args%
+)

tests/bootstrap.php

@@ -0,0 +1,16 @@
+<?php
+declare(strict_types=1);
+namespace dW\HTML5;
+
+const NS_BASE = __NAMESPACE__."\\";
+define(NS_BASE."BASE", dirname(__DIR__).DIRECTORY_SEPARATOR);
+const DOCROOT = BASE."tests".DIRECTORY_SEPARATOR."docroot".DIRECTORY_SEPARATOR;
+ini_set("memory_limit", "-1");
+ini_set("zend.assertions", "1");
+ini_set("assert.exception", "true");
+error_reporting(\E_ALL);
+require_once BASE."vendor".DIRECTORY_SEPARATOR."autoload.php";
+
+if (function_exists("xdebug_set_filter")) {
+    xdebug_set_filter(\XDEBUG_FILTER_CODE_COVERAGE, \XDEBUG_PATH_WHITELIST, [BASE."lib/"]);
+}

tests/cases/TestTokenizer.php

@@ -0,0 +1,31 @@
+<?php
+declare(strict_types=1);
+namespace dW\HTML5\TestCase;
+
+use dW\HTML5\Data;
+use dW\HTML5\EOFToken;
+use dW\HTML5\OpenElementsStack;
+use dW\HTML5\Tokenizer;
+
+class TestTokenizer extends \dW\HTML5\Test\StandardTest {
+    /** @dataProvider provideStandardTokenizerTests */
+    public function testStandardTokenizerTests(string $input, array $expected, int $state, string $open = null, array $errors) {
+        $data = new Data($input);
+        $stack = new OpenElementsStack();
+        if ($open) {
+            $stack[] = $open;
+        }
+        $tokenizer = new Tokenizer($data, $stack);
+        $tokenizer->state = $state;
+        $actual = [];
+        while (!($t = $tokenizer->createToken()) instanceof EOFToken) {
+            $actual[] = $t;
+        }
+        $this->assertEquals($expected, $actual);
+    }
+
+    public function provideStandardTokenizerTests() {
+        $out = $this->makeTokenTests(__DIR__."/../html5lib-tests/tokenizer/test1.test");
+        return array_slice(iterator_to_array($out), 0, 3);
+    }
+}

tests/lib/StandardTest.php

@@ -0,0 +1,69 @@
+<?php
+declare(strict_types=1);
+namespace dW\HTML5\Test;
+
+use dW\HTML5\CharacterToken;
+use dW\HTML5\CommentToken;
+use dW\HTML5\DOCTYPEToken;
+use dW\HTML5\EndTagToken;
+use dW\HTML5\StartTagToken;
+use dW\HTML5\Tokenizer;
+
+class StandardTest extends \PHPUnit\Framework\TestCase {
+    const STATE_MAP = [
+        'Data state'          => Tokenizer::DATA_STATE,
+        'PLAINTEXT state'     => Tokenizer::PLAINTEXT_STATE,
+        'RCDATA state'        => Tokenizer::RCDATA_STATE,
+        'RAWTEXT state'       => Tokenizer::RAWTEXT_STATE,
+        'Script data state'   => Tokenizer::SCRIPT_DATA_STATE,
+        'CDATA section state' => Tokenizer::CDATA_SECTION_STATE,
+    ];
+
+    protected function makeTokenTests(string $file): iterable {
+        $testSet = json_decode(file_get_contents($file), true);
+        $index = 0;
+        foreach ($testSet['tests'] as $test) {
+            $index += 1;
+            $test['initialStates'] = $test['initialStates'] ?? ["Data state"];
+            for ($a = 0; $a < sizeof($test['initialStates']); $a++) {
+                $tokens = [];
+                foreach ($test['output'] as $token) {
+                    switch ($token[0]) {
+                        case "DOCTYPE":
+                            $t = new DOCTYPEToken((string) $token[1], (string) $token[2], (string) $token[3]);
+                            $t->forceQuirks = !$token[4];
+                            $tokens[] = $t;
+                            break;
+                        case "StartTag":
+                            $t = new StartTagToken($token[1], $token[3] ?? false);
+                            foreach ($token[2] ?? [] as $name => $value) {
+                                $t->setAttribute($name, $value);
+                            }
+                            $tokens[] = $t;
+                            break;
+                        case "EndTag":
+                            $tokens[] = new EndTagToken($token[1]);
+                            break;
+                        case "Character":
+                            $tokens[] = new CharacterToken($token[1]);
+                            break;
+                        case "Comment":
+                            $tokens[] = new CommentToken($token[1]);
+                            break;
+                        default:
+                            throw new \Exception("Token type '{$token[0]}' not implemented in standard test interpreter");
+                    }
+                    unset($t);
+                    //yield "#$index {$test['description']} ({$test['initialStates'][$a]})" => [
+                    yield [
+                        $test['input'],                                 // input
+                        $tokens,                                        // output
+                        self::STATE_MAP[$test['initialStates'][$a]],    // initial state
+                        $test['lastStartTag'] ?? null,                  // open element, if any
+                        $test['errors'] ?? [],                          // errors, if any
+                    ];
+                }
+            }
+        }
+    }
+}

tests/phpunit.dist.xml

@@ -0,0 +1,26 @@
+<?xml version="1.0"?>
+<phpunit
+    colors="true"
+    bootstrap="bootstrap.php"
+    convertErrorsToExceptions="false"
+    convertNoticesToExceptions="false"
+    convertWarningsToExceptions="false"
+    beStrictAboutTestsThatDoNotTestAnything="true"
+    forceCoversAnnotation="true"
+    executionOrder="defects"
+>
+
+<!--    beStrictAboutOutputDuringTests="true" -->
+
+<filter>
+  <whitelist processUncoveredFilesFromWhitelist="true">
+    <directory suffix=".php">../lib</directory>
+  </whitelist>
+</filter>
+
+<testsuites>
+    <testsuite name="Tokenizer">
+        <file>cases/TestTokenizer.php</file>
+    </testsuite>
+</testsuites>
+</phpunit>

vendor-bin/phpunit/composer.json

@@ -0,0 +1,5 @@
+{
+    "require": {
+        "phpunit/phpunit": "^8.5"
+    }
+}

vendor-bin/robo/composer.json

@@ -0,0 +1,5 @@
+{
+    "require": {
+        "consolidation/robo": "^2.0"
+    }
+}