Changed Pattern to Rule to be consistent with other implementations

lib/Grammar.php

@@ -9,8 +9,8 @@ use dW\Lit\Grammar\CaptureList,
     dW\Lit\Grammar\Exception,
     dW\Lit\Grammar\GrammarInclude,
     dW\Lit\Grammar\InjectionList,
-    dW\Lit\Grammar\Pattern,
+    dW\Lit\Grammar\Rule,
-    dW\Lit\Grammar\PatternList,
+    dW\Lit\Grammar\RuleList,
     dW\Lit\Grammar\Registry,
     dW\Lit\Grammar\Repository;
 
@@ -26,15 +26,15 @@ class Grammar {
     protected ?string $_firstLineMatch;
     protected ?InjectionList $_injections;
     protected ?string $_name;
-    protected PatternList $_patterns;
+    protected RuleList $_patterns;
     protected ?Repository $_repository;
     protected string $_scopeName;
 
 
-    public function __construct(string $scopeName, PatternList $patterns, ?string $name = null, ?string $contentRegex = null, ?string $firstLineMatch = null, ?InjectionList $injections = null, ?Repository $repository = null) {
+    public function __construct(string $scopeName, RuleList $rules, ?string $name = null, ?string $contentRegex = null, ?string $firstLineMatch = null, ?InjectionList $injections = null, ?Repository $repository = null) {
         $this->_name = $name;
         $this->_scopeName = $scopeName;
-        $this->_patterns = $patterns;
+        $this->_patterns = $rules;
         $this->_contentRegex = $contentRegex;
         $this->_firstLineMatch = $firstLineMatch;
         $this->_injections = $injections;
@@ -65,13 +65,13 @@ class Grammar {
         $contentRegex = (isset($json['contentRegex'])) ? "/{$json['contentRegex']}/" : null;
         $firstLineMatch = (isset($json['firstLineMatch'])) ? "/{$json['firstLineMatch']}/" : null;
 
-        $patterns = self::parseJSONPatternList($json['patterns'], $jsonPath);
+        $rules = self::parseJSONRuleList($json['patterns'], $jsonPath);
 
         $injections = null;
         if (isset($json['injections'])) {
             $injections = [];
             foreach ($json['injections'] as $key => $injection) {
-                $injsections[$key] = (count($injection) === 1 && key($injection) === 'patterns') ?  self::parseJSONPatternList($injection['patterns'], $jsonPath) : self::parseJSONPattern($injection, $jsonPath);
+                $injsections[$key] = (count($injection) === 1 && key($injection) === 'patterns') ?  self::parseJSONRuleList($injection['patterns'], $jsonPath) : self::parseJSONRule($injection, $jsonPath);
             }
 
             if (count($injections) > 0) {
@@ -85,7 +85,7 @@ class Grammar {
         if (isset($json['repository'])) {
             $respository = [];
             foreach ($json['repository'] as $key => $r) {
-                $repository[$key] = (count($r) === 1 && key($r) === 'patterns') ? self::parseJSONPatternList($r['patterns'], $jsonPath) : self::parseJSONPattern($r, $jsonPath);
+                $repository[$key] = (count($r) === 1 && key($r) === 'patterns') ? self::parseJSONRuleList($r['patterns'], $jsonPath) : self::parseJSONRule($r, $jsonPath);
             }
 
             if (count($repository) > 0) {
@@ -95,13 +95,13 @@ class Grammar {
             }
         }
 
-        return new self($scopeName, $patterns, $name, $contentRegex, $firstLineMatch, $injections, $repository);
+        return new self($scopeName, $rules, $name, $contentRegex, $firstLineMatch, $injections, $repository);
     }
 
 
-    protected static function parseJSONPattern(array $pattern, string $jsonPath): GrammarInclude|Pattern|null {
+    protected static function parseJSONRule(array $rule, string $jsonPath): GrammarInclude|Rule|null {
-        if (array_keys($pattern) === [ 'include' ]) {
+        if (array_keys($rule) === [ 'include' ]) {
-            return new GrammarInclude($pattern['include']);
+            return new GrammarInclude($rule['include']);
         }
 
         $p = [
@@ -118,7 +118,7 @@ class Grammar {
         ];
 
         $modified = false;
-        foreach ($pattern as $key => $value) {
+        foreach ($rule as $key => $value) {
             switch ($key) {
                 case 'applyEndPatternLast':
                     if (!is_bool($value) || (!is_int($value) && ($value !== 0 && $value !== 1))) {
@@ -161,7 +161,7 @@ class Grammar {
                     }, array_keys($value));
 
                     $v = array_map(function($n) use ($jsonPath) {
-                        return (count($n) === 1 && key($n) === 'patterns') ? self::parseJSONPatternList($n['patterns'], $jsonPath) : self::parseJSONPattern($n, $jsonPath);
+                        return (count($n) === 1 && key($n) === 'patterns') ? self::parseJSONRuleList($n['patterns'], $jsonPath) : self::parseJSONRule($n, $jsonPath);
                     }, array_values($value));
 
                     $p[$key] = new CaptureList(array_combine($k, $v));
@@ -172,24 +172,24 @@ class Grammar {
                         throw new Exception(Exception::JSON_INVALID_TYPE, 'Array', $key, gettype($value), $jsonPath);
                     }
 
-                    $p[$key] = self::parseJSONPatternList($value, $jsonPath);
+                    $p[$key] = self::parseJSONRuleList($value, $jsonPath);
                     $modified = true;
                 break;
             }
         }
 
-        return ($modified) ? new Pattern(...$p) : null;
+        return ($modified) ? new Rule(...$p) : null;
     }
 
-    protected static function parseJSONPatternList(array $list, string $jsonPath): ?PatternList {
+    protected static function parseJSONRuleList(array $list, string $jsonPath): ?RuleList {
         $result = [];
-        foreach ($list as $pattern) {
+        foreach ($list as $rule) {
-            $p = self::parseJSONPattern($pattern, $jsonPath);
+            $p = self::parseJSONRule($rule, $jsonPath);
             if ($p !== null) {
                 $result[] = $p;
             }
         }
 
-        return (count($result) > 0) ? new PatternList(...$result) : null;
+        return (count($result) > 0) ? new RuleList(...$result) : null;
     }
 }

lib/Grammar/CaptureList.php

@@ -14,8 +14,8 @@ class CaptureList extends ImmutableList {
                 throw new Exception(Exception::LIST_INVALID_TYPE, 'Integer', 'supplied array index', gettype($k));
             }
 
-            if (!$v instanceof GrammarInclude && !$v instanceof Pattern && !$v instanceof PatternList) {
+            if (!$v instanceof GrammarInclude && !$v instanceof Rule && !$v instanceof RuleList) {
-                throw new Exception(Exception::LIST_INVALID_TYPE,  __NAMESPACE__.'\GrammarInclude, '.__NAMESPACE__.'\Pattern, or '.__NAMESPACE__.'\PatternList', 'supplied array value', gettype($v));
+                throw new Exception(Exception::LIST_INVALID_TYPE,  __NAMESPACE__.'\GrammarInclude, '.__NAMESPACE__.'\Rule, or '.__NAMESPACE__.'\RuleList', 'supplied array value', gettype($v));
             }
         }
 

lib/Grammar/Exception.php

@@ -29,6 +29,8 @@ class Exception extends \Exception {
     const LIST_INVALID_INDEX = 301;
     const LIST_INVALID_TYPE = 302;
 
+    const GRAMMAR_MISSING = 400;
+
     protected static $messages = [
         100 => 'Invalid error code',
         101 => 'Unknown error; escaping',
@@ -50,7 +52,9 @@ class Exception extends \Exception {
 
         300 => '%s is immutable',
         301 => 'Invalid %1$s index at offset %2$s',
-        302 => '%1$s expected for %2$s, found %3$s'
+        302 => '%1$s expected for %2$s, found %3$s',
+
+        400 => 'A grammar for scope %s does not exist; one may be added using Grammar\\Registry::set'
     ];
 
     public function __construct(int $code, ...$args) {

lib/Grammar/InjectionList.php

@@ -11,4 +11,4 @@ namespace dW\Lit\Grammar;
  * new grammar; instead of applying to an entire file it's instead applied to a
  * specific scope selector.
  */
-class InjectionList extends NamedPatternListList {}
+class InjectionList extends NamedRuleListList {}

lib/Grammar/NamedPatternListList.php → lib/Grammar/NamedRuleListList.php

@@ -6,7 +6,7 @@
 declare(strict_types=1);
 namespace dW\Lit\Grammar;
 
-abstract class NamedPatternListList extends ImmutableList {
+abstract class NamedRuleListList extends ImmutableList {
     public function __construct(array $array) {
         /* This shit is here because PHP doesn't have array types or generics :) */
         foreach ($array as $k => $v) {
@@ -14,8 +14,8 @@ abstract class NamedPatternListList extends ImmutableList {
                 throw new Exception(Exception::LIST_INVALID_TYPE, 'String', 'supplied array index', gettype($k));
             }
 
-            if (!$v instanceof GrammarInclude && !$v instanceof Pattern && !$v instanceof PatternList) {
+            if (!$v instanceof GrammarInclude && !$v instanceof Rule && !$v instanceof RuleList) {
-                throw new Exception(Exception::LIST_INVALID_TYPE,  __NAMESPACE__.'\GrammarInclude, '.__NAMESPACE__.'\Pattern, or '.__NAMESPACE__.'\PatternList', 'supplied array value', gettype($v));
+                throw new Exception(Exception::LIST_INVALID_TYPE,  __NAMESPACE__.'\GrammarInclude, '.__NAMESPACE__.'\Rule, or '.__NAMESPACE__.'\RuleList', 'supplied array value', gettype($v));
             }
         }
 

lib/Grammar/Repository.php

@@ -10,4 +10,4 @@ namespace dW\Lit\Grammar;
  * An immutable list of rules which can be included from other places in the
  * grammar; The key is the name of the rule and the value is the actual rule.
  */
-class Repository extends NamedPatternListList {}
+class Repository extends NamedRuleListList {}

lib/Grammar/Pattern.php → lib/Grammar/Rule.php

@@ -8,8 +8,8 @@ namespace dW\Lit\Grammar;
 use dW\Lit\FauxReadOnly;
 use dW\Lit\Grammar;
 
-/** Rule responsible for matching a portion of the document */
+/** Contains patterns responsible for matching a portion of the document */
-class Pattern {
+class Rule {
     use FauxReadOnly;
 
     protected bool $_applyEndPatternLast = false;
@@ -21,16 +21,16 @@ class Pattern {
     protected ?CaptureList $_endCaptures;
     protected ?string $_match;
     protected ?string $_name;
-    protected ?PatternList $_patterns;
+    protected ?RuleList $_patterns;
 
 
-    public function __construct(?string $name = null, ?string $contentName = null, ?string $begin = null, ?string $end = null, ?string $match = null, ?PatternList $patterns = null, ?CaptureList $captures = null, ?CaptureList $beginCaptures = null, ?CaptureList $endCaptures = null, bool $applyEndPatternLast = false) {
+    public function __construct(?string $name = null, ?string $contentName = null, ?string $begin = null, ?string $end = null, ?string $match = null, ?RuleList $rules = null, ?CaptureList $captures = null, ?CaptureList $beginCaptures = null, ?CaptureList $endCaptures = null, bool $applyEndPatternLast = false) {
         $this->_name = $name;
         $this->_contentName = $contentName;
         $this->_begin = $begin;
         $this->_end = $end;
         $this->_match = $match;
-        $this->_patterns = $patterns;
+        $this->_patterns = $rules;
         $this->_captures = $captures;
         $this->_beginCaptures = $beginCaptures;
         $this->_endCaptures = $endCaptures;

lib/Grammar/PatternList.php → lib/Grammar/RuleList.php

@@ -7,8 +7,8 @@ declare(strict_types=1);
 namespace dW\Lit\Grammar;
 
 /** Immutable list of pattern rules */
-class PatternList extends ImmutableList {
+class RuleList extends ImmutableList {
-    public function __construct(Pattern|GrammarInclude ...$values) {
+    public function __construct(Rule|GrammarInclude ...$values) {
         parent::__construct(...$values);
     }
 }

lib/Highlight.php

@@ -0,0 +1,34 @@
+<?php
+/** @license MIT
+ * Copyright 2021 Dustin Wilson et al.
+ * See LICENSE file for details */
+
+declare(strict_types=1);
+namespace dW\Lit;
+use dW\Lit\Grammar\Exception,
+    dW\Lit\Grammar\Registry as GrammarRegistry;
+
+class Highlight {
+    public static function withFile(string $filepath, string $scopeName) {
+        return self::highlight(Data::fileToGenerator($filepath), $scopeName);
+    }
+
+    public static function withString(string $string, string $scopeName) {
+        return self::highlight(Data::stringToGenerator($string), $scopeName);
+    }
+
+
+    protected static function highlight(\Generator $data, string $scopeName) {
+        $grammar = GrammarRegistry::get($scopeName);
+        if ($grammar === false) {
+            throw new Exception(Exception::GRAMMAR_MISSING, $scopeName);
+        }
+
+        $tokenizer = new Tokenizer($data, $grammar);
+        $tokenList = $tokenizer->tokenize();
+
+        foreach ($tokenList as $lineNumber => $line) {
+            echo "$lineNumber: $line\n";
+        }
+    }
+}

lib/Highlighter.php

@@ -1,17 +0,0 @@
-<?php
-/** @license MIT
- * Copyright 2021 Dustin Wilson et al.
- * See LICENSE file for details */
-
-declare(strict_types=1);
-namespace dW\Lit;
-
-class Highlighter {
-    public static function highlightFile(string $filepath, string $scopeName) {
-        $data = Data::fileToGenerator($filepath);
-    }
-
-    public static function highlightString(string $string, string $scopeName) {
-        $data = Data::stringToGenerator($string);
-    }
-}

lib/Tokenizer.php

@@ -8,14 +8,21 @@ namespace dW\Lit;
 
 class Tokenizer {
     protected \Generator $data;
+    protected Grammar $grammar;
 
 
     public function __construct(\Generator $data, Grammar $grammar) {
         $this->data = $data;
+        $this->grammar = $grammar;
     }
 
+
     public function tokenize(): \Generator {
+        $ruleStack = [ $this->grammar ];
+
         foreach ($this->data as $lineNumber => $line) {
+
+
             yield $lineNumber => $line;
         }
     }

run

@@ -53,10 +53,10 @@
 ##?         grammars.
 ##?
 ##?     build
-##?         Build the language grammars. Puts them into data/.
+##?         Builds the language grammars; puts them into data/.
 ##?
 ##?     needed
-##?         List the language grammars that are included in the current grammars
+##?         Lists the language grammars that are included in the current grammars
 ##?         but are not in the data/ folder.
 ##?