Includes now work like lazy WeakReferences except self

lib/Grammar.php

@@ -5,14 +5,18 @@
 
 declare(strict_types=1);
 namespace dW\Lit;
-use dW\Lit\Grammar\CaptureList,
+use dW\Lit\Grammar\BaseReference,
+    dW\Lit\Grammar\CaptureList,
     dW\Lit\Grammar\Exception,
-    dW\Lit\Grammar\GrammarInclude,
+    dW\Lit\Grammar\GrammarReference,
     dW\Lit\Grammar\InjectionList,
-    dW\Lit\Grammar\Rule,
-    dW\Lit\Grammar\RuleList,
+    dW\Lit\Grammar\Pattern,
+    dW\Lit\Grammar\PatternList,
+    dW\Lit\Grammar\Reference,
     dW\Lit\Grammar\Registry,
-    dW\Lit\Grammar\Repository;
+    dW\Lit\Grammar\Repository,
+    dW\Lit\Grammar\RepositoryReference,
+    dW\Lit\Grammar\SelfReference;
 
 
 /**
@@ -26,52 +30,76 @@ class Grammar {
     protected ?string $_firstLineMatch;
     protected ?InjectionList $_injections;
     protected ?string $_name;
-    protected RuleList $_patterns;
+    protected ?\WeakReference $_ownerGrammar;
+    protected ?PatternList $_patterns;
     protected ?Repository $_repository;
-    protected string $_scopeName;
+    protected ?string $_scopeName;
 
 
-    public function __construct(string $scopeName, RuleList $rules, ?string $name = null, ?string $contentRegex = null, ?string $firstLineMatch = null, ?InjectionList $injections = null, ?Repository $repository = null) {
+    public function __construct(?string $scopeName = null, ?PatternList $patterns = null, ?string $name = null, ?string $contentRegex = null, ?string $firstLineMatch = null, ?InjectionList $injections = null, ?Repository $repository = null, ?Grammar $ownerGrammar = null) {
         $this->_name = $name;
         $this->_scopeName = $scopeName;
-        $this->_patterns = $rules;
+        $this->_patterns = $patterns;
         $this->_contentRegex = $contentRegex;
         $this->_firstLineMatch = $firstLineMatch;
         $this->_injections = $injections;
         $this->_repository = $repository;
+        $this->_ownerGrammar = (is_null($ownerGrammar)) ? null : \WeakReference::create($ownerGrammar);
     }
 
-    /** Parses an Atom JSON grammar and converts to a Grammar object */
-    public static function fromJSON(string $jsonPath): self {
-        if (!is_file($jsonPath)) {
-            throw new Exception(Exception::JSON_INVALID_FILE, $jsonPath);
+
+    /** Clones the supplied grammar with this grammar set as its owner grammar */
+    public function adoptGrammar(self $grammar): self {
+        return new self($grammar->name, $grammar->scopeName, $grammar->patterns, $grammar->contentRegex, $grammar->firstLineMatch, $grammar->injections, $this, $grammar->repository);
+    }
+
+
+    /** Imports an Atom JSON grammar into the Grammar object */
+    public function loadJSON(string $filename) {
+        if (!is_file($filename)) {
+            throw new Exception(Exception::JSON_INVALID_FILE, $filename);
         }
 
-        $json = json_decode(file_get_contents($jsonPath), true);
+        $json = json_decode(file_get_contents($filename), true);
         if ($json === null) {
-            throw new Exception(json_last_error() + 200, $jsonPath);
+            throw new Exception(json_last_error() + 200, $filename);
         }
 
         if (!isset($json['scopeName'])) {
-            throw new Exception(Exception::JSON_MISSING_PROPERTY, $jsonPath, 'scopeName');
+            throw new Exception(Exception::JSON_MISSING_PROPERTY, $filename, 'scopeName');
         }
 
         if (!isset($json['patterns'])) {
-            throw new Exception(Exception::JSON_MISSING_PROPERTY, $jsonPath, 'patterns');
+            throw new Exception(Exception::JSON_MISSING_PROPERTY, $filename, 'patterns');
         }
 
-        $name = $json['name'] ?? null;
-        $scopeName = $json['scopeName'];
-        $contentRegex = (isset($json['contentRegex'])) ? "/{$json['contentRegex']}/" : null;
-        $firstLineMatch = (isset($json['firstLineMatch'])) ? "/{$json['firstLineMatch']}/" : null;
+        $this->_name = $json['name'] ?? null;
+        $this->_scopeName = $json['scopeName'];
+        $this->_contentRegex = (isset($json['contentRegex'])) ? "/{$json['contentRegex']}/" : null;
+        $this->_firstLineMatch = (isset($json['firstLineMatch'])) ? "/{$json['firstLineMatch']}/" : null;
 
-        $rules = self::parseJSONRuleList($json['patterns'], $jsonPath);
+        $repository = null;
+        if (isset($json['repository'])) {
+            $respository = [];
+            foreach ($json['repository'] as $key => $r) {
+                $repository[$key] = $this->parseJSONPattern($r, $filename);
+            }
+
+            if (count($repository) > 0) {
+                $repository = new Repository($repository);
+            } else {
+                $repository = null;
+            }
+        }
+        $this->_repository = $repository;
+
+        $this->_patterns = $this->parseJSONPatternList($json['patterns'], $filename);
 
         $injections = null;
         if (isset($json['injections'])) {
             $injections = [];
             foreach ($json['injections'] as $key => $injection) {
-                $injsections[$key] = (count($injection) === 1 && key($injection) === 'patterns') ?  self::parseJSONRuleList($injection['patterns'], $jsonPath) : self::parseJSONRule($injection, $jsonPath);
+                $injections[$key] = $this->parseJSONPattern($injection, $filename);
             }
 
             if (count($injections) > 0) {
@@ -80,30 +108,23 @@ class Grammar {
                 $injections = null;
             }
         }
+        $this->_injections = $injections;
+    }
 
-        $repository = null;
-        if (isset($json['repository'])) {
-            $respository = [];
-            foreach ($json['repository'] as $key => $r) {
-                $repository[$key] = (count($r) === 1 && key($r) === 'patterns') ? self::parseJSONRuleList($r['patterns'], $jsonPath) : self::parseJSONRule($r, $jsonPath);
-            }
 
-            if (count($repository) > 0) {
-                $repository = new Repository($repository);
+    protected function parseJSONPattern(array $pattern, string $filename): Pattern|Reference|\WeakReference|null {
+        if (isset($pattern['include'])) {
+            if ($pattern['include'][0] === '#') {
+                return new RepositoryReference(substr($pattern['include'], 1), $this);
+            } elseif ($pattern['include'] === '$base') {
+                return new BaseReference($this);
+            } elseif ($pattern['include'] === '$self') {
+                return \WeakReference::create($this);
             } else {
-                $repository = null;
+                return new GrammarReference($pattern['include'], $this);
             }
         }
 
-        return new self($scopeName, $rules, $name, $contentRegex, $firstLineMatch, $injections, $repository);
-    }
-
-
-    protected static function parseJSONRule(array $rule, string $jsonPath): GrammarInclude|Rule|null {
-        if (array_keys($rule) === [ 'include' ]) {
-            return new GrammarInclude($rule['include']);
-        }
-
         $p = [
             'name' => null,
             'contentName' => null,
@@ -118,11 +139,11 @@ class Grammar {
         ];
 
         $modified = false;
-        foreach ($rule as $key => $value) {
+        foreach ($pattern as $key => $value) {
             switch ($key) {
                 case 'applyEndPatternLast':
                     if (!is_bool($value) || (!is_int($value) && ($value !== 0 && $value !== 1))) {
-                        throw new Exception(Exception::JSON_INVALID_TYPE, 'Boolean, 0, or 1', 'applyEndPatternLast', gettype($value), $jsonPath);
+                        throw new Exception(Exception::JSON_INVALID_TYPE, 'Boolean, 0, or 1', 'applyEndPatternLast', gettype($value), $filename);
                     }
 
                     $value = (bool)$value;
@@ -141,27 +162,27 @@ class Grammar {
                 case 'beginCaptures':
                 case 'endCaptures':
                     if (!is_array($value)) {
-                        throw new Exception(Exception::JSON_INVALID_TYPE, 'Array', $key, gettype($value), $jsonPath);
+                        throw new Exception(Exception::JSON_INVALID_TYPE, 'Array', $key, gettype($value), $filename);
                     }
 
                     if (count($value) === 0) {
                         continue 2;
                     }
 
-                    $k = array_map(function($n) use ($jsonPath) {
+                    $k = array_map(function($n) use ($filename) {
                         if (is_int($n)) {
                             return $n;
                         }
 
                         if (strspn($n, '0123456789') !== strlen($n)) {
-                            throw new Exception(Exception::JSON_INVALID_TYPE, 'Integer', 'capture list index', $n, $jsonPath);
+                            throw new Exception(Exception::JSON_INVALID_TYPE, 'Integer', 'capture list index', $n, $filename);
                         }
 
                         return (int)$n;
                     }, array_keys($value));
 
-                    $v = array_map(function($n) use ($jsonPath) {
-                        return (count($n) === 1 && key($n) === 'patterns') ? self::parseJSONRuleList($n['patterns'], $jsonPath) : self::parseJSONRule($n, $jsonPath);
+                    $v = array_map(function($n) use ($filename) {
+                        return $this->parseJSONPattern($n, $filename);
                     }, array_values($value));
 
                     $p[$key] = new CaptureList(array_combine($k, $v));
@@ -169,27 +190,27 @@ class Grammar {
                 break;
                 case 'patterns':
                     if (!is_array($value)) {
-                        throw new Exception(Exception::JSON_INVALID_TYPE, 'Array', $key, gettype($value), $jsonPath);
+                        throw new Exception(Exception::JSON_INVALID_TYPE, 'Array', $key, gettype($value), $filename);
                     }
 
-                    $p[$key] = self::parseJSONRuleList($value, $jsonPath);
+                    $p[$key] = $this->parseJSONPatternList($value, $filename);
                     $modified = true;
                 break;
             }
         }
 
-        return ($modified) ? new Rule(...$p) : null;
+        return ($modified) ? new Pattern(...$p) : null;
     }
 
-    protected static function parseJSONRuleList(array $list, string $jsonPath): ?RuleList {
+    protected function parseJSONPatternList(array $list, string $filename): Pattern|PatternList|null {
         $result = [];
-        foreach ($list as $rule) {
-            $p = self::parseJSONRule($rule, $jsonPath);
+        foreach ($list as $pattern) {
+            $p = $this->parseJSONPattern($pattern, $filename);
             if ($p !== null) {
                 $result[] = $p;
             }
         }
 
-        return (count($result) > 0) ? new RuleList(...$result) : null;
+        return (count($result) > 0) ? new PatternList(...$result) : null;
     }
 }

lib/Grammar/BaseReference.php

@@ -0,0 +1,36 @@
+<?php
+/** @license MIT
+ * Copyright 2021 Dustin Wilson et al.
+ * See LICENSE file for details */
+
+declare(strict_types=1);
+namespace dW\Lit\Grammar;
+use dW\Lit\Grammar;
+
+/**
+ * Acts as a sort of lazy weak reference for a base grammar in a grammar.
+ */
+class BaseReference extends Reference {
+    protected \WeakReference $grammar;
+    protected ?\WeakReference $object;
+
+
+    public function __construct(Grammar $grammar) {
+        $this->grammar = \WeakReference::create($grammar);
+    }
+
+
+    public function get(): Grammar {
+        if ($this->object !== null) {
+            return $this->object->get();
+        }
+
+        $grammar = $this->grammar->get();
+        do {
+            $result = $grammar;
+        } while ($grammar = $grammar->ownerGrammar);
+
+        $this->object = $result;
+        return $result->get();
+    }
+}

lib/Grammar/CaptureList.php

@@ -8,14 +8,19 @@ namespace dW\Lit\Grammar;
 
 class CaptureList extends ImmutableList {
     public function __construct(array $array) {
-        /* This shit is here because PHP doesn't have array types or generics :) */
+        // This shit is here because PHP doesn't have array types or generics :)
         foreach ($array as $k => $v) {
             if (!is_int($k)) {
                 throw new Exception(Exception::LIST_INVALID_TYPE, 'Integer', 'supplied array index', gettype($k));
             }
 
-            if (!$v instanceof GrammarInclude && !$v instanceof Rule && !$v instanceof RuleList) {
-                throw new Exception(Exception::LIST_INVALID_TYPE,  __NAMESPACE__.'\GrammarInclude, '.__NAMESPACE__.'\Rule, or '.__NAMESPACE__.'\RuleList', 'supplied array value', gettype($v));
+            if (!$v instanceof Pattern && !$v instanceof PatternList && !$v instanceof Reference && !$v instanceof \WeakReference) {
+                $type = gettype($v);
+                if ($type === 'object') {
+                    $type = get_class($v);
+                }
+
+                throw new Exception(Exception::LIST_INVALID_TYPE,  __NAMESPACE__.'\Pattern, '.__NAMESPACE__.'\PatternList, '.__NAMESPACE__.'\Reference, or \WeakReference', 'supplied array value', $type);
             }
         }
 

lib/Grammar/GrammarReference.php

@@ -0,0 +1,44 @@
+<?php
+/** @license MIT
+ * Copyright 2021 Dustin Wilson et al.
+ * See LICENSE file for details */
+
+declare(strict_types=1);
+namespace dW\Lit\Grammar;
+use dW\Lit\FauxReadOnly,
+    dW\Lit\Grammar;
+
+/**
+ * Acts as a sort of lazy reference for entire grammars in grammars.
+ */
+class GrammarReference extends Reference {
+    use FauxReadOnly;
+
+    protected ?Grammar $object;
+    protected \WeakReference $ownerGrammar;
+    protected string $_scopeName;
+
+
+    public function __construct(string $scopeName, Grammar $ownerGrammar) {
+        $this->ownerGrammar = \WeakReference::create($ownerGrammar);
+        $this->_scopeName = $scopeName;
+    }
+
+
+    public function get(): Grammar {
+        if ($this->object !== null) {
+            return $this->object;
+        } elseif ($this->object === false) {
+            return null;
+        }
+
+        $grammar = Registry::get($this->_scopeName);
+        if ($grammar === null) {
+            $this->object = false;
+            return null;
+        }
+
+        $this->object = $this->ownerGrammar->get()->adopt($grammar);
+        return $this->object;
+    }
+}

lib/Grammar/InjectionList.php

@@ -11,4 +11,4 @@ namespace dW\Lit\Grammar;
  * new grammar; instead of applying to an entire file it's instead applied to a
  * specific scope selector.
  */
-class InjectionList extends NamedRuleListList {}
+class InjectionList extends NamedPatternListList {}

lib/Grammar/NamedPatternListList.php

@@ -0,0 +1,32 @@
+<?php
+/** @license MIT
+ * Copyright 2021 Dustin Wilson et al.
+ * See LICENSE file for details */
+
+declare(strict_types=1);
+namespace dW\Lit\Grammar;
+use dW\Lit\FauxReadOnly;
+
+abstract class NamedPatternListList extends ImmutableList {
+    use FauxReadOnly;
+
+    public function __construct(array $array) {
+        // This shit is here because PHP doesn't have array types or generics :)
+        foreach ($array as $k => $v) {
+            if (!is_string($k)) {
+                throw new Exception(Exception::LIST_INVALID_TYPE, 'String', 'supplied array index', gettype($k));
+            }
+
+            if (!$v instanceof Pattern && !$v instanceof PatternList && !$v instanceof Reference && !$v instanceof \WeakReference) {
+                $type = gettype($v);
+                if ($type === 'object') {
+                    $type = get_class($v);
+                }
+
+                throw new Exception(Exception::LIST_INVALID_TYPE,  __NAMESPACE__.'\Pattern, '.__NAMESPACE__.'\PatternList, '.__NAMESPACE__.'\Reference, or \WeakReference', 'supplied array value', $type);
+            }
+        }
+
+        $this->storage = $array;
+    }
+}

lib/Grammar/NamedRuleListList.php

@@ -1,24 +0,0 @@
-<?php
-/** @license MIT
- * Copyright 2021 Dustin Wilson et al.
- * See LICENSE file for details */
-
-declare(strict_types=1);
-namespace dW\Lit\Grammar;
-
-abstract class NamedRuleListList extends ImmutableList {
-    public function __construct(array $array) {
-        /* This shit is here because PHP doesn't have array types or generics :) */
-        foreach ($array as $k => $v) {
-            if (!is_string($k)) {
-                throw new Exception(Exception::LIST_INVALID_TYPE, 'String', 'supplied array index', gettype($k));
-            }
-
-            if (!$v instanceof GrammarInclude && !$v instanceof Rule && !$v instanceof RuleList) {
-                throw new Exception(Exception::LIST_INVALID_TYPE,  __NAMESPACE__.'\GrammarInclude, '.__NAMESPACE__.'\Rule, or '.__NAMESPACE__.'\RuleList', 'supplied array value', gettype($v));
-            }
-        }
-
-        $this->storage = $array;
-    }
-}

lib/Grammar/Rule.php → lib/Grammar/Pattern.php

@@ -9,7 +9,7 @@ use dW\Lit\FauxReadOnly;
 use dW\Lit\Grammar;
 
 /** Contains patterns responsible for matching a portion of the document */
-class Rule {
+class Pattern {
     use FauxReadOnly;
 
     protected bool $_applyEndPatternLast = false;
@@ -21,16 +21,16 @@ class Rule {
     protected ?CaptureList $_endCaptures;
     protected ?string $_match;
     protected ?string $_name;
-    protected ?RuleList $_patterns;
+    protected ?PatternList $_patterns;
 
 
-    public function __construct(?string $name = null, ?string $contentName = null, ?string $begin = null, ?string $end = null, ?string $match = null, ?RuleList $rules = null, ?CaptureList $captures = null, ?CaptureList $beginCaptures = null, ?CaptureList $endCaptures = null, bool $applyEndPatternLast = false) {
+    public function __construct(?string $name = null, ?string $contentName = null, ?string $begin = null, ?string $end = null, ?string $match = null, ?PatternList $patterns = null, ?CaptureList $captures = null, ?CaptureList $beginCaptures = null, ?CaptureList $endCaptures = null, bool $applyEndPatternLast = false) {
         $this->_name = $name;
         $this->_contentName = $contentName;
         $this->_begin = $begin;
         $this->_end = $end;
         $this->_match = $match;
-        $this->_patterns = $rules;
+        $this->_patterns = $patterns;
         $this->_captures = $captures;
         $this->_beginCaptures = $beginCaptures;
         $this->_endCaptures = $endCaptures;

lib/Grammar/RuleList.php → lib/Grammar/PatternList.php

@@ -5,10 +5,11 @@
 
 declare(strict_types=1);
 namespace dW\Lit\Grammar;
+use dW\Lit\Grammar;
 
 /** Immutable list of pattern rules */
-class RuleList extends ImmutableList {
-    public function __construct(Rule|GrammarInclude ...$values) {
+class PatternList extends ImmutableList {
+    public function __construct(Pattern|Reference|\WeakReference ...$values) {
         parent::__construct(...$values);
     }
 }

lib/Grammar/Reference.php

@@ -0,0 +1,12 @@
+<?php
+/** @license MIT
+ * Copyright 2021 Dustin Wilson et al.
+ * See LICENSE file for details */
+
+declare(strict_types=1);
+namespace dW\Lit\Grammar;
+
+/**
+ * Acts as a sort of lazy reference for including self in a grammar.
+ */
+abstract class Reference {}

lib/Grammar/Registry.php

@@ -30,10 +30,10 @@ class Registry implements \IteratorAggregate {
         if (array_key_exists($scopeName, self::$storage)) {
             return self::$storage[$scopeName];
         } else {
-            $jsonPath = __DIR__ . "/../../data/$scopeName.json";
-            if (file_exists($jsonPath)) {
-                $grammar = Grammar::fromJSON($jsonPath);
-                self::set($scopeName, $grammar);
+            $filename = __DIR__ . "/../../data/$scopeName.json";
+            if (file_exists($filename)) {
+                $grammar = new Grammar();
+                $grammar->loadJSON($filename);
                 return $grammar;
             }
         }

lib/Grammar/Repository.php

@@ -10,4 +10,4 @@ namespace dW\Lit\Grammar;
  * An immutable list of rules which can be included from other places in the
  * grammar; The key is the name of the rule and the value is the actual rule.
  */
-class Repository extends NamedRuleListList {}
+class Repository extends NamedPatternListList {}

lib/Grammar/RepositoryReference.php

@@ -0,0 +1,44 @@
+<?php
+/** @license MIT
+ * Copyright 2021 Dustin Wilson et al.
+ * See LICENSE file for details */
+
+declare(strict_types=1);
+namespace dW\Lit\Grammar;
+use dW\Lit\FauxReadOnly,
+    dW\Lit\Grammar;
+
+/**
+ * Acts as a sort of lazy reference for repository items in grammars.
+ */
+class RepositoryReference extends Reference {
+    use FauxReadOnly;
+
+    protected \WeakReference $grammar;
+    protected string $_name;
+    protected PatternList|Pattern|null|false $object;
+
+
+    public function __construct(string $name, Grammar $grammar) {
+        $this->_name = $name;
+        $this->grammar = \WeakReference::create($grammar);
+    }
+
+
+    public function get(): PatternList|Pattern {
+        if ($this->object !== null) {
+            return $this->object;
+        } elseif ($this->object === false) {
+            return null;
+        }
+
+        $grammar = $this->grammar->get();
+        if (!isset($grammar->repository[$this->name])) {
+            $this->object = false;
+            return null;
+        }
+
+        $this->object = $grammar->repository[$this->name];
+        return $this->object;
+    }
+}

lib/Tokenizer.php

@@ -21,8 +21,6 @@ class Tokenizer {
         $ruleStack = [ $this->grammar ];
 
         foreach ($this->data as $lineNumber => $line) {
-
-
             yield $lineNumber => $line;
         }
     }