MensBeam
/
HTML-Parser
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Emit whitespace-only character tokens 

This makes tree building simpler in certain circumstances
ns
J. King 3 years ago
parent
504731cba0
commit
1dc3d9c23e
4 changed files with 114 additions and 45 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      lib/Token.php
  2. 72
      lib/Tokenizer.php
  3. 40
      lib/TreeBuilder.php
  4. 5
      tests/cases/TestTokenizer.php

2
lib/Token.php
View File

@ -39,6 +39,8 @@ class CharacterToken extends DataToken {
public const NAME = "Character token";
}
class WhitespaceToken extends CharacterToken {}
class CommentToken extends DataToken {
public const NAME = "Comment token";

72
lib/Tokenizer.php
View File

@ -268,7 +268,12 @@ class Tokenizer {
# Switch to the character reference state.
// DEVIATION: Character reference consumption implemented as a function
return new CharacterToken($this->switchToCharacterReferenceState(self::DATA_STATE));
$outChar = $this->switchToCharacterReferenceState(self::DATA_STATE);
if (strspn($outChar, Data::WHITESPACE)) {
return new WhitespaceToken($outChar); // a character reference is either all whitespace is no whitespace
} else {
return new CharacterToken($outChar);
}
}
# U+003C LESS-THAN SIGN (<)
elseif ($char === '<') {
@ -295,9 +300,13 @@ class Tokenizer {
// Consume all characters that don't match what is above and emit
// that as a character token instead to prevent having to loop back
// through here every single time.
if (strspn($char, Data::WHITESPACE)) {
return new WhitespaceToken($char.$this->data->consumeWhile(Data::WHITESPACE));
} else {
return new CharacterToken($char.$this->data->consumeUntil("&<\0"));
}
}
}
# 13.2.5.2 RCDATA state
elseif ($this->state === self::RCDATA_STATE) {
@ -310,7 +319,12 @@ class Tokenizer {
# Switch to the character reference state.
// DEVIATION: Character reference consumption implemented as a function
return new CharacterToken($this->switchToCharacterReferenceState(self::RCDATA_STATE));
$outChar = $this->switchToCharacterReferenceState(self::RCDATA_STATE);
if (strspn($outChar, Data::WHITESPACE)) {
return new WhitespaceToken($outChar); // a character reference is either all whitespace is no whitespace
} else {
return new CharacterToken($outChar);
}
}
# U+003C LESS-THAN SIGN (<)
elseif ($char === '<') {
@ -337,9 +351,13 @@ class Tokenizer {
// Consume all characters that don't match what is above and emit
// that as a character token instead to prevent having to loop back
// through here every single time.
if (strspn($char, Data::WHITESPACE)) {
return new WhitespaceToken($char.$this->data->consumeWhile(Data::WHITESPACE));
} else {
return new CharacterToken($char.$this->data->consumeUntil("&<\0"));
}
}
}
# 13.2.5.3 RAWTEXT state
elseif ($this->state === self::RAWTEXT_STATE) {
@ -371,9 +389,13 @@ class Tokenizer {
// Consume all characters that don't match what is above and emit
// that as a character token instead to prevent having to loop back
// through here every single time.
if (strspn($char, Data::WHITESPACE)) {
return new WhitespaceToken($char.$this->data->consumeWhile(Data::WHITESPACE));
} else {
return new CharacterToken($char.$this->data->consumeUntil("<\0"));
}
}
}
# 13.2.5.4 Script data state
elseif ($this->state === self::SCRIPT_DATA_STATE) {
@ -405,9 +427,13 @@ class Tokenizer {
// Consume all characters that don't match what is above and emit
// that as a character token instead to prevent having to loop back
// through here every single time.
if (strspn($char, Data::WHITESPACE)) {
return new WhitespaceToken($char.$this->data->consumeWhile(Data::WHITESPACE));
} else {
return new CharacterToken($char.$this->data->consumeUntil("<\0"));
}
}
}
# 13.2.5.5 PLAINTEXT state
elseif ($this->state === self::PLAINTEXT_STATE) {
@ -434,9 +460,13 @@ class Tokenizer {
// Consume all characters that don't match what is above and emit
// that as a character token instead to prevent having to loop back
// through here every single time.
if (strspn($char, Data::WHITESPACE)) {
return new WhitespaceToken($char.$this->data->consumeWhile(Data::WHITESPACE));
} else {
return new CharacterToken($char.$this->data->consumeUntil("\0"));
}
}
}
# 13.2.5.6 Tag open state
elseif ($this->state === self::TAG_OPEN_STATE) {
@ -1062,9 +1092,13 @@ class Tokenizer {
// OPTIMIZATION:
// Consume all characters that aren't listed above to prevent having
// to loop back through here every single time.
if (strspn($char, Data::WHITESPACE)) {
return new WhitespaceToken($char.$this->data->consumeWhile(Data::WHITESPACE));
} else {
return new CharacterToken($char.$this->data->consumeUntil("-<\0"));
}
}
}
# 13.2.5.21 Script data escaped dash state
elseif ($this->state === self::SCRIPT_DATA_ESCAPED_DASH_STATE) {
@ -1104,9 +1138,13 @@ class Tokenizer {
# Switch to the script data escaped state.
# Emit the current input character as a character token.
$this->state = self::SCRIPT_DATA_ESCAPED_STATE;
if (strspn($char, Data::WHITESPACE)) {
return new WhitespaceToken($char);
} else {
return new CharacterToken($char);
}
}
}
# 13.2.5.22 Script data escaped dash dash state
elseif ($this->state === self::SCRIPT_DATA_ESCAPED_DASH_DASH_STATE) {
@ -1151,9 +1189,13 @@ class Tokenizer {
# Switch to the script data escaped state.
# Emit the current input character as a character token.
$this->state = self::SCRIPT_DATA_ESCAPED_STATE;
if (strspn($char, Data::WHITESPACE)) {
return new WhitespaceToken($char);
} else {
return new CharacterToken($char);
}
}
}
# 13.2.5.23 Script data escaped less-than sign state
elseif ($this->state === self::SCRIPT_DATA_ESCAPED_LESS_THAN_SIGN_STATE) {
@ -1313,8 +1355,12 @@ class Tokenizer {
} else {
$this->state = self::SCRIPT_DATA_ESCAPED_STATE;
}
if (strspn($char, Data::WHITESPACE)) {
return new WhitespaceToken($char);
} else {
return new CharacterToken($char);
}
}
# ASCII upper alpha
# ASCII lower alpha
elseif (ctype_alpha($char)) {
@ -1378,9 +1424,13 @@ class Tokenizer {
// OPTIMIZATION:
// Consume all characters that aren't listed above to prevent having
// to loop back through here every single time.
if (strspn($char, Data::WHITESPACE)) {
return new WhitespaceToken($char.$this->data->consumeWhile(Data::WHITESPACE));
} else {
return new CharacterToken($char.$this->data->consumeUntil("-<\0"));
}
}
}
# 13.2.5.28 Script data double escaped dash state
elseif ($this->state == self::SCRIPT_DATA_DOUBLE_ESCAPED_DASH_STATE) {
@ -1422,9 +1472,13 @@ class Tokenizer {
# Switch to the script data double escaped state.
# Emit the current input character as a character token.
$this->state = self::SCRIPT_DATA_DOUBLE_ESCAPED_STATE;
if (strspn($char, Data::WHITESPACE)) {
return new WhitespaceToken($char);
} else {
return new CharacterToken($char);
}
}
}
# 13.2.5.29 Script data double escaped dash dash state
elseif ($this->state == self::SCRIPT_DATA_DOUBLE_ESCAPED_DASH_DASH_STATE) {
@ -1471,9 +1525,13 @@ class Tokenizer {
# Switch to the script data double escaped state.
# Emit the current input character as a character token.
$this->state = self::SCRIPT_DATA_DOUBLE_ESCAPED_STATE;
if (strspn($char, Data::WHITESPACE)) {
return new WhitespaceToken($char);
} else {
return new CharacterToken($char);
}
}
}
# 13.2.5.30 Script data double escaped less-than sign state
elseif ($this->state === self::SCRIPT_DATA_DOUBLE_ESCAPED_LESS_THAN_SIGN_STATE) {
@ -1518,8 +1576,12 @@ class Tokenizer {
} else {
$this->state = self::SCRIPT_DATA_DOUBLE_ESCAPED_STATE;
}
if (strspn($char, Data::WHITESPACE)) {
return new WhitespaceToken($char);
} else {
return new CharacterToken($char);
}
}
# ASCII upper alpha
# ASCII lower alpha
elseif (ctype_alpha($char)) {
@ -3344,9 +3406,13 @@ class Tokenizer {
// OPTIMIZATION:
// Consume all characters that aren't listed above to prevent having
// to loop back through here every single time.
if (strspn($char, Data::WHITESPACE)) {
return new WhitespaceToken($char.$this->data->consumeWhile(Data::WHITESPACE));
} else {
return new CharacterToken($char.$this->data->consumeUntil(']'));
}
}
}
# 13.2.5.70 CDATA section bracket state
elseif ($this->state === self::CDATA_SECTION_BRACKET_STATE) {
@ -3378,7 +3444,7 @@ class Tokenizer {
# Emit a U+005D RIGHT SQUARE BRACKET character token.
// OTPIMIZATION: Consume any additional right square brackets
return new CharacterToken($char.$this->data->consumeWhile(']'));
return new CharacterToken(']'.$this->data->consumeWhile(']'));
}
# U+003E GREATER-THAN SIGN character
elseif ($char === '>') {

40
lib/TreeBuilder.php
View File

@ -221,7 +221,7 @@ class TreeBuilder {
# (LF), U+000C FORM FEED (FF), U+000D CARRIAGE RETURN (CR), or U+0020 SPACE
// OPTIMIZATION: Will check for multiple space characters at once as character
// tokens can contain more than one character.
if ($token instanceof CharacterToken && (strspn($token->data, Data::WHITESPACE) === strlen($token->data))) {
if ($token instanceof WhitespaceToken) {
# Ignore the token.
}
# A comment token
@ -389,7 +389,7 @@ class TreeBuilder {
# (LF), U+000C FORM FEED (FF), U+000D CARRIAGE RETURN (CR), or U+0020 SPACE
// OPTIMIZATION: Will check for multiple space characters at once as character
// tokens can contain more than one character.
elseif ($token instanceof CharacterToken && (strspn($token->data, Data::WHITESPACE) === strlen($token->data))) {
elseif ($token instanceof WhitespaceToken) {
# Ignore the token.
}
# A start tag whose tag name is "html"
@ -433,7 +433,7 @@ class TreeBuilder {
# (LF), U+000C FORM FEED (FF), U+000D CARRIAGE RETURN (CR), or U+0020 SPACE
// OPTIMIZATION: Will check for multiple space characters at once as character
// tokens can contain more than one character.
if ($token instanceof CharacterToken && (strspn($token->data, Data::WHITESPACE) === strlen($token->data))) {
if ($token instanceof WhitespaceToken) {
# Ignore the token.
}
# A comment token
@ -485,7 +485,7 @@ class TreeBuilder {
# (LF), U+000C FORM FEED (FF), U+000D CARRIAGE RETURN (CR), or U+0020 SPACE
// OPTIMIZATION: Will check for multiple space characters at once as character
// tokens can contain more than one character.
if ($token instanceof CharacterToken && (strspn($token->data, Data::WHITESPACE) === strlen($token->data))) {
if ($token instanceof WhitespaceToken) {
# Insert the character.
$this->insertCharacterToken($token);
}
@ -745,7 +745,7 @@ class TreeBuilder {
# A comment token
// OPTIMIZATION: Will check for multiple space characters at once as character
// tokens can contain more than one character.
elseif ($token instanceof CommentToken || ($token instanceof CharacterToken && (strspn($token->data, Data::WHITESPACE) === strlen($token->data)))) {
elseif ($token instanceof CommentToken || $token instanceof WhitespaceToken) {
# Process the token using the rules for the "in head" insertion mode.
return $this->parseTokenInHTMLContent($token, self::IN_HEAD_MODE);
}
@ -768,7 +768,7 @@ class TreeBuilder {
# (LF), U+000C FORM FEED (FF), U+000D CARRIAGE RETURN (CR), or U+0020 SPACE
// OPTIMIZATION: Will check for multiple space characters at once as character
// tokens can contain more than one character.
if ($token instanceof CharacterToken && (strspn($token->data, Data::WHITESPACE) === strlen($token->data))) {
if ($token instanceof WhitespaceToken) {
# Insert the character.
$this->insertCharacterToken($token);
}
@ -882,24 +882,21 @@ class TreeBuilder {
}
# A character token that is one of U+0009 CHARACTER TABULATION, U+000A LINE FEED
# (LF), U+000C FORM FEED (FF), U+000D CARRIAGE RETURN (CR), or U+0020 SPACE
#
elseif ($token instanceof WhitespaceToken) {
# Reconstruct the active formatting elements, if any.
$this->activeFormattingElementsList->reconstruct();
# Insert the token’s character.
$this->insertCharacterToken($token);
}
# Any other character token
// Space characters and any other characters are exactly the same except any
// other characters sets the frameset-ok flag to "not ok".
elseif ($token instanceof CharacterToken) {
# Reconstruct the active formatting elements, if any.
$this->activeFormattingElementsList->reconstruct();
# Insert the token’s character.
$this->insertCharacterToken($token);
// OPTIMIZATION: Will check for multiple space characters at once as character
// tokens can contain more than one character.
if (strspn($token->data, Data::WHITESPACE) !== strlen($token->data)) {
# Set the frameset-ok flag to "not ok".
$this->framesetOk = false;
}
}
# A comment token
elseif ($token instanceof CommentToken) {
# Insert a comment.
@ -1402,18 +1399,17 @@ class TreeBuilder {
#
# When the user agent is to apply the rules for parsing tokens in foreign
# content, the user agent must handle the token as follows:
#
if ($token instanceof CharacterToken) {
# A character token that is one of U+0009 CHARACTER TABULATION, "LF" (U+000A),
# "FF" (U+000C), "CR" (U+000D), or U+0020 SPACE
if ($token instanceof WhitespaceToken) {
# Insert the token's character.
$this->insertCharacterToken($token);
}
# Any other character token
// OPTIMIZATION: Will check for multiple space characters at once as character
// tokens can contain more than one character.
if (strspn($token->data, Data::WHITESPACE) !== strlen($token->data)) {
elseif ($token instanceof CharacterToken) {
# Set the frameset-ok flag to "not ok".
$this->framesetOk = false;
}
# Insert the token's character.
$this->insertCharacterToken($token);
}

5
tests/cases/TestTokenizer.php
View File

@ -12,6 +12,7 @@ use dW\HTML5\CommentToken;
use dW\HTML5\DOCTYPEToken;
use dW\HTML5\EndTagToken;
use dW\HTML5\StartTagToken;
use dW\HTML5\WhitespaceToken;
/**
* @covers \dW\HTML5\Tokenizer
@ -61,6 +62,7 @@ class TestTokenizer extends \PHPUnit\Framework\TestCase {
try {
do {
$t = $tokenizer->createToken();
assert(!$t instanceof CharacterToken || ($t instanceof WhitespaceToken && strspn($t->data, Data::WHITESPACE) === strlen($t->data)) || strspn($t->data, Data::WHITESPACE) === 0, new \Exception("Character token must either consist only of whitespace, or start with other than whitespace: ".var_export($t->data ?? "''", true)));
if (!($t instanceof EOFToken)) {
$actual[] = $t;
}
@ -100,6 +102,9 @@ class TestTokenizer extends \PHPUnit\Framework\TestCase {
foreach ($tokens as $t) {
if ($t instanceof CharacterToken) {
if (!$pending) {
if ($t instanceof WhitespaceToken) {
$t = new CharacterToken($t->data);
}
$pending = $t;
} else {
$pending->data .= $t->data;

Write Preview
Loading…
Cancel
Save