MensBeam
/
Intl
3
0
Fork 0
Code Pull Requests Releases 10 Activity
Browse Source

Implement gb18030 seeking 

Also fix some bugs in EOF handling
span
J. King 6 years ago
parent
40d0054bd1
commit
467c565e8c
6 changed files with 134 additions and 23 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      lib/Encoding/GB18030.php
  2. 68
      lib/Encoding/GBCommon.php
  3. 1
      lib/Encoding/GBK.php
  4. 30
      lib/Encoding/GenericEncoding.php
  5. 37
      tests/cases/Encoding/TestGB18030.php
  6. 20
      tools/mktestgbk.html

1
lib/Encoding/GB18030.php
View File

@ -10,5 +10,4 @@ class GB18030 extends GBCommon {
const GBK = false;
const NAME = "gb18030";
const LABELS = ["gb18030"];
}

68
lib/Encoding/GBCommon.php
View File

File diff suppressed because one or more lines are too long

1
lib/Encoding/GBK.php
View File

@ -20,5 +20,4 @@ class GBK extends GBCommon {
"iso-ir-58",
"x-gbk",
];
}

30
lib/Encoding/GenericEncoding.php
View File

@ -64,6 +64,36 @@ trait GenericEncoding {
}
}
/** Advance $distance characters through the string
*
* If $distance is negative, the operation will be performed in reverse
*
* If the end (or beginning) of the string was reached before the end of the operation, the remaining number of requested characters is returned
*/
public function seek(int $distance): int {
if ($distance > 0) {
if ($this->posByte == strlen($this->string)) {
return $distance;
}
do {
$p = $this->nextCode();
} while (--$distance && $p !== false);
return $distance;
} elseif ($distance < 0) {
$distance = abs($distance);
if (!$this->posByte) {
return $distance;
}
$mode = $this->errMode;
$this->errMode = self::MODE_NULL;
$out = $this->seekBack($distance);
$this->errMode = $mode;
return $out;
} else {
return 0;
}
}
/** Retrieves the next $num characters (in UTF-8 encoding) from the string without advancing the character pointer */
public function peekChar(int $num = 1): string {
$out = "";

37
tests/cases/Encoding/TestGB18030.php
View File

@ -16,8 +16,8 @@ class TestGB18030 extends \PHPUnit\Framework\TestCase {
/**
* @dataProvider provideCodePoints
* @covers MensBeam\Intl\Encoding\GB18030::encode
* @covers MensBeam\Intl\Encoding\GBK::encode
* @covers MensBeam\Intl\Encoding\GB18030::err
* @covers MensBeam\Intl\Encoding\GBK::encode
* @covers MensBeam\Intl\Encoding\GBK::err
*/
public function testEncodeCodePoints(string $class, bool $fatal, int $input, $exp) {
@ -33,11 +33,15 @@ class TestGB18030 extends \PHPUnit\Framework\TestCase {
* @dataProvider provideStrings
* @covers MensBeam\Intl\Encoding\GB18030::__construct
* @covers MensBeam\Intl\Encoding\GB18030::nextCode
* @covers MensBeam\Intl\Encoding\GB18030::posChar
*/
public function testDecodeMultipleCharactersAsCodePoints(string $input, array $exp) {
$s = new GB18030(hex2bin($input));
$out = [];
$a = 0;
$this->assertSame($a, $s->posChar());
while (($p = $s->nextCode()) !== false) {
$this->assertSame(++$a, $s->posChar());
$out[] = $p;
}
$this->assertSame($exp, $out);
@ -60,6 +64,27 @@ class TestGB18030 extends \PHPUnit\Framework\TestCase {
$this->assertSame($exp, $out);
}
/**
* @dataProvider provideStrings
* @covers MensBeam\Intl\Encoding\GB18030::seekBack
*/
public function testSTepBackThroughAString(string $input, array $points) {
$s = new GB18030(hex2bin($input));
$a = 0;
$test1 = [];
$test2 = [];
while (($p1 = $s->nextCode()) !== false) {
$test1[] = $p1;
$this->assertSame(0, $s->seek(-1));
$p2 = $s->nextCode();
$test2[] = $p2;
$this->assertSame($p1, $p2, "Mismatch at character position $a");
$this->assertSame(++$a, $s->posChar(), "Character position should be $a");
}
$this->assertSame($points, $test1);
$this->assertSame($points, $test2);
}
public function provideCodePoints() {
// bytes confirmed using Firefox
return [
@ -104,6 +129,7 @@ class TestGB18030 extends \PHPUnit\Framework\TestCase {
public function provideStrings() {
return [
'empty string' => ["", []],
// valid single characters
'sanity check' => ["40", [64]],
'special case for 0x80' => ["80", [8364]],
@ -123,6 +149,15 @@ class TestGB18030 extends \PHPUnit\Framework\TestCase {
'control second byte' => ["8100F437", [65533, 0, 65533]],
'control third byte' => ["81350037", [65533, 53, 0, 55]],
'control fourth byte' => ["8135F400", [65533, 53, 65533, 0]],
// invalid sequences with clean EOF
'bad first byte (padded)' => ["FF35F43700000000", [65533, 53, 65533, 55, 0, 0, 0, 0]],
'bad second byte (padded)' => ["81FFF43700000000", [65533, 65533, 55, 0, 0, 0, 0]],
'bad third byte (padded)' => ["8135FF3700000000", [65533, 53, 65533, 55, 0, 0, 0, 0]],
'bad fourth byte (padded)' => ["8135F4FF00000000", [65533, 53, 65533, 0, 0, 0, 0]],
'control first byte (padded)' => ["0035F43700000000", [0, 53, 65533, 55, 0, 0, 0, 0]],
'control second byte (padded)' => ["8100F43700000000", [65533, 0, 65533, 55, 0, 0, 0, 0]],
'control third byte (padded)' => ["8135003700000000", [65533, 53, 0, 55, 0, 0, 0, 0]],
'control fourth byte (padded)' => ["8135F40000000000", [65533, 53, 65533, 0, 0, 0, 0, 0]],
// out-of-range sequences
'void sequence' => ["8432A439", [65533]],
'void sequence 2' => ["FE39FE39", [65533]],

20
tools/mktestgbk.html
View File

@ -21,14 +21,14 @@ var data = [
{ encoding: 'gb18030', input: [0x81, 0x00, 0xF4, 0x37], name: 'control second byte' },
{ encoding: 'gb18030', input: [0x81, 0x35, 0x00, 0x37], name: 'control third byte' },
{ encoding: 'gb18030', input: [0x81, 0x35, 0xF4, 0x00], name: 'control fourth byte' },
{ encoding: 'gb18030', input: [0xFF, 0x35, 0xF4, 0x37, 0x00, 0x00, 0x00, 0x00], name: 'bad first byte 2' },
{ encoding: 'gb18030', input: [0x81, 0xFF, 0xF4, 0x37, 0x00, 0x00, 0x00, 0x00], name: 'bad second byte 2' },
{ encoding: 'gb18030', input: [0x81, 0x35, 0xFF, 0x37, 0x00, 0x00, 0x00, 0x00], name: 'bad third byte 2' },
{ encoding: 'gb18030', input: [0x81, 0x35, 0xF4, 0xFF, 0x00, 0x00, 0x00, 0x00], name: 'bad fourth byte 2' },
{ encoding: 'gb18030', input: [0x00, 0x35, 0xF4, 0x37, 0x00, 0x00, 0x00, 0x00], name: 'control first byte 2' },
{ encoding: 'gb18030', input: [0x81, 0x00, 0xF4, 0x37, 0x00, 0x00, 0x00, 0x00], name: 'control second byte 2' },
{ encoding: 'gb18030', input: [0x81, 0x35, 0x00, 0x37, 0x00, 0x00, 0x00, 0x00], name: 'control third byte 2' },
{ encoding: 'gb18030', input: [0x81, 0x35, 0xF4, 0x00, 0x00, 0x00, 0x00, 0x00], name: 'control fourth byte 2' },
{ encoding: 'gb18030', input: [0xFF, 0x35, 0xF4, 0x37, 0x00, 0x00, 0x00, 0x00], name: 'bad first byte (padded)' },
{ encoding: 'gb18030', input: [0x81, 0xFF, 0xF4, 0x37, 0x00, 0x00, 0x00, 0x00], name: 'bad second byte (padded)' },
{ encoding: 'gb18030', input: [0x81, 0x35, 0xFF, 0x37, 0x00, 0x00, 0x00, 0x00], name: 'bad third byte (padded)' },
{ encoding: 'gb18030', input: [0x81, 0x35, 0xF4, 0xFF, 0x00, 0x00, 0x00, 0x00], name: 'bad fourth byte (padded)' },
{ encoding: 'gb18030', input: [0x00, 0x35, 0xF4, 0x37, 0x00, 0x00, 0x00, 0x00], name: 'control first byte (padded)' },
{ encoding: 'gb18030', input: [0x81, 0x00, 0xF4, 0x37, 0x00, 0x00, 0x00, 0x00], name: 'control second byte (padded)' },
{ encoding: 'gb18030', input: [0x81, 0x35, 0x00, 0x37, 0x00, 0x00, 0x00, 0x00], name: 'control third byte (padded)' },
{ encoding: 'gb18030', input: [0x81, 0x35, 0xF4, 0x00, 0x00, 0x00, 0x00, 0x00], name: 'control fourth byte (padded)' },
{ encoding: 'gb18030', input: [0x84, 0x32, 0xA4, 0x39], name: 'void sequence' },
{ encoding: 'gb18030', input: [0xFE, 0x39, 0xFE, 0x39], name: 'void sequence 2' },
];
@ -62,8 +62,8 @@ document.getElementsByTagName("pre")[0].appendChild(document.createTextNode("\n\
var bytes = "";
for (let a = 0; a < url.length; a++) {
if (url.charAt(a) == "%") {
bytes = bytes.concat(url.charAt(a + 1), url.charAt(a + 2));
a = a + 2;
bytes = bytes.concat(url.charAt(a + 1), url.charAt(a + (padded)));
a = a + (padded);
} else {
bytes = bytes.concat(url.charCodeAt(a).toString(16).padStart(2, "0"));
}

Write Preview
Loading…
Cancel
Save