reiver
/
go-utf8
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

utf8s.ReadRune()

master
Charles Iliya Krempeaux 2018-07-02 01:27:49 -07:00
parent d4b29542cb
commit 03b9afa710
2 changed files with 478 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

158
readrune.go 100644
View File

@ -0,0 +1,158 @@
package utf8s
import (
"io"
)
// ReadRune reads a single UTF-8 encoded Unicode character from an io.Reader,
// and returns the Unicode character (as a Go rune) and the number of bytes read.
func ReadRune(reader io.Reader) (rune, int, error) {
if nil == reader {
return 0, 0, errNilReader
}
var count int
var b0 byte
{
var buffer [1]byte
var p []byte = buffer[:]
n, err := reader.Read(p)
count += n
if nil != err {
return 0, count, err
}
if 1 != n {
return 0, count, errInternalError
}
b0 = buffer[0]
}
if 127 >= b0 {
return rune(b0), count, nil
}
var more int
{
switch {
// 110x,xxxx 110x,xxxx
// 0b1100,0000 == (0b1110,0000 & b0)
case 0xC0 == (0xE0 & b0):
more = 2-1
// 1110,xxxx 1110,xxxx
// 0b1110,0000 == (0b1111,0000 & b0)
case 0xE0 == (0xF0 & b0):
more = 3-1
// 1111,0xxx 1111,0xxx
// 0b1111,0000 == (0b1111,1000 & b0)
case 0xF0 == (0xF8 & b0):
more = 4-1
// 1111,10xx 1111,10xx
// 0b1111,1000 == (0b1111,1100 & b0)
case 0xF8 == (0xFC & b0):
more = 5-1
// 1111,110x 1111,110x
// 0b1111,1100 == (0b1111,1110 & b0)
case 0xFC == (0xFE & b0):
more = 6-1
// 1111,1111 1111,1111
// 0b1111,1110 == (0b1111,1111 & b0)
case 0xFE == (0xFF & b0):
more = 7-1
default:
return 0, count, errInternalError
}
}
var bs [6]byte
{
p := bs[:more]
n, err := reader.Read(p)
count += n
if nil != err {
return 0, count, err
}
if more != n {
return 0, count, errInternalError
}
}
var r rune
{
var b byte
switch {
// 110x,xxxx 110x,xxxx
// 0b1100,0000 == (0b1110,0000 & b0)
case 0xC0 == (0xE0 & b0):
b = (0xE0^0xFF) & b0
// 1110,xxxx 1110,xxxx
// 0b1110,0000 == (0b1111,0000 & b0)
case 0xE0 == (0xF0 & b0):
b = (0xF0^0xFF) & b0
// 1111,0xxx 1111,0xxx
// 0b1111,0000 == (0b1111,1000 & b0)
case 0xF0 == (0xF8 & b0):
b = (0xF8^0xFF) & b0
// 1111,10xx 1111,10xx
// 0b1111,1000 == (0b1111,1100 & b0)
case 0xF8 == (0xFC & b0):
b = (0xFC^0xFF) & b0
// 1111,110x 1111,110x
// 0b1111,1100 == (0b1111,1110 & b0)
case 0xFC == (0xFE & b0):
b = (0xFE^0xFF) & b0
// 1111,1111 1111,1111
// 0b1111,1110 == (0b1111,1111 & b0)
case 0xFE == (0xFF & b0):
//b := (0xFF^0xFF) & b0
default:
return 0, count, errInternalError
}
r = rune(b)
r <<= 6
for i:=0; i<more; i++ {
bsi := bs[i]
// if 0b1000,0000 != (0b0b1100,0000 & bsi) {
if 0x80 != (0xC0 & bsi) {
return 0, count, errInvalidUTF8
}
// b := 0b0011,1111 & bsi
b := 0x3F & bsi
r2 := rune(b)
r |= r2
if i < (more-1) {
r <<= 6
}
}
}
return r, count, nil
}

320
readrune_test.go 100644
View File

@ -0,0 +1,320 @@
package utf8s
import (
"io"
"strings"
"testing"
)
func TestReadRune(t *testing.T) {
tests := []struct{
Reader io.Reader
ExpectedRune rune
ExpectedInt int
}{
{
Reader: strings.NewReader("a"),
ExpectedRune: 'a',
ExpectedInt: 1,
},
{
Reader: strings.NewReader("ap"),
ExpectedRune: 'a',
ExpectedInt: 1,
},
{
Reader: strings.NewReader("app"),
ExpectedRune: 'a',
ExpectedInt: 1,
},
{
Reader: strings.NewReader("appl"),
ExpectedRune: 'a',
ExpectedInt: 1,
},
{
Reader: strings.NewReader("apple"),
ExpectedRune: 'a',
ExpectedInt: 1,
},
{
Reader: strings.NewReader("b"),
ExpectedRune: 'b',
ExpectedInt: 1,
},
{
Reader: strings.NewReader("ba"),
ExpectedRune: 'b',
ExpectedInt: 1,
},
{
Reader: strings.NewReader("ban"),
ExpectedRune: 'b',
ExpectedInt: 1,
},
{
Reader: strings.NewReader("bana"),
ExpectedRune: 'b',
ExpectedInt: 1,
},
{
Reader: strings.NewReader("banan"),
ExpectedRune: 'b',
ExpectedInt: 1,
},
{
Reader: strings.NewReader("banana"),
ExpectedRune: 'b',
ExpectedInt: 1,
},
{
Reader: strings.NewReader("c"),
ExpectedRune: 'c',
ExpectedInt: 1,
},
{
Reader: strings.NewReader("ch"),
ExpectedRune: 'c',
ExpectedInt: 1,
},
{
Reader: strings.NewReader("che"),
ExpectedRune: 'c',
ExpectedInt: 1,
},
{
Reader: strings.NewReader("cher"),
ExpectedRune: 'c',
ExpectedInt: 1,
},
{
Reader: strings.NewReader("cherr"),
ExpectedRune: 'c',
ExpectedInt: 1,
},
{
Reader: strings.NewReader("cherry"),
ExpectedRune: 'c',
ExpectedInt: 1,
},
{
Reader: strings.NewReader("A"),
ExpectedRune: 'A',
ExpectedInt: 1,
},
{
Reader: strings.NewReader("r"),
ExpectedRune: 'r',
ExpectedInt: 1,
},
{
Reader: strings.NewReader("¡"),
ExpectedRune: '¡',
ExpectedInt: 2,
},
{
Reader: strings.NewReader("¡!"),
ExpectedRune: '¡',
ExpectedInt: 2,
},
{
Reader: strings.NewReader("۵"),
ExpectedRune: '۵',
ExpectedInt: 2,
},
{
Reader: strings.NewReader("۵5"),
ExpectedRune: '۵',
ExpectedInt: 2,
},
{
Reader: strings.NewReader("‱"),
ExpectedRune: '‱',
ExpectedInt: 3,
},
{
Reader: strings.NewReader("‱%"),
ExpectedRune: '‱',
ExpectedInt: 3,
},
{
Reader: strings.NewReader("≡"),
ExpectedRune: '≡',
ExpectedInt: 3,
},
{
Reader: strings.NewReader("≡="),
ExpectedRune: '≡',
ExpectedInt: 3,
},
{
Reader: strings.NewReader("𐏕"),
ExpectedRune: '𐏕',
ExpectedInt: 4,
},
{
Reader: strings.NewReader("𐏕100"),
ExpectedRune: '𐏕',
ExpectedInt: 4,
},
{
Reader: strings.NewReader("🙂"),
ExpectedRune: '🙂',
ExpectedInt: 4,
},
{
Reader: strings.NewReader("🙂:-)"),
ExpectedRune: '🙂',
ExpectedInt: 4,
},
{
Reader: strings.NewReader("\u0000"),
ExpectedRune: 0x0,
ExpectedInt: 1,
},
{
Reader: strings.NewReader("\u0001"),
ExpectedRune: 0x1,
ExpectedInt: 1,
},
{
Reader: strings.NewReader("\u007e"),
ExpectedRune: 0x7e,
ExpectedInt: 1,
},
{
Reader: strings.NewReader("\u007f"),
ExpectedRune: 0x7f,
ExpectedInt: 1,
},
{
Reader: strings.NewReader("\u0080"),
ExpectedRune: 0x80,
ExpectedInt: 2,
},
{
Reader: strings.NewReader("\u0081"),
ExpectedRune: 0x81,
ExpectedInt: 2,
},
{
Reader: strings.NewReader("\u07fe"),
ExpectedRune: 0x7fe,
ExpectedInt: 2,
},
{
Reader: strings.NewReader("\u07ff"),
ExpectedRune: 0x7ff,
ExpectedInt: 2,
},
{
Reader: strings.NewReader("\u0800"),
ExpectedRune: 0x800,
ExpectedInt: 3,
},
{
Reader: strings.NewReader("\u0801"),
ExpectedRune: 0x801,
ExpectedInt: 3,
},
{
Reader: strings.NewReader("\ufffe"),
ExpectedRune: 0xfffe,
ExpectedInt: 3,
},
{
Reader: strings.NewReader("\uffff"),
ExpectedRune: 0xffff,
ExpectedInt: 3,
},
{
Reader: strings.NewReader("\U00010000"),
ExpectedRune: 0x10000,
ExpectedInt: 4,
},
{
Reader: strings.NewReader("\U00010001"),
ExpectedRune: 0x10001,
ExpectedInt: 4,
},
{
Reader: strings.NewReader("\U0010fffe"),
ExpectedRune: 0x10fffe,
ExpectedInt: 4,
},
{
Reader: strings.NewReader("\U0010ffff"),
ExpectedRune: 0x10ffff,
ExpectedInt: 4,
},
}
for testNumber, test := range tests {
actualRune, actualInt, err := ReadRune(test.Reader)
if nil != err {
t.Errorf("For test #%d, did not expect an error, but actually got one: (%T) %q", testNumber, err, err)
t.Errorf("\tEXPECTED: %s", FormatBinary(test.ExpectedRune))
t.Errorf("\tACTUAL: %s", FormatBinary(actualRune))
continue
}
if expected, actual := test.ExpectedRune, actualRune; expected != actual {
t.Errorf("For test #%d, expected %q (0x%X), but actually got %q (0x%X).", testNumber, expected, expected, actual, actual)
t.Errorf("\tEXPECTED: %s", FormatBinary(test.ExpectedRune))
t.Errorf("\tACTUAL: %s", FormatBinary(actualRune))
continue
}
if expected, actual := test.ExpectedInt, actualInt; expected != actual {
t.Errorf("For test #%d, expected %d, but actually got %d.", testNumber, expected, actual)
t.Errorf("\tEXPECTED: %s", FormatBinary(test.ExpectedRune))
t.Errorf("\tACTUAL: %s", FormatBinary(actualRune))
continue
}
}
}