Compare commits

...

10 Commits

14 changed files with 155 additions and 153 deletions

View File

@ -1,7 +1,8 @@
# go-utf8 # go-utf8
Package **utf8** provides tools for working with Unicode encoded as UTF-8, for the Go programming language. Package **utf8** implements encoding and decoding of UTF-8, for the Go programming language.
This package is meant to be a replacement for Go's built-in `"unicode/utf8"` package.
## Documention ## Documention
@ -9,8 +10,9 @@ Online documentation, which includes examples, can be found at: http://godoc.org
[![GoDoc](https://godoc.org/github.com/reiver/go-utf8?status.svg)](https://godoc.org/github.com/reiver/go-utf8) [![GoDoc](https://godoc.org/github.com/reiver/go-utf8?status.svg)](https://godoc.org/github.com/reiver/go-utf8)
## Reading a Single UTF-8 Character
## Example This is the simplest way of reading a single UTF-8 character.
```go ```go
var reader io.Reader var reader io.Reader
@ -19,6 +21,9 @@ var reader io.Reader
r, n, err := utf8.ReadRune(reader) r, n, err := utf8.ReadRune(reader)
``` ```
## Write a Single UTF-8 Character
This is the simplest way of writing a single UTF-8 character.
```go ```go
var writer io.Writer var writer io.Writer
@ -31,25 +36,31 @@ var r rune
n, err := utf8.WriteRune(w, r) n, err := utf8.WriteRune(w, r)
``` ```
## io.RuneReader
This is how you can create an `io.RuneReader`:
```go ```go
var reader io.Reader var reader io.Reader
// ... // ...
runeReader := utf8.NewRuneReader(reader) var runeReader io.RuneReader = utf8.RuneReaderWrap(reader)
// ... // ...
r, n, err := runeReader.ReadRune() r, n, err := runeReader.ReadRune()
``` ```
## io.RuneScanner
This is how you can create an `io.RuneScanner`:
```go ```go
var reader io.Reader var reader io.Reader
// ... // ...
runeScanner := utf8.NewRuneScanner(reader) var runeScanner io.RuneScanner := utf8.RuneScannerWrap(reader)
// ... // ...
@ -59,3 +70,60 @@ r, n, err := runeScanner.ReadRune()
err = runeScanner.UnreadRune() err = runeScanner.UnreadRune()
``` ```
## UTF-8
UTF-8 is a variable length encoding of Unicode.
An encoding of a single Unicode code point can be from 1 to 4 bytes longs.
Some examples of UTF-8 encoding of Unicode code points are:
```
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ UTF-8 encoding ┃ ┃ ┃ ┃ ┃ ┃
┣━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┫ ┃ ┃ ┃ ┃ ┃
┃ byte 1 ┋ byte 2 ┋ byte 3 ┋ byte 4 ┃ value ┃ code point ┃ decimal ┃ binary ┃ name ┃
┡━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
│ 0b0,1000001 ┊ ┊ ┊ │ A │ U+0041 │ 65 │ 0b0000,0000,0100,0001 │ LATIN CAPITAL LETTER A │
├─────────────┼─────────────┼─────────────┼─────────────┼───────┼────────────┼─────────┼────────────────────────────┼──────────────────────────────────┤
│ 0b0,1110010 ┊ ┊ ┊ │ r │ U+0072 │ 114 │ 0b0000,0000,0111,0010 │ LATIN SMALL LETTER R │
├─────────────┼─────────────┼─────────────┼─────────────┼───────┼────────────┼─────────┼────────────────────────────┼──────────────────────────────────┤
│ 0b110,00010 ┊ 0b10,100001 ┊ ┊ │ ¡ │ U+00A1 │ 161 │ 0b0000,0000,1010,0001 │ INVERTED EXCLAMATION MARK │
├─────────────┼─────────────┼─────────────┼─────────────┼───────┼────────────┼─────────┼────────────────────────────┼──────────────────────────────────┤
│ 0b110,11011 ┊ 0b10,110101 ┊ ┊ │ ۵ │ U+06F5 │ 1781 │ 0b0000,0110,1111,0101 │ EXTENDED ARABIC-INDIC DIGIT FIVE │
├─────────────┼─────────────┼─────────────┼─────────────┼───────┼────────────┼─────────┼────────────────────────────┼──────────────────────────────────┤
│ 0b1110,0010 ┊ 0b10,000000 ┊ 0b10,110001 ┊ │ ‱ │ U+2031 │ 8241 │ 0b0010,0000,0011,0001 │ PER TEN THOUSAND SIGN │
├─────────────┼─────────────┼─────────────┼─────────────┼───────┼────────────┼─────────┼────────────────────────────┼──────────────────────────────────┤
│ 0b1110,0010 ┊ 0b10,001001 ┊ 0b10,100001 ┊ │ ≡ │ U+2261 │ 8801 │ 0b0010,0010,0110,0001 │ IDENTICAL TO │
├─────────────┼─────────────┼─────────────┼─────────────┼───────┼────────────┼─────────┼────────────────────────────┼──────────────────────────────────┤
│ 0b11110,000 ┊ 0b10,010000 ┊ 0b10,001111 ┊ 0b10,010101 │ 𐏕 │ U+000103D5 │ 66517 │ 0b0001,0000,0011,1101,0101 │ OLD PERSIAN NUMBER HUNDRED │
├─────────────┼─────────────┼─────────────┼─────────────┼───────┼────────────┼─────────┼────────────────────────────┼──────────────────────────────────┤
│ 0b11110,000 ┊ 0b10,011111 ┊ 0b10,011001 ┊ 0b10,000010 │ 🙂 │ U+0001F642 │ 128578 │ 0b0001,1111,0110,0100,0010 │ SLIGHTLY SMILING FACE │
└─────────────┴─────────────┴─────────────┴─────────────┴───────┴────────────┴─────────┴────────────────────────────┴──────────────────────────────────┘
```
## UTF-8 Versus ASCII
UTF-8 was (partially) designed to be backwards compatible with 7-bit ASCII.
Thus, all 7-bit ASCII is valid UTF-8.
## UTF-8 Encoding
Since, at least as of 2003, Unicode fits into 21 bits, and thus UTF-8 was designed to support at most 21 bits of information.
This is done as described in the following table:
```
┏━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━┓
┃ # of bytes ┃ # bits for code point ┃ 1st code point ┃ last code point ┃ byte 1 ┃ byte 2 ┃ byte 3 ┃ byte 4 ┃
┡━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━┩
│ 1 │ 7 │ U+000000 │ U+00007F │ 0xxxxxxx │ │ │ │
├────────────┼───────────────────────┼────────────────┼──────────────────┼──────────┼──────────┼──────────┼──────────┤
│ 2 │ 11 │ U+000080 │ U+0007FF │ 110xxxxx │ 10xxxxxx │ │ │
├────────────┼───────────────────────┼────────────────┼──────────────────┼──────────┼──────────┼──────────┼──────────┤
│ 3 │ 16 │ U+000800 │ U+00FFFF │ 1110xxxx │ 10xxxxxx │ 10xxxxxx │ │
├────────────┼───────────────────────┼────────────────┼──────────────────┼──────────┼──────────┼──────────┼──────────┤
│ 4 │ 21 │ U+010000 │ U+10FFFF │ 11110xxx │ 10xxxxxx │ 10xxxxxx │ 10xxxxxx │
└────────────┴───────────────────────┴────────────────┴──────────────────┴──────────┴──────────┴──────────┴──────────┘
```

123
doc.go
View File

@ -1,123 +0,0 @@
/*
Package utf8 provides tools for working with Unicode encoded as UTF-8.
Example
var reader io.Reader
// ...
r, n, err := utf8.ReadRune(reader)
Example
var writer io.Writer
// ...
var r rune
// ...
n, err := utf8.WriteRune(w, r)
Example
var reader io.Reader
// ...
runeReader := utf8.NewRuneReader(reader)
// ..
r, n, err := runeReader.ReadRune()
Example
var reader io.Reader
// ...
runeScanner := utf8.NewRuneScanner(reader)
// ...
r, n, err := runeScanner.ReadRune()
// ...
err = runeScanner.UnreadRune()
Example
var r rune
// ...
s := utfs.FormatBinary(r)
fmt.Printf("%q encoded in UTF-8 as binary is %s \n", r, s)
UTF-8
UTF-8 is a variable length encoding of Unicode. An encoding of a single Unicode code point can be from 1 to 4 bytes longs.
Some examples of UTF-8 encoding of Unicode code points are:
UTF-8 encoding
byte 1 byte 2 byte 3 byte 4 value code point decimal binary name
0b0,1000001 A U+0041 65 0b0000,0000,0100,0001 LATIN CAPITAL LETTER A
0b0,1110010 r U+0072 114 0b0000,0000,0111,0010 LATIN SMALL LETTER R
0b110,00010 0b10,100001 ¡ U+00A1 161 0b0000,0000,1010,0001 INVERTED EXCLAMATION MARK
0b110,11011 0b10,110101 ۵ U+06F5 1781 0b0000,0110,1111,0101 EXTENDED ARABIC-INDIC DIGIT FIVE
0b1110,0010 0b10,000000 0b10,110001 U+2031 8241 0b0010,0000,0011,0001 PER TEN THOUSAND SIGN
0b1110,0010 0b10,001001 0b10,100001 U+2261 8801 0b0010,0010,0110,0001 IDENTICAL TO
0b11110,000 0b10,010000 0b10,001111 0b10,010101 𐏕 U+000103D5 66517 0b0001,0000,0011,1101,0101 OLD PERSIAN NUMBER HUNDRED
0b11110,000 0b10,011111 0b10,011001 0b10,000010 🙂 U+0001F642 128578 0b0001,1111,0110,0100,0010 SLIGHTLY SMILING FACE
UTF-8 Versus ASCII
UTF-8 was (partially) designed to be backwards compatible with 7-bit ASCII.
Thus, all 7-bit ASCII is valid UTF-8.
UTF-8 Encoding
Since, at least as of 2003, Unicode fit into 21 bits, UTF-8 was designed to support at most 21 bits of information.
This is done as described in the following table:
# of bytes # bits for code point 1st code point last code point byte 1 byte 2 byte 3 byte 4
1 7 U+000000 U+00007F 0xxxxxxx
2 11 U+000080 U+0007FF 110xxxxx 10xxxxxx
3 16 U+000800 U+00FFFF 1110xxxx 10xxxxxx 10xxxxxx
4 21 U+010000 U+10FFFF 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
*/
package utf8

View File

@ -1,5 +1,21 @@
package utf8 package utf8
// InvalidUTF8Complainer is a type of error that could be returned
// by the utf8.ReadRune() function,
// by the utf8.RuneReader.ReadRune() method, and
// by the utf8.RuneScanner.ReadRune() method.
//
// Here is how one might use this type:
//
// r, n, err := utf8.ReadRune(reader)
// if nil != err {
// switch {
// case utf8.InvalidUTF8Complainer:
// //@TODO
// default:
// //@TODO
// }
// }
type InvalidUTF8Complainer interface { type InvalidUTF8Complainer interface {
error error
InvalidUTF8Complainer() InvalidUTF8Complainer()

View File

@ -7,6 +7,30 @@ import (
// ReadRune reads a single UTF-8 encoded Unicode character from an io.Reader, // ReadRune reads a single UTF-8 encoded Unicode character from an io.Reader,
// and returns the Unicode character (as a Go rune) and the number of bytes read. // and returns the Unicode character (as a Go rune) and the number of bytes read.
// //
// If reader is nil then ReaderRune will return an error that matches utf8.NilReaderComplainer.
//
// Example
//
// Here is an example usage of ReadRune:
//
// r, n, err := utf8.ReadRune(reader)
// if nil != err {
//
// switch err.(type) {
// case utf8.NilReaderComplainer:
// //@TODO
// case utf8.InvalidUTF8Complainer:
// //@TODO
// default:
// //TODO
// }
// }
// if utf8.RuneError == r {
// //@TODO
// }
//
// Number Of Bytes
//
// Note that a single UTF-8 encoded Unicode character could be more than one byte. // Note that a single UTF-8 encoded Unicode character could be more than one byte.
// //
// For example, the Unicode "≡" (IDENTICAL TO) character gets encoded using 3 bytes under UTF-8. // For example, the Unicode "≡" (IDENTICAL TO) character gets encoded using 3 bytes under UTF-8.

View File

@ -601,7 +601,7 @@ func TestReadRunes(t *testing.T) {
t.Errorf("\tACTUAL: %s", FormatBinary(actualRune)) t.Errorf("\tACTUAL: %s", FormatBinary(actualRune))
continue TestLoop continue TestLoop
} }
if expected, actual := Len(test.Expected[runeNumber]), actualInt; expected != actual { if expected, actual := RuneLength(test.Expected[runeNumber]), actualInt; expected != actual {
t.Errorf("For test #%d and rune #%d, expected %d, but actually got %d.", testNumber, runeNumber, expected, actual) t.Errorf("For test #%d and rune #%d, expected %d, but actually got %d.", testNumber, runeNumber, expected, actual)
t.Errorf("\tEXPECTED: %s", FormatBinary(test.Expected[runeNumber])) t.Errorf("\tEXPECTED: %s", FormatBinary(test.Expected[runeNumber]))
t.Errorf("\tACTUAL: %s", FormatBinary(actualRune)) t.Errorf("\tACTUAL: %s", FormatBinary(actualRune))

View File

@ -1,31 +1,31 @@
package utf8 package utf8
// Len returns the number of bytes in a UTF-8 encoding of this Unicode code point. // RuneLength returns the number of bytes in a UTF-8 encoding of this Unicode code point.
// //
// Example // Example
// //
// length := utf8.Len('A') // length := utf8.RuneLength('A')
// //
// // length == 1 // // length == 1
// //
// Example // Example
// //
// length := utf8.Len('r') // length := utf8.RuneLength('r')
// //
// // length == 1 // // length == 1
// //
// Example // Example
// //
// length := utf8.Len('¡') // length := utf8.RuneLength('¡')
// //
// // length == 2 // // length == 2
// //
// Example // Example
// //
// length := utf8.Len('۵') // length := utf8.RuneLength('۵')
// //
// // length == 2 // // length == 2
func Len(r rune) int { func RuneLength(r rune) int {
switch { switch {
case 127 >= r: case 127 >= r:

View File

@ -4,7 +4,7 @@ import (
"testing" "testing"
) )
func TestLen(t *testing.T) { func TestRuneLength(t *testing.T) {
tests := []struct{ tests := []struct{
Datum rune Datum rune
@ -123,7 +123,7 @@ func TestLen(t *testing.T) {
for testNumber, test := range tests { for testNumber, test := range tests {
actual := Len(test.Datum) actual := RuneLength(test.Datum)
if expected := test.Expected; expected != actual { if expected := test.Expected; expected != actual {
t.Errorf("For test #%d, expected %d, but actually got %d.", testNumber, expected, actual) t.Errorf("For test #%d, expected %d, but actually got %d.", testNumber, expected, actual)
continue continue

View File

@ -9,8 +9,8 @@ type RuneReader struct {
reader io.Reader reader io.Reader
} }
func NewRuneReader(reader io.Reader) *RuneReader { func RuneReaderWrap(reader io.Reader) RuneReader {
return &RuneReader{ return RuneReader{
reader: reader, reader: reader,
} }
} }

View File

@ -295,7 +295,7 @@ func TestRuneReader(t *testing.T) {
for testNumber, test := range tests { for testNumber, test := range tests {
runeReader := NewRuneReader(test.Reader) runeReader := RuneReaderWrap(test.Reader)
actualRune, actualInt, err := runeReader.ReadRune() actualRune, actualInt, err := runeReader.ReadRune()
if nil != err { if nil != err {
@ -563,7 +563,7 @@ func TestRuneReaders(t *testing.T) {
var runeNumber int var runeNumber int
for { for {
runeReader := NewRuneReader(test.Reader) runeReader := RuneReaderWrap(test.Reader)
actualRune, actualInt, err := runeReader.ReadRune() actualRune, actualInt, err := runeReader.ReadRune()
if nil != err && io.EOF != err { if nil != err && io.EOF != err {
@ -585,7 +585,7 @@ func TestRuneReaders(t *testing.T) {
t.Errorf("\tACTUAL: %s", FormatBinary(actualRune)) t.Errorf("\tACTUAL: %s", FormatBinary(actualRune))
continue TestLoop continue TestLoop
} }
if expected, actual := Len(test.Expected[runeNumber]), actualInt; expected != actual { if expected, actual := RuneLength(test.Expected[runeNumber]), actualInt; expected != actual {
t.Errorf("For test #%d and rune #%d, expected %d, but actually got %d.", testNumber, runeNumber, expected, actual) t.Errorf("For test #%d and rune #%d, expected %d, but actually got %d.", testNumber, runeNumber, expected, actual)
t.Errorf("\tEXPECTED: %s", FormatBinary(test.Expected[runeNumber])) t.Errorf("\tEXPECTED: %s", FormatBinary(test.Expected[runeNumber]))
t.Errorf("\tACTUAL: %s", FormatBinary(actualRune)) t.Errorf("\tACTUAL: %s", FormatBinary(actualRune))

View File

@ -15,8 +15,8 @@ type RuneScanner struct {
peeked bool peeked bool
} }
func NewRuneScanner(reader io.Reader) *RuneScanner { func RuneScannerWrap(reader io.Reader) RuneScanner {
return &RuneScanner{ return RuneScanner{
reader: reader, reader: reader,
} }
} }

View File

@ -295,7 +295,7 @@ func TestRuneScanner(t *testing.T) {
for testNumber, test := range tests { for testNumber, test := range tests {
runeReader := NewRuneScanner(test.Reader) runeReader := RuneScannerWrap(test.Reader)
actualRune, actualInt, err := runeReader.ReadRune() actualRune, actualInt, err := runeReader.ReadRune()
if nil != err { if nil != err {
@ -563,7 +563,7 @@ func TestRuneScanners(t *testing.T) {
var runeNumber int var runeNumber int
for { for {
runeReader := NewRuneScanner(test.Reader) runeReader := RuneScannerWrap(test.Reader)
actualRune, actualInt, err := runeReader.ReadRune() actualRune, actualInt, err := runeReader.ReadRune()
if nil != err && io.EOF != err { if nil != err && io.EOF != err {
@ -585,7 +585,7 @@ func TestRuneScanners(t *testing.T) {
t.Errorf("\tACTUAL: %s", FormatBinary(actualRune)) t.Errorf("\tACTUAL: %s", FormatBinary(actualRune))
continue TestLoop continue TestLoop
} }
if expected, actual := Len(test.Expected[runeNumber]), actualInt; expected != actual { if expected, actual := RuneLength(test.Expected[runeNumber]), actualInt; expected != actual {
t.Errorf("For test #%d and rune #%d, expected %d, but actually got %d.", testNumber, runeNumber, expected, actual) t.Errorf("For test #%d and rune #%d, expected %d, but actually got %d.", testNumber, runeNumber, expected, actual)
t.Errorf("\tEXPECTED: %s", FormatBinary(test.Expected[runeNumber])) t.Errorf("\tEXPECTED: %s", FormatBinary(test.Expected[runeNumber]))
t.Errorf("\tACTUAL: %s", FormatBinary(actualRune)) t.Errorf("\tACTUAL: %s", FormatBinary(actualRune))
@ -616,7 +616,7 @@ func TestRuneScannerUnread(t *testing.T) {
TestLoop: for testNumber, test := range tests { TestLoop: for testNumber, test := range tests {
runeScanner := NewRuneScanner(test.Reader) runeScanner := RuneScannerWrap(test.Reader)
var readCount int var readCount int
for instructionNumber, instruction := range test.Instructions { for instructionNumber, instruction := range test.Instructions {

View File

@ -9,9 +9,9 @@ type RuneWriter struct {
writer io.Writer writer io.Writer
} }
// NewRuneWriter wraps an io.Writer and returns a RuneWriter. // RuneWriterWrap wraps an io.Writer and returns a RuneWriter.
func NewRuneWriter(writer io.Writer) *RuneWriter { func RuneWriterWrap(writer io.Writer) RuneWriter {
return &RuneWriter{ return RuneWriter{
writer: writer, writer: writer,
} }
} }

View File

@ -58,7 +58,7 @@ func TestRuneWriter(t *testing.T) {
var buffer strings.Builder var buffer strings.Builder
var total int var total int
runeWriter := utf8.NewRuneWriter(&buffer) runeWriter := utf8.RuneWriterWrap(&buffer)
for runeNumber, r := range test.Runes { for runeNumber, r := range test.Runes {
@ -73,7 +73,7 @@ func TestRuneWriter(t *testing.T) {
continue TestLoop continue TestLoop
} }
if expected, actual := n, utf8.Len(r); expected != actual { if expected, actual := n, utf8.RuneLength(r); expected != actual {
t.Errorf("For test #%d and rune #%d, expected a certain number of bytes to be written, but actually wasn't.", testNumber, runeNumber) t.Errorf("For test #%d and rune #%d, expected a certain number of bytes to be written, but actually wasn't.", testNumber, runeNumber)
for i, rr := range test.Runes { for i, rr := range test.Runes {
t.Logf("\t[%d] %q (%d)", i, string(rr), rr) t.Logf("\t[%d] %q (%d)", i, string(rr), rr)

View File

@ -5,6 +5,24 @@ import (
) )
// WriteRune writes a single UTF-8 encoded Unicode character and returns the number of bytes written. // WriteRune writes a single UTF-8 encoded Unicode character and returns the number of bytes written.
//
// If writer is nil then WriteRune will return an error that matches utf8.NilWriterComplainer.
//
// Example
//
// Here is an example usage of WriteRune:
//
// n, err := utf8.WriteRune(writer, r)
// if nil != err {
//
// switch err.(type) {
// case utf8.NilWriterComplainer:
// //@TODO
// default:
// //TODO
// }
//
// }
func WriteRune(writer io.Writer, r rune) (int, error) { func WriteRune(writer io.Writer, r rune) (int, error) {
if nil == writer { if nil == writer {
return 0, errNilWriter return 0, errNilWriter
@ -56,5 +74,4 @@ func WriteRune(writer io.Writer, r rune) (int, error) {
default: default:
return 0, errInternalError return 0, errInternalError
} }
} }