From 5075eff7cc9c00a7c49ed2f8100272b6ab99dbe8 Mon Sep 17 00:00:00 2001 From: Charles Iliya Krempeaux Date: Wed, 6 Dec 2023 14:24:54 -0800 Subject: [PATCH] initial commits --- token/bytes.go | 76 ++ token/bytes_test.go | 2040 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 2116 insertions(+) create mode 100644 token/bytes.go create mode 100644 token/bytes_test.go diff --git a/token/bytes.go b/token/bytes.go new file mode 100644 index 0000000..07af6aa --- /dev/null +++ b/token/bytes.go @@ -0,0 +1,76 @@ +package token + +import ( + "sourcecode.social/reiver/go-rfc2616/char" + "sourcecode.social/reiver/go-rfc2616/ctl" + "sourcecode.social/reiver/go-rfc2616/separator" +) + +// Bytes returns the 'Method', as defiend by IETF RFC-2616. +// +// Method = "OPTIONS" +// | "GET" +// | "HEAD" +// | "POST" +// | "PUT" +// | "DELETE" +// | "TRACE" +// | "CONNECT" +// | extension-method +// +// extension-method = token +// +// Which, in practice, can be simplified to: +// +// Method = token +// +// And 'token' is defined as: +// +// token = 1* +// +// CHAR = +// +// CTL = +// +// separators = "(" | ")" | "<" | ">" | "@" +// | "," | ";" | ":" | "\" | <"> +// | "/" | "[" | "]" | "?" | "=" +// | "{" | "}" | SP | HT +// +// SP = +// +// HT = +func Bytes(p []byte) (method []byte, rest []byte, ok bool) { + if 0 == len(p) { + return nil, nil, false + } + + { + p0 := p[0] + + if !char.ByteIsChar(p0) { + return nil, p, false + } + + if separator.ByteIsSeparator(p0) { + return nil, p, false + } + + if ctl.ByteIsControlCharacter(p0) { + return nil, p, false + } + } + + { + var i int + var b byte + + for i,b = range p { + if !char.ByteIsChar(b) || separator.ByteIsSeparator(b) || ctl.ByteIsControlCharacter(b) { + return p[:i], p[i:], true + } + } + + return p, nil, true + } +} diff --git a/token/bytes_test.go b/token/bytes_test.go new file mode 100644 index 0000000..08ae3f8 --- /dev/null +++ b/token/bytes_test.go @@ -0,0 +1,2040 @@ +package token_test + +import ( + "testing" + + "bytes" + + "sourcecode.social/reiver/go-rfc2616/token" +) + +func TestBytes(t *testing.T) { + + tests := []struct{ + Value []byte + ExpectedToken []byte + ExpectedRest []byte + ExpectedOK bool + }{ + { + Value: []byte("CONNECT /once/twice/thrice/fource.txt HTTP/1.1\r\n"), + ExpectedToken: []byte("CONNECT"), + ExpectedRest: []byte(" /once/twice/thrice/fource.txt HTTP/1.1\r\n"), + ExpectedOK: true, + }, + { + Value: []byte("DELETE /once/twice/thrice/fource.txt HTTP/1.1\r\n"), + ExpectedToken: []byte("DELETE"), + ExpectedRest: []byte(" /once/twice/thrice/fource.txt HTTP/1.1\r\n"), + ExpectedOK: true, + }, + { + Value: []byte("GET /once/twice/thrice/fource.txt HTTP/1.1\r\n"), + ExpectedToken: []byte("GET"), + ExpectedRest: []byte(" /once/twice/thrice/fource.txt HTTP/1.1\r\n"), + ExpectedOK: true, + }, + { + Value: []byte("HEAD /once/twice/thrice/fource.txt HTTP/1.1\r\n"), + ExpectedToken: []byte("HEAD"), + ExpectedRest: []byte(" /once/twice/thrice/fource.txt HTTP/1.1\r\n"), + ExpectedOK: true, + }, + { + Value: []byte("PATCH /once/twice/thrice/fource.txt HTTP/1.1\r\n"), + ExpectedToken: []byte("PATCH"), + ExpectedRest: []byte(" /once/twice/thrice/fource.txt HTTP/1.1\r\n"), + ExpectedOK: true, + }, + { + Value: []byte("POST /once/twice/thrice/fource.txt HTTP/1.1\r\n"), + ExpectedToken: []byte("POST"), + ExpectedRest: []byte(" /once/twice/thrice/fource.txt HTTP/1.1\r\n"), + ExpectedOK: true, + }, + { + Value: []byte("PUT /once/twice/thrice/fource.txt HTTP/1.1\r\n"), + ExpectedToken: []byte("PUT"), + ExpectedRest: []byte(" /once/twice/thrice/fource.txt HTTP/1.1\r\n"), + ExpectedOK: true, + }, + { + Value: []byte("TRACE /once/twice/thrice/fource.txt HTTP/1.1\r\n"), + ExpectedToken: []byte("TRACE"), + ExpectedRest: []byte(" /once/twice/thrice/fource.txt HTTP/1.1\r\n"), + ExpectedOK: true, + }, + + + + { + Value: []byte("KICK /apple/banana/cherry.php HTTP/1.1\r\n"), + ExpectedToken: []byte("KICK"), + ExpectedRest: []byte(" /apple/banana/cherry.php HTTP/1.1\r\n"), + ExpectedOK: true, + }, + { + Value: []byte("PUNCH /apple/banana/cherry.php HTTP/1.1\r\n"), + ExpectedToken: []byte("PUNCH"), + ExpectedRest: []byte(" /apple/banana/cherry.php HTTP/1.1\r\n"), + ExpectedOK: true, + }, + + + + { + Value: nil, + ExpectedToken: nil, + ExpectedRest: nil, + ExpectedOK: false, + }, + { + Value: []byte(nil), + ExpectedToken: nil, + ExpectedRest: nil, + ExpectedOK: false, + }, + + + + { + Value: []byte(""), + ExpectedToken: nil, + ExpectedRest: nil, + ExpectedOK: false, + }, + + + + { + Value: []byte{0}, + ExpectedToken: nil, + ExpectedRest: []byte{0}, + ExpectedOK: false, + }, + { + Value: []byte{1}, + ExpectedToken: nil, + ExpectedRest: []byte{1}, + ExpectedOK: false, + }, + { + Value: []byte{2}, + ExpectedToken: nil, + ExpectedRest: []byte{2}, + ExpectedOK: false, + }, + { + Value: []byte{3}, + ExpectedToken: nil, + ExpectedRest: []byte{3}, + ExpectedOK: false, + }, + { + Value: []byte{4}, + ExpectedToken: nil, + ExpectedRest: []byte{4}, + ExpectedOK: false, + }, + { + Value: []byte{5}, + ExpectedToken: nil, + ExpectedRest: []byte{5}, + ExpectedOK: false, + }, + { + Value: []byte{6}, + ExpectedToken: nil, + ExpectedRest: []byte{6}, + ExpectedOK: false, + }, + { + Value: []byte{7}, + ExpectedToken: nil, + ExpectedRest: []byte{7}, + ExpectedOK: false, + }, + { + Value: []byte{8}, + ExpectedToken: nil, + ExpectedRest: []byte{8}, + ExpectedOK: false, + }, + { + Value: []byte{9}, + ExpectedToken: nil, + ExpectedRest: []byte{9}, + ExpectedOK: false, + }, + { + Value: []byte{10}, + ExpectedToken: nil, + ExpectedRest: []byte{10}, + ExpectedOK: false, + }, + { + Value: []byte{11}, + ExpectedToken: nil, + ExpectedRest: []byte{11}, + ExpectedOK: false, + }, + { + Value: []byte{12}, + ExpectedToken: nil, + ExpectedRest: []byte{12}, + ExpectedOK: false, + }, + { + Value: []byte{13}, + ExpectedToken: nil, + ExpectedRest: []byte{13}, + ExpectedOK: false, + }, + { + Value: []byte{14}, + ExpectedToken: nil, + ExpectedRest: []byte{14}, + ExpectedOK: false, + }, + { + Value: []byte{15}, + ExpectedToken: nil, + ExpectedRest: []byte{15}, + ExpectedOK: false, + }, + { + Value: []byte{16}, + ExpectedToken: nil, + ExpectedRest: []byte{16}, + ExpectedOK: false, + }, + { + Value: []byte{17}, + ExpectedToken: nil, + ExpectedRest: []byte{17}, + ExpectedOK: false, + }, + { + Value: []byte{18}, + ExpectedToken: nil, + ExpectedRest: []byte{18}, + ExpectedOK: false, + }, + { + Value: []byte{19}, + ExpectedToken: nil, + ExpectedRest: []byte{19}, + ExpectedOK: false, + }, + { + Value: []byte{20}, + ExpectedToken: nil, + ExpectedRest: []byte{20}, + ExpectedOK: false, + }, + { + Value: []byte{21}, + ExpectedToken: nil, + ExpectedRest: []byte{21}, + ExpectedOK: false, + }, + { + Value: []byte{22}, + ExpectedToken: nil, + ExpectedRest: []byte{22}, + ExpectedOK: false, + }, + { + Value: []byte{23}, + ExpectedToken: nil, + ExpectedRest: []byte{23}, + ExpectedOK: false, + }, + { + Value: []byte{24}, + ExpectedToken: nil, + ExpectedRest: []byte{24}, + ExpectedOK: false, + }, + { + Value: []byte{25}, + ExpectedToken: nil, + ExpectedRest: []byte{25}, + ExpectedOK: false, + }, + { + Value: []byte{26}, + ExpectedToken: nil, + ExpectedRest: []byte{26}, + ExpectedOK: false, + }, + { + Value: []byte{27}, + ExpectedToken: nil, + ExpectedRest: []byte{27}, + ExpectedOK: false, + }, + { + Value: []byte{28}, + ExpectedToken: nil, + ExpectedRest: []byte{28}, + ExpectedOK: false, + }, + { + Value: []byte{29}, + ExpectedToken: nil, + ExpectedRest: []byte{29}, + ExpectedOK: false, + }, + { + Value: []byte{30}, + ExpectedToken: nil, + ExpectedRest: []byte{30}, + ExpectedOK: false, + }, + { + Value: []byte{31}, + ExpectedToken: nil, + ExpectedRest: []byte{31}, + ExpectedOK: false, + }, + { + Value: []byte{127}, + ExpectedToken: nil, + ExpectedRest: []byte{127}, + ExpectedOK: false, + }, + + + + { + Value: []byte("("), + ExpectedToken: nil, + ExpectedRest: []byte("("), + ExpectedOK: false, + }, + { + Value: []byte(")"), + ExpectedToken: nil, + ExpectedRest: []byte(")"), + ExpectedOK: false, + }, + { + Value: []byte("<"), + ExpectedToken: nil, + ExpectedRest: []byte("<"), + ExpectedOK: false, + }, + { + Value: []byte(">"), + ExpectedToken: nil, + ExpectedRest: []byte(">"), + ExpectedOK: false, + }, + { + Value: []byte("@"), + ExpectedToken: nil, + ExpectedRest: []byte("@"), + ExpectedOK: false, + }, + { + Value: []byte(","), + ExpectedToken: nil, + ExpectedRest: []byte(","), + ExpectedOK: false, + }, + { + Value: []byte(";"), + ExpectedToken: nil, + ExpectedRest: []byte(";"), + ExpectedOK: false, + }, + { + Value: []byte(":"), + ExpectedToken: nil, + ExpectedRest: []byte(":"), + ExpectedOK: false, + }, + { + Value: []byte(`\`), + ExpectedToken: nil, + ExpectedRest: []byte(`\`), + ExpectedOK: false, + }, + { + Value: []byte(`"`), + ExpectedToken: nil, + ExpectedRest: []byte(`"`), + ExpectedOK: false, + }, + { + Value: []byte("/"), + ExpectedToken: nil, + ExpectedRest: []byte("/"), + ExpectedOK: false, + }, + { + Value: []byte("["), + ExpectedToken: nil, + ExpectedRest: []byte("["), + ExpectedOK: false, + }, + { + Value: []byte("]"), + ExpectedToken: nil, + ExpectedRest: []byte("]"), + ExpectedOK: false, + }, + { + Value: []byte("?"), + ExpectedToken: nil, + ExpectedRest: []byte("?"), + ExpectedOK: false, + }, + { + Value: []byte("="), + ExpectedToken: nil, + ExpectedRest: []byte("="), + ExpectedOK: false, + }, + { + Value: []byte("{"), + ExpectedToken: nil, + ExpectedRest: []byte("{"), + ExpectedOK: false, + }, + { + Value: []byte("}"), + ExpectedToken: nil, + ExpectedRest: []byte("}"), + ExpectedOK: false, + }, + { + Value: []byte(" "), + ExpectedToken: nil, + ExpectedRest: []byte(" "), + ExpectedOK: false, + }, + { + Value: []byte("\t"), + ExpectedToken: nil, + ExpectedRest: []byte("\t"), + ExpectedOK: false, + }, + + + + { + Value: []byte{0x00}, + ExpectedToken: nil, + ExpectedRest: []byte{0x00}, + ExpectedOK: false, + }, + { + Value: []byte{0x01}, + ExpectedToken: nil, + ExpectedRest: []byte{0x01}, + ExpectedOK: false, + }, + { + Value: []byte{0x02}, + ExpectedToken: nil, + ExpectedRest: []byte{0x02}, + ExpectedOK: false, + }, + { + Value: []byte{0x03}, + ExpectedToken: nil, + ExpectedRest: []byte{0x03}, + ExpectedOK: false, + }, + { + Value: []byte{0x04}, + ExpectedToken: nil, + ExpectedRest: []byte{0x04}, + ExpectedOK: false, + }, + { + Value: []byte{0x05}, + ExpectedToken: nil, + ExpectedRest: []byte{0x05}, + ExpectedOK: false, + }, + { + Value: []byte{0x06}, + ExpectedToken: nil, + ExpectedRest: []byte{0x06}, + ExpectedOK: false, + }, + { + Value: []byte{0x07}, + ExpectedToken: nil, + ExpectedRest: []byte{0x07}, + ExpectedOK: false, + }, + { + Value: []byte{0x08}, + ExpectedToken: nil, + ExpectedRest: []byte{0x08}, + ExpectedOK: false, + }, + { + Value: []byte{0x09}, + ExpectedToken: nil, + ExpectedRest: []byte{0x09}, + ExpectedOK: false, + }, + { + Value: []byte{0x0A}, + ExpectedToken: nil, + ExpectedRest: []byte{0x0A}, + ExpectedOK: false, + }, + { + Value: []byte{0x0B}, + ExpectedToken: nil, + ExpectedRest: []byte{0x0B}, + ExpectedOK: false, + }, + { + Value: []byte{0x0C}, + ExpectedToken: nil, + ExpectedRest: []byte{0x0C}, + ExpectedOK: false, + }, + { + Value: []byte{0x0D}, + ExpectedToken: nil, + ExpectedRest: []byte{0x0D}, + ExpectedOK: false, + }, + { + Value: []byte{0x0E}, + ExpectedToken: nil, + ExpectedRest: []byte{0x0E}, + ExpectedOK: false, + }, + { + Value: []byte{0x0F}, + ExpectedToken: nil, + ExpectedRest: []byte{0x0F}, + ExpectedOK: false, + }, + + + { + Value: []byte{0x10}, + ExpectedToken: nil, + ExpectedRest: []byte{0x10}, + ExpectedOK: false, + }, + { + Value: []byte{0x11}, + ExpectedToken: nil, + ExpectedRest: []byte{0x11}, + ExpectedOK: false, + }, + { + Value: []byte{0x12}, + ExpectedToken: nil, + ExpectedRest: []byte{0x12}, + ExpectedOK: false, + }, + { + Value: []byte{0x13}, + ExpectedToken: nil, + ExpectedRest: []byte{0x13}, + ExpectedOK: false, + }, + { + Value: []byte{0x14}, + ExpectedToken: nil, + ExpectedRest: []byte{0x14}, + ExpectedOK: false, + }, + { + Value: []byte{0x15}, + ExpectedToken: nil, + ExpectedRest: []byte{0x15}, + ExpectedOK: false, + }, + { + Value: []byte{0x16}, + ExpectedToken: nil, + ExpectedRest: []byte{0x16}, + ExpectedOK: false, + }, + { + Value: []byte{0x17}, + ExpectedToken: nil, + ExpectedRest: []byte{0x17}, + ExpectedOK: false, + }, + { + Value: []byte{0x18}, + ExpectedToken: nil, + ExpectedRest: []byte{0x18}, + ExpectedOK: false, + }, + { + Value: []byte{0x19}, + ExpectedToken: nil, + ExpectedRest: []byte{0x19}, + ExpectedOK: false, + }, + { + Value: []byte{0x1A}, + ExpectedToken: nil, + ExpectedRest: []byte{0x1A}, + ExpectedOK: false, + }, + { + Value: []byte{0x1B}, + ExpectedToken: nil, + ExpectedRest: []byte{0x1B}, + ExpectedOK: false, + }, + { + Value: []byte{0x1C}, + ExpectedToken: nil, + ExpectedRest: []byte{0x1C}, + ExpectedOK: false, + }, + { + Value: []byte{0x1D}, + ExpectedToken: nil, + ExpectedRest: []byte{0x1D}, + ExpectedOK: false, + }, + { + Value: []byte{0x1E}, + ExpectedToken: nil, + ExpectedRest: []byte{0x1E}, + ExpectedOK: false, + }, + { + Value: []byte{0x1F}, + ExpectedToken: nil, + ExpectedRest: []byte{0x1F}, + ExpectedOK: false, + }, + + + { + Value: []byte{0x20}, + ExpectedToken: nil, + ExpectedRest: []byte{0x20}, + ExpectedOK: false, + }, + { + Value: []byte{0x21}, // == '!' + ExpectedToken: []byte{0x21}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x22}, + ExpectedToken: nil, + ExpectedRest: []byte{0x22}, + ExpectedOK: false, + }, + { + Value: []byte{0x23}, // == '#' + ExpectedToken: []byte{0x23}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x24}, // == '$' + ExpectedToken: []byte{0x24}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x25}, // == '%' + ExpectedToken: []byte{0x25}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x26}, // == '&' + ExpectedToken: []byte{0x26}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x27}, // == '\'' + ExpectedToken: []byte{0x27}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x28}, + ExpectedToken: nil, + ExpectedRest: []byte{0x28}, + ExpectedOK: false, + }, + { + Value: []byte{0x29}, + ExpectedToken: nil, + ExpectedRest: []byte{0x29}, + ExpectedOK: false, + }, + { + Value: []byte{0x2A}, // == '*' + ExpectedToken: []byte{0x2A}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x2B}, // == '+' + ExpectedToken: []byte{0x2B}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x2C}, + ExpectedToken: nil, + ExpectedRest: []byte{0x2C}, + ExpectedOK: false, + }, + { + Value: []byte{0x2D}, // == '-' + ExpectedToken: []byte{0x2D}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x2E}, // == '.' + ExpectedToken: []byte{0x2E}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x2F}, + ExpectedToken: nil, + ExpectedRest: []byte{0x2F}, + ExpectedOK: false, + }, + + + { + Value: []byte{0x30}, // == '0' + ExpectedToken: []byte{0x30}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x31}, // == '1' + ExpectedToken: []byte{0x31}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x32}, // == '2' + ExpectedToken: []byte{0x32}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x33}, // == '3' + ExpectedToken: []byte{0x33}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x34}, // == '4' + ExpectedToken: []byte{0x34}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x35}, // == '5' + ExpectedToken: []byte{0x35}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x36}, // == '6' + ExpectedToken: []byte{0x36}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x37}, // == '7' + ExpectedToken: []byte{0x37}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x38}, // == '8' + ExpectedToken: []byte{0x38}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x39}, // == '9' + ExpectedToken: []byte{0x39}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x3A}, + ExpectedToken: nil, + ExpectedRest: []byte{0x3A}, + ExpectedOK: false, + }, + { + Value: []byte{0x3B}, + ExpectedToken: nil, + ExpectedRest: []byte{0x3B}, + ExpectedOK: false, + }, + { + Value: []byte{0x3C}, + ExpectedToken: nil, + ExpectedRest: []byte{0x3C}, + ExpectedOK: false, + }, + { + Value: []byte{0x3D}, + ExpectedToken: nil, + ExpectedRest: []byte{0x3D}, + ExpectedOK: false, + }, + { + Value: []byte{0x3E}, + ExpectedToken: nil, + ExpectedRest: []byte{0x3E}, + ExpectedOK: false, + }, + { + Value: []byte{0x3F}, + ExpectedToken: nil, + ExpectedRest: []byte{0x3F}, + ExpectedOK: false, + }, + + + { + Value: []byte{0x40}, + ExpectedToken: nil, + ExpectedRest: []byte{0x40}, + ExpectedOK: false, + }, + { + Value: []byte{0x41}, // == 'A' + ExpectedToken: []byte{0x41}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x42}, // == 'B' + ExpectedToken: []byte{0x42}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x43}, // == 'C' + ExpectedToken: []byte{0x43}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x44}, // == 'D' + ExpectedToken: []byte{0x44}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x45}, // == 'E' + ExpectedToken: []byte{0x45}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x46}, // == 'F' + ExpectedToken: []byte{0x46}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x47}, // == 'G' + ExpectedToken: []byte{0x47}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x48}, // == 'H' + ExpectedToken: []byte{0x48}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x49}, // == 'I' + ExpectedToken: []byte{0x49}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x4A}, // == 'J' + ExpectedToken: []byte{0x4A}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x4B}, // == 'K' + ExpectedToken: []byte{0x4B}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x4C}, // == 'L' + ExpectedToken: []byte{0x4C}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x4D}, // == 'M' + ExpectedToken: []byte{0x4D}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x4E}, // == 'N' + ExpectedToken: []byte{0x4E}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x4F}, // == 'O' + ExpectedToken: []byte{0x4F}, + ExpectedRest: nil, + ExpectedOK: true, + }, + + + { + Value: []byte{0x50}, // == 'P' + ExpectedToken: []byte{0x50}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x51}, // == 'Q' + ExpectedToken: []byte{0x51}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x52}, // == 'R' + ExpectedToken: []byte{0x52}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x53}, // == 'S' + ExpectedToken: []byte{0x53}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x54}, // == 'T' + ExpectedToken: []byte{0x54}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x55}, // == 'U' + ExpectedToken: []byte{0x55}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x56}, // == 'V' + ExpectedToken: []byte{0x56}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x57}, // == 'W' + ExpectedToken: []byte{0x57}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x58}, // == 'X' + ExpectedToken: []byte{0x58}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x59}, // == 'Y' + ExpectedToken: []byte{0x59}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x5A}, // == 'Z' + ExpectedToken: []byte{0x5A}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x5B}, + ExpectedToken: nil, + ExpectedRest: []byte{0x5B}, + ExpectedOK: false, + }, + { + Value: []byte{0x5C}, + ExpectedToken: nil, + ExpectedRest: []byte{0x5C}, + ExpectedOK: false, + }, + { + Value: []byte{0x5D}, + ExpectedToken: nil, + ExpectedRest: []byte{0x5D}, + ExpectedOK: false, + }, + { + Value: []byte{0x5E}, // == '^' + ExpectedToken: []byte{0x5E}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x5F}, // == '_' + ExpectedToken: []byte{0x5F}, + ExpectedRest: nil, + ExpectedOK: true, + }, + + + { + Value: []byte{0x60}, // == '`' + ExpectedToken: []byte{0x60}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x61}, // == 'a' + ExpectedToken: []byte{0x61}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x62}, // == 'b' + ExpectedToken: []byte{0x62}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x63}, // == 'c' + ExpectedToken: []byte{0x63}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x64}, // == 'd' + ExpectedToken: []byte{0x64}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x65}, // == 'e' + ExpectedToken: []byte{0x65}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x66}, // == 'f' + ExpectedToken: []byte{0x66}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x67}, // == 'g' + ExpectedToken: []byte{0x67}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x68}, // == 'h' + ExpectedToken: []byte{0x68}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x69}, // == 'i' + ExpectedToken: []byte{0x69}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x6A}, // == 'j' + ExpectedToken: []byte{0x6A}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x6B}, // == 'k' + ExpectedToken: []byte{0x6B}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x6C}, // == 'l' + ExpectedToken: []byte{0x6C}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x6D}, // == 'm' + ExpectedToken: []byte{0x6D}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x6E}, // == 'n' + ExpectedToken: []byte{0x6E}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x6F}, // == 'o' + ExpectedToken: []byte{0x6F}, + ExpectedRest: nil, + ExpectedOK: true, + }, + + + { + Value: []byte{0x70}, // == 'p' + ExpectedToken: []byte{0x70}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x71}, // == 'q' + ExpectedToken: []byte{0x71}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x72}, // == 'r' + ExpectedToken: []byte{0x72}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x73}, // == 's' + ExpectedToken: []byte{0x73}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x74}, // == 't' + ExpectedToken: []byte{0x74}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x75}, // == 'u' + ExpectedToken: []byte{0x75}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x76}, // == 'v' + ExpectedToken: []byte{0x76}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x77}, // == 'w' + ExpectedToken: []byte{0x77}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x78}, // == 'x' + ExpectedToken: []byte{0x78}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x79}, // == 'y' + ExpectedToken: []byte{0x79}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x7A}, // == 'z' + ExpectedToken: []byte{0x7A}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x7B}, + ExpectedToken: nil, + ExpectedRest: []byte{0x7B}, + ExpectedOK: false, + }, + { + Value: []byte{0x7C}, // == '|' + ExpectedToken: []byte{0x7C}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x7D}, + ExpectedToken: nil, + ExpectedRest: []byte{0x7D}, + ExpectedOK: false, + }, + { + Value: []byte{0x7E}, // == '~' + ExpectedToken: []byte{0x7E}, + ExpectedRest: nil, + ExpectedOK: true, + }, + { + Value: []byte{0x7F}, + ExpectedToken: nil, + ExpectedRest: []byte{0x7F}, + ExpectedOK: false, + }, + + + { + Value: []byte{0x80}, + ExpectedToken: nil, + ExpectedRest: []byte{0x80}, + ExpectedOK: false, + }, + { + Value: []byte{0x81}, + ExpectedToken: nil, + ExpectedRest: []byte{0x81}, + ExpectedOK: false, + }, + { + Value: []byte{0x82}, + ExpectedToken: nil, + ExpectedRest: []byte{0x82}, + ExpectedOK: false, + }, + { + Value: []byte{0x83}, + ExpectedToken: nil, + ExpectedRest: []byte{0x83}, + ExpectedOK: false, + }, + { + Value: []byte{0x84}, + ExpectedToken: nil, + ExpectedRest: []byte{0x84}, + ExpectedOK: false, + }, + { + Value: []byte{0x85}, + ExpectedToken: nil, + ExpectedRest: []byte{0x85}, + ExpectedOK: false, + }, + { + Value: []byte{0x86}, + ExpectedToken: nil, + ExpectedRest: []byte{0x86}, + ExpectedOK: false, + }, + { + Value: []byte{0x87}, + ExpectedToken: nil, + ExpectedRest: []byte{0x87}, + ExpectedOK: false, + }, + { + Value: []byte{0x88}, + ExpectedToken: nil, + ExpectedRest: []byte{0x88}, + ExpectedOK: false, + }, + { + Value: []byte{0x89}, + ExpectedToken: nil, + ExpectedRest: []byte{0x89}, + ExpectedOK: false, + }, + { + Value: []byte{0x8A}, + ExpectedToken: nil, + ExpectedRest: []byte{0x8A}, + ExpectedOK: false, + }, + { + Value: []byte{0x8B}, + ExpectedToken: nil, + ExpectedRest: []byte{0x8B}, + ExpectedOK: false, + }, + { + Value: []byte{0x8C}, + ExpectedToken: nil, + ExpectedRest: []byte{0x8C}, + ExpectedOK: false, + }, + { + Value: []byte{0x8D}, + ExpectedToken: nil, + ExpectedRest: []byte{0x8D}, + ExpectedOK: false, + }, + { + Value: []byte{0x8E}, + ExpectedToken: nil, + ExpectedRest: []byte{0x8E}, + ExpectedOK: false, + }, + { + Value: []byte{0x8F}, + ExpectedToken: nil, + ExpectedRest: []byte{0x8F}, + ExpectedOK: false, + }, + + + { + Value: []byte{0x90}, + ExpectedToken: nil, + ExpectedRest: []byte{0x90}, + ExpectedOK: false, + }, + { + Value: []byte{0x91}, + ExpectedToken: nil, + ExpectedRest: []byte{0x91}, + ExpectedOK: false, + }, + { + Value: []byte{0x92}, + ExpectedToken: nil, + ExpectedRest: []byte{0x92}, + ExpectedOK: false, + }, + { + Value: []byte{0x93}, + ExpectedToken: nil, + ExpectedRest: []byte{0x93}, + ExpectedOK: false, + }, + { + Value: []byte{0x94}, + ExpectedToken: nil, + ExpectedRest: []byte{0x94}, + ExpectedOK: false, + }, + { + Value: []byte{0x95}, + ExpectedToken: nil, + ExpectedRest: []byte{0x95}, + ExpectedOK: false, + }, + { + Value: []byte{0x96}, + ExpectedToken: nil, + ExpectedRest: []byte{0x96}, + ExpectedOK: false, + }, + { + Value: []byte{0x97}, + ExpectedToken: nil, + ExpectedRest: []byte{0x97}, + ExpectedOK: false, + }, + { + Value: []byte{0x98}, + ExpectedToken: nil, + ExpectedRest: []byte{0x98}, + ExpectedOK: false, + }, + { + Value: []byte{0x99}, + ExpectedToken: nil, + ExpectedRest: []byte{0x99}, + ExpectedOK: false, + }, + { + Value: []byte{0x9A}, + ExpectedToken: nil, + ExpectedRest: []byte{0x9A}, + ExpectedOK: false, + }, + { + Value: []byte{0x9B}, + ExpectedToken: nil, + ExpectedRest: []byte{0x9B}, + ExpectedOK: false, + }, + { + Value: []byte{0x9C}, + ExpectedToken: nil, + ExpectedRest: []byte{0x9C}, + ExpectedOK: false, + }, + { + Value: []byte{0x9D}, + ExpectedToken: nil, + ExpectedRest: []byte{0x9D}, + ExpectedOK: false, + }, + { + Value: []byte{0x9E}, + ExpectedToken: nil, + ExpectedRest: []byte{0x9E}, + ExpectedOK: false, + }, + { + Value: []byte{0x9F}, + ExpectedToken: nil, + ExpectedRest: []byte{0x9F}, + ExpectedOK: false, + }, + + + { + Value: []byte{0xA0}, + ExpectedToken: nil, + ExpectedRest: []byte{0xA0}, + ExpectedOK: false, + }, + { + Value: []byte{0xA1}, + ExpectedToken: nil, + ExpectedRest: []byte{0xA1}, + ExpectedOK: false, + }, + { + Value: []byte{0xA2}, + ExpectedToken: nil, + ExpectedRest: []byte{0xA2}, + ExpectedOK: false, + }, + { + Value: []byte{0xA3}, + ExpectedToken: nil, + ExpectedRest: []byte{0xA3}, + ExpectedOK: false, + }, + { + Value: []byte{0xA4}, + ExpectedToken: nil, + ExpectedRest: []byte{0xA4}, + ExpectedOK: false, + }, + { + Value: []byte{0xA5}, + ExpectedToken: nil, + ExpectedRest: []byte{0xA5}, + ExpectedOK: false, + }, + { + Value: []byte{0xA6}, + ExpectedToken: nil, + ExpectedRest: []byte{0xA6}, + ExpectedOK: false, + }, + { + Value: []byte{0xA7}, + ExpectedToken: nil, + ExpectedRest: []byte{0xA7}, + ExpectedOK: false, + }, + { + Value: []byte{0xA8}, + ExpectedToken: nil, + ExpectedRest: []byte{0xA8}, + ExpectedOK: false, + }, + { + Value: []byte{0xA9}, + ExpectedToken: nil, + ExpectedRest: []byte{0xA9}, + ExpectedOK: false, + }, + { + Value: []byte{0xAA}, + ExpectedToken: nil, + ExpectedRest: []byte{0xAA}, + ExpectedOK: false, + }, + { + Value: []byte{0xAB}, + ExpectedToken: nil, + ExpectedRest: []byte{0xAB}, + ExpectedOK: false, + }, + { + Value: []byte{0xAC}, + ExpectedToken: nil, + ExpectedRest: []byte{0xAC}, + ExpectedOK: false, + }, + { + Value: []byte{0xAD}, + ExpectedToken: nil, + ExpectedRest: []byte{0xAD}, + ExpectedOK: false, + }, + { + Value: []byte{0xAE}, + ExpectedToken: nil, + ExpectedRest: []byte{0xAE}, + ExpectedOK: false, + }, + { + Value: []byte{0xAF}, + ExpectedToken: nil, + ExpectedRest: []byte{0xAF}, + ExpectedOK: false, + }, + + + { + Value: []byte{0xB0}, + ExpectedToken: nil, + ExpectedRest: []byte{0xB0}, + ExpectedOK: false, + }, + { + Value: []byte{0xB1}, + ExpectedToken: nil, + ExpectedRest: []byte{0xB1}, + ExpectedOK: false, + }, + { + Value: []byte{0xB2}, + ExpectedToken: nil, + ExpectedRest: []byte{0xB2}, + ExpectedOK: false, + }, + { + Value: []byte{0xB3}, + ExpectedToken: nil, + ExpectedRest: []byte{0xB3}, + ExpectedOK: false, + }, + { + Value: []byte{0xB4}, + ExpectedToken: nil, + ExpectedRest: []byte{0xB4}, + ExpectedOK: false, + }, + { + Value: []byte{0xB5}, + ExpectedToken: nil, + ExpectedRest: []byte{0xB5}, + ExpectedOK: false, + }, + { + Value: []byte{0xB6}, + ExpectedToken: nil, + ExpectedRest: []byte{0xB6}, + ExpectedOK: false, + }, + { + Value: []byte{0xB7}, + ExpectedToken: nil, + ExpectedRest: []byte{0xB7}, + ExpectedOK: false, + }, + { + Value: []byte{0xB8}, + ExpectedToken: nil, + ExpectedRest: []byte{0xB8}, + ExpectedOK: false, + }, + { + Value: []byte{0xB9}, + ExpectedToken: nil, + ExpectedRest: []byte{0xB9}, + ExpectedOK: false, + }, + { + Value: []byte{0xBA}, + ExpectedToken: nil, + ExpectedRest: []byte{0xBA}, + ExpectedOK: false, + }, + { + Value: []byte{0xBB}, + ExpectedToken: nil, + ExpectedRest: []byte{0xBB}, + ExpectedOK: false, + }, + { + Value: []byte{0xBC}, + ExpectedToken: nil, + ExpectedRest: []byte{0xBC}, + ExpectedOK: false, + }, + { + Value: []byte{0xBD}, + ExpectedToken: nil, + ExpectedRest: []byte{0xBD}, + ExpectedOK: false, + }, + { + Value: []byte{0xBE}, + ExpectedToken: nil, + ExpectedRest: []byte{0xBE}, + ExpectedOK: false, + }, + { + Value: []byte{0xBF}, + ExpectedToken: nil, + ExpectedRest: []byte{0xBF}, + ExpectedOK: false, + }, + + + { + Value: []byte{0xC0}, + ExpectedToken: nil, + ExpectedRest: []byte{0xC0}, + ExpectedOK: false, + }, + { + Value: []byte{0xC1}, + ExpectedToken: nil, + ExpectedRest: []byte{0xC1}, + ExpectedOK: false, + }, + { + Value: []byte{0xC2}, + ExpectedToken: nil, + ExpectedRest: []byte{0xC2}, + ExpectedOK: false, + }, + { + Value: []byte{0xC3}, + ExpectedToken: nil, + ExpectedRest: []byte{0xC3}, + ExpectedOK: false, + }, + { + Value: []byte{0xC4}, + ExpectedToken: nil, + ExpectedRest: []byte{0xC4}, + ExpectedOK: false, + }, + { + Value: []byte{0xC5}, + ExpectedToken: nil, + ExpectedRest: []byte{0xC5}, + ExpectedOK: false, + }, + { + Value: []byte{0xC6}, + ExpectedToken: nil, + ExpectedRest: []byte{0xC6}, + ExpectedOK: false, + }, + { + Value: []byte{0xC7}, + ExpectedToken: nil, + ExpectedRest: []byte{0xC7}, + ExpectedOK: false, + }, + { + Value: []byte{0xC8}, + ExpectedToken: nil, + ExpectedRest: []byte{0xC8}, + ExpectedOK: false, + }, + { + Value: []byte{0xC9}, + ExpectedToken: nil, + ExpectedRest: []byte{0xC9}, + ExpectedOK: false, + }, + { + Value: []byte{0xCA}, + ExpectedToken: nil, + ExpectedRest: []byte{0xCA}, + ExpectedOK: false, + }, + { + Value: []byte{0xCB}, + ExpectedToken: nil, + ExpectedRest: []byte{0xCB}, + ExpectedOK: false, + }, + { + Value: []byte{0xCC}, + ExpectedToken: nil, + ExpectedRest: []byte{0xCC}, + ExpectedOK: false, + }, + { + Value: []byte{0xCD}, + ExpectedToken: nil, + ExpectedRest: []byte{0xCD}, + ExpectedOK: false, + }, + { + Value: []byte{0xCE}, + ExpectedToken: nil, + ExpectedRest: []byte{0xCE}, + ExpectedOK: false, + }, + { + Value: []byte{0xCF}, + ExpectedToken: nil, + ExpectedRest: []byte{0xCF}, + ExpectedOK: false, + }, + + + { + Value: []byte{0xD0}, + ExpectedToken: nil, + ExpectedRest: []byte{0xD0}, + ExpectedOK: false, + }, + { + Value: []byte{0xD1}, + ExpectedToken: nil, + ExpectedRest: []byte{0xD1}, + ExpectedOK: false, + }, + { + Value: []byte{0xD2}, + ExpectedToken: nil, + ExpectedRest: []byte{0xD2}, + ExpectedOK: false, + }, + { + Value: []byte{0xD3}, + ExpectedToken: nil, + ExpectedRest: []byte{0xD3}, + ExpectedOK: false, + }, + { + Value: []byte{0xD4}, + ExpectedToken: nil, + ExpectedRest: []byte{0xD4}, + ExpectedOK: false, + }, + { + Value: []byte{0xD5}, + ExpectedToken: nil, + ExpectedRest: []byte{0xD5}, + ExpectedOK: false, + }, + { + Value: []byte{0xD6}, + ExpectedToken: nil, + ExpectedRest: []byte{0xD6}, + ExpectedOK: false, + }, + { + Value: []byte{0xD7}, + ExpectedToken: nil, + ExpectedRest: []byte{0xD7}, + ExpectedOK: false, + }, + { + Value: []byte{0xD8}, + ExpectedToken: nil, + ExpectedRest: []byte{0xD8}, + ExpectedOK: false, + }, + { + Value: []byte{0xD9}, + ExpectedToken: nil, + ExpectedRest: []byte{0xD9}, + ExpectedOK: false, + }, + { + Value: []byte{0xDA}, + ExpectedToken: nil, + ExpectedRest: []byte{0xDA}, + ExpectedOK: false, + }, + { + Value: []byte{0xDB}, + ExpectedToken: nil, + ExpectedRest: []byte{0xDB}, + ExpectedOK: false, + }, + { + Value: []byte{0xDC}, + ExpectedToken: nil, + ExpectedRest: []byte{0xDC}, + ExpectedOK: false, + }, + { + Value: []byte{0xDD}, + ExpectedToken: nil, + ExpectedRest: []byte{0xDD}, + ExpectedOK: false, + }, + { + Value: []byte{0xDE}, + ExpectedToken: nil, + ExpectedRest: []byte{0xDE}, + ExpectedOK: false, + }, + { + Value: []byte{0xDF}, + ExpectedToken: nil, + ExpectedRest: []byte{0xDF}, + ExpectedOK: false, + }, + + + { + Value: []byte{0xE0}, + ExpectedToken: nil, + ExpectedRest: []byte{0xE0}, + ExpectedOK: false, + }, + { + Value: []byte{0xE1}, + ExpectedToken: nil, + ExpectedRest: []byte{0xE1}, + ExpectedOK: false, + }, + { + Value: []byte{0xE2}, + ExpectedToken: nil, + ExpectedRest: []byte{0xE2}, + ExpectedOK: false, + }, + { + Value: []byte{0xE3}, + ExpectedToken: nil, + ExpectedRest: []byte{0xE3}, + ExpectedOK: false, + }, + { + Value: []byte{0xE4}, + ExpectedToken: nil, + ExpectedRest: []byte{0xE4}, + ExpectedOK: false, + }, + { + Value: []byte{0xE5}, + ExpectedToken: nil, + ExpectedRest: []byte{0xE5}, + ExpectedOK: false, + }, + { + Value: []byte{0xE6}, + ExpectedToken: nil, + ExpectedRest: []byte{0xE6}, + ExpectedOK: false, + }, + { + Value: []byte{0xE7}, + ExpectedToken: nil, + ExpectedRest: []byte{0xE7}, + ExpectedOK: false, + }, + { + Value: []byte{0xE8}, + ExpectedToken: nil, + ExpectedRest: []byte{0xE8}, + ExpectedOK: false, + }, + { + Value: []byte{0xE9}, + ExpectedToken: nil, + ExpectedRest: []byte{0xE9}, + ExpectedOK: false, + }, + { + Value: []byte{0xEA}, + ExpectedToken: nil, + ExpectedRest: []byte{0xEA}, + ExpectedOK: false, + }, + { + Value: []byte{0xEB}, + ExpectedToken: nil, + ExpectedRest: []byte{0xEB}, + ExpectedOK: false, + }, + { + Value: []byte{0xEC}, + ExpectedToken: nil, + ExpectedRest: []byte{0xEC}, + ExpectedOK: false, + }, + { + Value: []byte{0xED}, + ExpectedToken: nil, + ExpectedRest: []byte{0xED}, + ExpectedOK: false, + }, + { + Value: []byte{0xEE}, + ExpectedToken: nil, + ExpectedRest: []byte{0xEE}, + ExpectedOK: false, + }, + { + Value: []byte{0xEF}, + ExpectedToken: nil, + ExpectedRest: []byte{0xEF}, + ExpectedOK: false, + }, + + + { + Value: []byte{0xF0}, + ExpectedToken: nil, + ExpectedRest: []byte{0xF0}, + ExpectedOK: false, + }, + { + Value: []byte{0xF1}, + ExpectedToken: nil, + ExpectedRest: []byte{0xF1}, + ExpectedOK: false, + }, + { + Value: []byte{0xF2}, + ExpectedToken: nil, + ExpectedRest: []byte{0xF2}, + ExpectedOK: false, + }, + { + Value: []byte{0xF3}, + ExpectedToken: nil, + ExpectedRest: []byte{0xF3}, + ExpectedOK: false, + }, + { + Value: []byte{0xF4}, + ExpectedToken: nil, + ExpectedRest: []byte{0xF4}, + ExpectedOK: false, + }, + { + Value: []byte{0xF5}, + ExpectedToken: nil, + ExpectedRest: []byte{0xF5}, + ExpectedOK: false, + }, + { + Value: []byte{0xF6}, + ExpectedToken: nil, + ExpectedRest: []byte{0xF6}, + ExpectedOK: false, + }, + { + Value: []byte{0xF7}, + ExpectedToken: nil, + ExpectedRest: []byte{0xF7}, + ExpectedOK: false, + }, + { + Value: []byte{0xF8}, + ExpectedToken: nil, + ExpectedRest: []byte{0xF8}, + ExpectedOK: false, + }, + { + Value: []byte{0xF9}, + ExpectedToken: nil, + ExpectedRest: []byte{0xF9}, + ExpectedOK: false, + }, + { + Value: []byte{0xFA}, + ExpectedToken: nil, + ExpectedRest: []byte{0xFA}, + ExpectedOK: false, + }, + { + Value: []byte{0xFB}, + ExpectedToken: nil, + ExpectedRest: []byte{0xFB}, + ExpectedOK: false, + }, + { + Value: []byte{0xFC}, + ExpectedToken: nil, + ExpectedRest: []byte{0xFC}, + ExpectedOK: false, + }, + { + Value: []byte{0xFD}, + ExpectedToken: nil, + ExpectedRest: []byte{0xFD}, + ExpectedOK: false, + }, + { + Value: []byte{0xFE}, + ExpectedToken: nil, + ExpectedRest: []byte{0xFE}, + ExpectedOK: false, + }, + { + Value: []byte{0xFF}, + ExpectedToken: nil, + ExpectedRest: []byte{0xFF}, + ExpectedOK: false, + }, + } + + for testNumber, test := range tests { + + actualToken, actualRest, actualOK := token.Bytes(test.Value) + + { + expected := test.ExpectedOK + actual := actualOK + + if expected != actual { + t.Errorf("For test #%d, the actual ok-result is not what was expected.", testNumber) + t.Logf("EXPECTED: %t", expected) + t.Logf("ACTUAL: %t", actual) + t.Logf("VALUE: %q", test.Value) + t.Logf("EXPECTED-TOKEN: %q (%#v)", test.ExpectedToken, test.ExpectedToken) + t.Logf("EXPECTED-REST: %q (%#v)", test.ExpectedRest, test.ExpectedRest) + continue + } + } + + { + expected := test.ExpectedToken + actual := actualToken + + if !bytes.Equal(expected, actual) { + t.Errorf("For test #%d, the actual token is not what was expected.", testNumber) + t.Logf("EXPECTED: %q", expected) + t.Logf("ACTUAL: %q", actual) + t.Logf("VALUE: %q", test.Value) + continue + } + } + + { + expected := test.ExpectedRest + actual := actualRest + + if !bytes.Equal(expected, actual) { + t.Errorf("For test #%d, the actual rest is not what was expected.", testNumber) + t.Logf("EXPECTED: %q", expected) + t.Logf("ACTUAL: %q", actual) + t.Logf("VALUE: %q", test.Value) + continue + } + } + } +}