Repository: goccy/go-yaml
Pull request #440 (2024-03-28 07:52:12)

Quote is required even if it begins with backquote.
Before submitting your PR, please confirm the following.

- [x] Describe the purpose for which you created this PR.  
- [x] Create test code that corresponds to the modification

Hi @goccy!

I noticed when using a mix of goccy/go-yaml and go-yaml/yaml that backquotes also cannot be used at the beginning of a token.

https://go.dev/play/p/ysabzaYClh3

I also got an error at https://www.yamllint.com/.

According to https://yaml.org/spec/1.2.2/  backquote (<code>`</code>) is reserved.

> The “@” (x40, at) and “`” (x60, grave accent) are reserved for future use.

Thank you.

codecov-commenter: ## [Codecov](https://app.codecov.io/gh/goccy/go-yaml/pull/440?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Masaaki+Goshima) Report
> Merging [#440](https://app.codecov.io/gh/goccy/go-yaml/pull/440?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Masaaki+Goshima) (e8b0e1d) into [master](https://app.codecov.io/gh/goccy/go-yaml/commit/4653a1bb5c0047bb37280ac341e2f091cb44352f?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Masaaki+Goshima) (4653a1b) will **not change** coverage.
> The diff coverage is `100.00%`.

:exclamation: Your organization needs to install the [Codecov GitHub app](https://github.com/apps/codecov/installations/select_target) to enable full functionality.

<details><summary>Additional details and impacted files</summary>


```diff
@@           Coverage Diff           @@
##           master     #440   +/-   ##
=======================================
  Coverage   76.02%   76.02%           
=======================================
  Files          13       13           
  Lines        4692     4692           
=======================================
  Hits         3567     3567           
  Misses        866      866           
  Partials      259      259           
```

</details>

Patch:
diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
index 647c692..74140f3 100644
--- a/.github/workflows/go.yml
+++ b/.github/workflows/go.yml
@@ -73,7 +73,8 @@ jobs:
       - name: measure coverage
         run: |
           make cover
-      - uses: codecov/codecov-action@v3
+      - uses: codecov/codecov-action@v4
         with:
           fail_ci_if_error: true
           verbose: true
+          token: ${{ secrets.CODECOV_TOKEN }}
diff --git a/decode.go b/decode.go
index 79117ca..72af5e2 100644
--- a/decode.go
+++ b/decode.go
@@ -15,11 +15,12 @@ import (
 	"strconv"
 	"time"
 
+	"golang.org/x/xerrors"
+
 	"github.com/goccy/go-yaml/ast"
 	"github.com/goccy/go-yaml/internal/errors"
 	"github.com/goccy/go-yaml/parser"
 	"github.com/goccy/go-yaml/token"
-	"golang.org/x/xerrors"
 )
 
 // Decoder reads and decodes YAML values from an input stream.
@@ -487,6 +488,21 @@ func (d *Decoder) fileToNode(f *ast.File) ast.Node {
 func (d *Decoder) convertValue(v reflect.Value, typ reflect.Type, src ast.Node) (reflect.Value, error) {
 	if typ.Kind() != reflect.String {
 		if !v.Type().ConvertibleTo(typ) {
+
+			// Special case for "strings -> floats" aka scientific notation
+			// If the destination type is a float and the source type is a string, check if we can
+			// use strconv.ParseFloat to convert the string to a float.
+			if (typ.Kind() == reflect.Float32 || typ.Kind() == reflect.Float64) &&
+				v.Type().Kind() == reflect.String {
+				if f, err := strconv.ParseFloat(v.String(), 64); err == nil {
+					if typ.Kind() == reflect.Float32 {
+						return reflect.ValueOf(float32(f)), nil
+					} else if typ.Kind() == reflect.Float64 {
+						return reflect.ValueOf(f), nil
+					}
+					// else, fall through to the error below
+				}
+			}
 			return reflect.Zero(typ), errTypeMismatch(typ, v.Type(), src.GetToken())
 		}
 		return v.Convert(typ), nil
@@ -876,6 +892,15 @@ func (d *Decoder) decodeValue(ctx context.Context, dst reflect.Value, src ast.No
 				dst.SetInt(int64(vv))
 				return nil
 			}
+		case string: // handle scientific notation
+			if i, err := strconv.ParseFloat(vv, 64); err == nil {
+				if 0 <= i && i <= math.MaxUint64 && !dst.OverflowInt(int64(i)) {
+					dst.SetInt(int64(i))
+					return nil
+				}
+			} else { // couldn't be parsed as float
+				return errTypeMismatch(valueType, reflect.TypeOf(v), src.GetToken())
+			}
 		default:
 			return errTypeMismatch(valueType, reflect.TypeOf(v), src.GetToken())
 		}
@@ -898,6 +923,16 @@ func (d *Decoder) decodeValue(ctx context.Context, dst reflect.Value, src ast.No
 				dst.SetUint(uint64(vv))
 				return nil
 			}
+		case string: // handle scientific notation
+			if i, err := strconv.ParseFloat(vv, 64); err == nil {
+				if 0 <= i && i <= math.MaxUint64 && !dst.OverflowUint(uint64(i)) {
+					dst.SetUint(uint64(i))
+					return nil
+				}
+			} else { // couldn't be parsed as float
+				return errTypeMismatch(valueType, reflect.TypeOf(v), src.GetToken())
+			}
+
 		default:
 			return errTypeMismatch(valueType, reflect.TypeOf(v), src.GetToken())
 		}
@@ -1500,10 +1535,19 @@ func (d *Decoder) decodeMap(ctx context.Context, dst reflect.Value, src ast.Node
 			}
 			continue
 		}
-		k := reflect.ValueOf(d.nodeToValue(key))
-		if k.IsValid() && k.Type().ConvertibleTo(keyType) {
-			k = k.Convert(keyType)
+
+		k := d.createDecodableValue(keyType)
+		if d.canDecodeByUnmarshaler(k) {
+			if err := d.decodeByUnmarshaler(ctx, k, key); err != nil {
+				return errors.Wrapf(err, "failed to decode by unmarshaler")
+			}
+		} else {
+			k = reflect.ValueOf(d.nodeToValue(key))
+			if k.IsValid() && k.Type().ConvertibleTo(keyType) {
+				k = k.Convert(keyType)
+			}
 		}
+
 		if k.IsValid() {
 			if err := d.validateDuplicateKey(keyMap, k.Interface(), key); err != nil {
 				return errors.Wrapf(err, "invalid map key")
diff --git a/path.go b/path.go
index 72554bd..b79c666 100644
--- a/path.go
+++ b/path.go
@@ -468,7 +468,7 @@ func (n *rootNode) String() string {
 
 func (n *rootNode) filter(node ast.Node) (ast.Node, error) {
 	if n.child == nil {
-		return nil, nil
+		return node, nil
 	}
 	filtered, err := n.child.filter(node)
 	if err != nil {
diff --git a/scanner/scanner.go b/scanner/scanner.go
index b0eac48..77acb41 100644
--- a/scanner/scanner.go
+++ b/scanner/scanner.go
@@ -4,8 +4,9 @@ import (
 	"io"
 	"strings"
 
-	"github.com/goccy/go-yaml/token"
 	"golang.org/x/xerrors"
+
+	"github.com/goccy/go-yaml/token"
 )
 
 // IndentState state for indent
@@ -316,100 +317,93 @@ func (s *Scanner) scanDoubleQuote(ctx *Context) (tk *token.Token, pos int) {
 			continue
 		} else if c == '\\' {
 			isFirstLineChar = false
-			if idx+1 < size {
-				nextChar := src[idx+1]
-				switch nextChar {
-				case 'b':
-					ctx.addOriginBuf(nextChar)
-					value = append(value, '\b')
-					idx++
-					continue
-				case 'e':
-					ctx.addOriginBuf(nextChar)
-					value = append(value, '\x1B')
-					idx++
-					continue
-				case 'f':
-					ctx.addOriginBuf(nextChar)
-					value = append(value, '\f')
-					idx++
-					continue
-				case 'n':
-					ctx.addOriginBuf(nextChar)
-					value = append(value, '\n')
-					idx++
-					continue
-				case 'r':
-					ctx.addOriginBuf(nextChar)
-					value = append(value, '\r')
-					idx++
-					continue
-				case 'v':
-					ctx.addOriginBuf(nextChar)
-					value = append(value, '\v')
-					idx++
-					continue
-				case 'L': // LS (#x2028)
-					ctx.addOriginBuf(nextChar)
-					value = append(value, []rune{'\xE2', '\x80', '\xA8'}...)
-					idx++
-					continue
-				case 'N': // NEL (#x85)
-					ctx.addOriginBuf(nextChar)
-					value = append(value, []rune{'\xC2', '\x85'}...)
-					idx++
-					continue
-				case 'P': // PS (#x2029)
-					ctx.addOriginBuf(nextChar)
-					value = append(value, []rune{'\xE2', '\x80', '\xA9'}...)
-					idx++
-					continue
-				case '_': // #xA0
-					ctx.addOriginBuf(nextChar)
-					value = append(value, []rune{'\xC2', '\xA0'}...)
-					idx++
-					continue
-				case '"':
-					ctx.addOriginBuf(nextChar)
-					value = append(value, nextChar)
-					idx++
-					continue
-				case 'x':
-					if idx+3 >= size {
-						// TODO: need to return error
-						//err = xerrors.New("invalid escape character \\x")
-						return
-					}
-					codeNum := hexRunesToInt(src[idx+2 : idx+4])
-					value = append(value, rune(codeNum))
-					idx += 3
-					continue
-				case 'u':
-					if idx+5 >= size {
-						// TODO: need to return error
-						//err = xerrors.New("invalid escape character \\u")
-						return
-					}
-					codeNum := hexRunesToInt(src[idx+2 : idx+6])
-					value = append(value, rune(codeNum))
-					idx += 5
-					continue
-				case 'U':
-					if idx+9 >= size {
-						// TODO: need to return error
-						//err = xerrors.New("invalid escape character \\U")
-						return
-					}
-					codeNum := hexRunesToInt(src[idx+2 : idx+10])
-					value = append(value, rune(codeNum))
-					idx += 9
-					continue
-				case '\\':
-					ctx.addOriginBuf(nextChar)
-					idx++
+			if idx+1 >= size {
+				value = append(value, c)
+				continue
+			}
+			nextChar := src[idx+1]
+			progress := 0
+			switch nextChar {
+			case 'b':
+				progress = 1
+				ctx.addOriginBuf(nextChar)
+				value = append(value, '\b')
+			case 'e':
+				progress = 1
+				ctx.addOriginBuf(nextChar)
+				value = append(value, '\x1B')
+			case 'f':
+				progress = 1
+				ctx.addOriginBuf(nextChar)
+				value = append(value, '\f')
+			case 'n':
+				progress = 1
+				ctx.addOriginBuf(nextChar)
+				value = append(value, '\n')
+			case 'r':
+				progress = 1
+				ctx.addOriginBuf(nextChar)
+				value = append(value, '\r')
+			case 'v':
+				progress = 1
+				ctx.addOriginBuf(nextChar)
+				value = append(value, '\v')
+			case 'L': // LS (#x2028)
+				progress = 1
+				ctx.addOriginBuf(nextChar)
+				value = append(value, []rune{'\xE2', '\x80', '\xA8'}...)
+			case 'N': // NEL (#x85)
+				progress = 1
+				ctx.addOriginBuf(nextChar)
+				value = append(value, []rune{'\xC2', '\x85'}...)
+			case 'P': // PS (#x2029)
+				progress = 1
+				ctx.addOriginBuf(nextChar)
+				value = append(value, []rune{'\xE2', '\x80', '\xA9'}...)
+			case '_': // #xA0
+				progress = 1
+				ctx.addOriginBuf(nextChar)
+				value = append(value, []rune{'\xC2', '\xA0'}...)
+			case '"':
+				progress = 1
+				ctx.addOriginBuf(nextChar)
+				value = append(value, nextChar)
+			case 'x':
+				progress = 3
+				if idx+progress >= size {
+					// TODO: need to return error
+					//err = xerrors.New("invalid escape character \\x")
+					return
 				}
+				codeNum := hexRunesToInt(src[idx+2 : idx+progress+1])
+				value = append(value, rune(codeNum))
+			case 'u':
+				progress = 5
+				if idx+progress >= size {
+					// TODO: need to return error
+					//err = xerrors.New("invalid escape character \\u")
+					return
+				}
+				codeNum := hexRunesToInt(src[idx+2 : idx+progress+1])
+				value = append(value, rune(codeNum))
+			case 'U':
+				progress = 9
+				if idx+progress >= size {
+					// TODO: need to return error
+					//err = xerrors.New("invalid escape character \\U")
+					return
+				}
+				codeNum := hexRunesToInt(src[idx+2 : idx+progress+1])
+				value = append(value, rune(codeNum))
+			case '\\':
+				progress = 1
+				ctx.addOriginBuf(nextChar)
+				value = append(value, c)
+			default:
+				value = append(value, c)
 			}
-			value = append(value, c)
+			idx += progress
+			s.progressColumn(ctx, progress)
 			continue
 		} else if c != '"' {
 			value = append(value, c)
@@ -621,6 +615,16 @@ func (s *Scanner) scanNewLine(ctx *Context, c rune) {
 		}
 	}
 
+	// There is no problem that we ignore CR which followed by LF and normalize it to LF, because of following YAML1.2 spec.
+	// > Line breaks inside scalar content must be normalized by the YAML processor. Each such line break must be parsed into a single line feed character.
+	// > Outside scalar content, YAML allows any line break to be used to terminate lines.
+	// > -- https://yaml.org/spec/1.2/spec.html
+	if c == '\r' && ctx.nextChar() == '\n' {
+		ctx.addOriginBuf('\r')
+		ctx.progress(1)
+		c = '\n'
+	}
+
 	if ctx.isEOS() {
 		s.addBufferedTokenIfExists(ctx)
 	} else if s.isAnchor {
@@ -840,15 +844,6 @@ func (s *Scanner) scan(ctx *Context) (pos int) {
 				return
 			}
 		case '\r', '\n':
-			// There is no problem that we ignore CR which followed by LF and normalize it to LF, because of following YAML1.2 spec.
-			// > Line breaks inside scalar content must be normalized by the YAML processor. Each such line break must be parsed into a single line feed character.
-			// > Outside scalar content, YAML allows any line break to be used to terminate lines.
-			// > -- https://yaml.org/spec/1.2/spec.html
-			if c == '\r' && ctx.nextChar() == '\n' {
-				ctx.addOriginBuf('\r')
-				ctx.progress(1)
-				c = '\n'
-			}
 			s.scanNewLine(ctx, c)
 			continue
 		case ' ':
diff --git a/token/token.go b/token/token.go
index c86caab..14d7622 100644
--- a/token/token.go
+++ b/token/token.go
@@ -623,7 +623,7 @@ func IsNeedQuoted(value string) bool {
 	}
 	first := value[0]
 	switch first {
-	case '*', '&', '[', '{', '}', ']', ',', '!', '|', '>', '%', '\'', '"', '@', ' ':
+	case '*', '&', '[', '{', '}', ']', ',', '!', '|', '>', '%', '\'', '"', '@', ' ', '`':
 		return true
 	}
 	last := value[len(value)-1]


Test patch:
diff --git a/decode_test.go b/decode_test.go
index cabfd33..f0b0c08 100644
--- a/decode_test.go
+++ b/decode_test.go
@@ -14,11 +14,12 @@ import (
 	"testing"
 	"time"
 
+	"golang.org/x/xerrors"
+
 	"github.com/goccy/go-yaml"
 	"github.com/goccy/go-yaml/ast"
 	"github.com/goccy/go-yaml/internal/errors"
 	"github.com/goccy/go-yaml/parser"
-	"golang.org/x/xerrors"
 )
 
 type Child struct {
@@ -252,6 +253,10 @@ func TestDecoder(t *testing.T) {
 			"v: 4294967295",
 			map[string]uint{"v": math.MaxUint32},
 		},
+		{
+			"v: 1e3",
+			map[string]uint{"v": 1000},
+		},
 
 		// uint64
 		{
@@ -270,6 +275,10 @@ func TestDecoder(t *testing.T) {
 			"v: 9223372036854775807",
 			map[string]uint64{"v": math.MaxInt64},
 		},
+		{
+			"v: 1e3",
+			map[string]uint64{"v": 1000},
+		},
 
 		// float32
 		{
@@ -288,6 +297,10 @@ func TestDecoder(t *testing.T) {
 			"v: 18446744073709551616",
 			map[string]float32{"v": float32(math.MaxUint64 + 1)},
 		},
+		{
+			"v: 1e-06",
+			map[string]float32{"v": 1e-6},
+		},
 
 		// float64
 		{
@@ -306,6 +319,10 @@ func TestDecoder(t *testing.T) {
 			"v: 18446744073709551616",
 			map[string]float64{"v": float64(math.MaxUint64 + 1)},
 		},
+		{
+			"v: 1e-06",
+			map[string]float64{"v": 1e-06},
+		},
 
 		// Timestamps
 		{
@@ -1092,6 +1109,73 @@ c:
 	}
 }
 
+func TestDecoder_ScientificNotation(t *testing.T) {
+	tests := []struct {
+		source string
+		value  interface{}
+	}{
+		{
+			"v: 1e3",
+			map[string]uint{"v": 1000},
+		},
+		{
+			"v: 1e-3",
+			map[string]uint{"v": 0},
+		},
+		{
+			"v: 1e3",
+			map[string]int{"v": 1000},
+		},
+		{
+			"v: 1e-3",
+			map[string]int{"v": 0},
+		},
+		{
+			"v: 1e3",
+			map[string]float32{"v": 1000},
+		},
+		{
+			"v: 1.0e3",
+			map[string]float64{"v": 1000},
+		},
+		{
+			"v: 1e-3",
+			map[string]float64{"v": 0.001},
+		},
+		{
+			"v: 1.0e-3",
+			map[string]float64{"v": 0.001},
+		},
+		{
+			"v: 1.0e+3",
+			map[string]float64{"v": 1000},
+		},
+		{
+			"v: 1.0e+3",
+			map[string]float64{"v": 1000},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.source, func(t *testing.T) {
+			buf := bytes.NewBufferString(test.source)
+			dec := yaml.NewDecoder(buf)
+			typ := reflect.ValueOf(test.value).Type()
+			value := reflect.New(typ)
+			if err := dec.Decode(value.Interface()); err != nil {
+				if err == io.EOF {
+					return
+				}
+				t.Fatalf("%s: %+v", test.source, err)
+			}
+			actual := fmt.Sprintf("%+v", value.Elem().Interface())
+			expect := fmt.Sprintf("%+v", test.value)
+			if actual != expect {
+				t.Fatalf("failed to test [%s], actual=[%s], expect=[%s]", test.source, actual, expect)
+			}
+		})
+	}
+}
+
 func TestDecoder_TypeConversionError(t *testing.T) {
 	t.Run("type conversion for struct", func(t *testing.T) {
 		type T struct {
@@ -1114,6 +1198,17 @@ func TestDecoder_TypeConversionError(t *testing.T) {
 				t.Fatalf("expected error message: %s to contain: %s", err.Error(), msg)
 			}
 		})
+		t.Run("string to uint", func(t *testing.T) {
+			var v T
+			err := yaml.Unmarshal([]byte(`b: str`), &v)
+			if err == nil {
+				t.Fatal("expected to error")
+			}
+			msg := "cannot unmarshal string into Go struct field T.B of type uint"
+			if !strings.Contains(err.Error(), msg) {
+				t.Fatalf("expected error message: %s to contain: %s", err.Error(), msg)
+			}
+		})
 		t.Run("string to bool", func(t *testing.T) {
 			var v T
 			err := yaml.Unmarshal([]byte(`d: str`), &v)
@@ -2906,3 +3001,29 @@ func TestSameNameInineStruct(t *testing.T) {
 		t.Fatalf("failed to decode")
 	}
 }
+
+type unmarshableMapKey struct {
+	Key string
+}
+
+func (mk *unmarshableMapKey) UnmarshalYAML(b []byte) error {
+	mk.Key = string(b)
+	return nil
+}
+
+func TestMapKeyCustomUnmarshaler(t *testing.T) {
+	var m map[unmarshableMapKey]string
+	if err := yaml.Unmarshal([]byte(`key: value`), &m); err != nil {
+		t.Fatalf("failed to unmarshal %v", err)
+	}
+	if len(m) != 1 {
+		t.Fatalf("expected 1 element in map, but got %d", len(m))
+	}
+	val, ok := m[unmarshableMapKey{Key: "key"}]
+	if !ok {
+		t.Fatal("expected to have element 'key' in map")
+	}
+	if val != "value" {
+		t.Fatalf("expected to have value \"value\", but got %q", val)
+	}
+}
diff --git a/encode_test.go b/encode_test.go
index 3ff6f1c..74388b9 100644
--- a/encode_test.go
+++ b/encode_test.go
@@ -80,6 +80,16 @@ func TestEncoder(t *testing.T) {
 			map[string]float32{"v": 0.99},
 			nil,
 		},
+		{
+			"v: 1e-06\n",
+			map[string]float32{"v": 1e-06},
+			nil,
+		},
+		{
+			"v: 1e-06\n",
+			map[string]float64{"v": 0.000001},
+			nil,
+		},
 		{
 			"v: 0.123456789\n",
 			map[string]float64{"v": 0.123456789},
@@ -100,6 +110,16 @@ func TestEncoder(t *testing.T) {
 			map[string]float64{"v": 1000000},
 			nil,
 		},
+		{
+			"v: 1e-06\n",
+			map[string]float64{"v": 0.000001},
+			nil,
+		},
+		{
+			"v: 1e-06\n",
+			map[string]float64{"v": 1e-06},
+			nil,
+		},
 		{
 			"v: .inf\n",
 			map[string]interface{}{"v": math.Inf(0)},
@@ -318,6 +338,11 @@ func TestEncoder(t *testing.T) {
 			map[string]string{"a": " b "},
 			nil,
 		},
+		{
+			"a: \"`b` c\"\n",
+			map[string]string{"a": "`b` c"},
+			nil,
+		},
 		{
 			"a: 100.5\n",
 			map[string]interface{}{
diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go
index 0d8f648..7660ba1 100644
--- a/lexer/lexer_test.go
+++ b/lexer/lexer_test.go
@@ -2,7 +2,6 @@ package lexer_test
 
 import (
 	"sort"
-	"strings"
 	"testing"
 
 	"github.com/goccy/go-yaml/lexer"
@@ -10,59 +9,1834 @@ import (
 )
 
 func TestTokenize(t *testing.T) {
-	sources := []string{
-		"null\n",
-		"{}\n",
-		"v: hi\n",
-		"v: \"true\"\n",
-		"v: \"false\"\n",
-		"v: true\n",
-		"v: false\n",
-		"v: 10\n",
-		"v: -10\n",
-		"v: 42\n",
-		"v: 4294967296\n",
-		"v: \"10\"\n",
-		"v: 0.1\n",
-		"v: 0.99\n",
-		"v: -0.1\n",
-		"v: .inf\n",
-		"v: -.inf\n",
-		"v: .nan\n",
-		"v: null\n",
-		"v: \"\"\n",
-		"v:\n- A\n- B\n",
-		"v:\n- A\n- |-\n  B\n  C\n",
-		"v:\n- A\n- 1\n- B:\n  - 2\n  - 3\n",
-		"a:\n  b: c\n",
-		"a: '-'\n",
-		"123\n",
-		"hello: world\n",
-		"a: null\n",
-		"a: {x: 1}\n",
-		"a: [1, 2]\n",
-		"t2: 2018-01-09T10:40:47Z\nt4: 2098-01-09T10:40:47Z\n",
-		"a: {b: c, d: e}\n",
-		"a: 3s\n",
-		"a: <foo>\n",
-		"a: \"1:1\"\n",
-		"a: \"\\0\"\n",
-		"a: !!binary gIGC\n",
-		"a: !!binary |\n  " + strings.Repeat("kJCQ", 17) + "kJ\n  CQ\n",
-		"b: 2\na: 1\nd: 4\nc: 3\nsub:\n  e: 5\n",
-		"a: 1.2.3.4\n",
-		"a: \"2015-02-24T18:19:39Z\"\n",
-		"a: 'b: c'\n",
-		"a: 'Hello #comment'\n",
-		"a: 100.5\n",
-		"a: bogus\n",
-		"\"a\": double quoted map key",
-		"'a': single quoted map key",
-		"a: \"double quoted\"\nb: \"value map\"",
-		"a: 'single quoted'\nb: 'value map'",
+	tests := []struct {
+		YAML   string
+		Tokens token.Tokens
+	}{
+		{
+			YAML: `null
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.NullType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "null",
+					Origin:        "null\n",
+				},
+			},
+		},
+		{
+			YAML: `{}
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.MappingStartType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.FlowCollectionIndicator,
+					Value:         "{",
+					Origin:        "{",
+				},
+				{
+					Type:          token.MappingEndType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.FlowCollectionIndicator,
+					Value:         "}",
+					Origin:        "}",
+				},
+			},
+		},
+		{
+			YAML: `v: hi
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "v",
+					Origin:        "v",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "hi",
+					Origin:        " hi\n",
+				},
+			},
+		},
+		{
+			YAML: `v: "true"
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "v",
+					Origin:        "v",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.DoubleQuoteType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.QuotedScalarIndicator,
+					Value:         "true",
+					Origin:        " \"true\"",
+				},
+			},
+		},
+		{
+			YAML: `v: "false"
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "v",
+					Origin:        "v",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.DoubleQuoteType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.QuotedScalarIndicator,
+					Value:         "false",
+					Origin:        " \"false\"",
+				},
+			},
+		},
+		{
+			YAML: `v: true
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "v",
+					Origin:        "v",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.BoolType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "true",
+					Origin:        " true\n",
+				},
+			},
+		},
+		{
+			YAML: `v: false
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "v",
+					Origin:        "v",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.BoolType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "false",
+					Origin:        " false\n",
+				},
+			},
+		},
+		{
+			YAML: `v: 10
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "v",
+					Origin:        "v",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.IntegerType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "10",
+					Origin:        " 10\n",
+				},
+			},
+		},
+		{
+			YAML: `v: -10
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "v",
+					Origin:        "v",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.IntegerType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "-10",
+					Origin:        " -10\n",
+				},
+			},
+		},
+		{
+			YAML: `v: 42
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "v",
+					Origin:        "v",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.IntegerType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "42",
+					Origin:        " 42\n",
+				},
+			},
+		},
+		{
+			YAML: `v: 4294967296
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "v",
+					Origin:        "v",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.IntegerType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "4294967296",
+					Origin:        " 4294967296\n",
+				},
+			},
+		},
+		{
+			YAML: `v: "10"
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "v",
+					Origin:        "v",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.DoubleQuoteType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.QuotedScalarIndicator,
+					Value:         "10",
+					Origin:        " \"10\"",
+				},
+			},
+		},
+		{
+			YAML: `v: 0.1
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "v",
+					Origin:        "v",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.FloatType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "0.1",
+					Origin:        " 0.1\n",
+				},
+			},
+		},
+		{
+			YAML: `v: 0.99
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "v",
+					Origin:        "v",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.FloatType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "0.99",
+					Origin:        " 0.99\n",
+				},
+			},
+		},
+		{
+			YAML: `v: -0.1
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "v",
+					Origin:        "v",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.FloatType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "-0.1",
+					Origin:        " -0.1\n",
+				},
+			},
+		},
+		{
+			YAML: `v: .inf
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "v",
+					Origin:        "v",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.InfinityType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         ".inf",
+					Origin:        " .inf\n",
+				},
+			},
+		},
+		{
+			YAML: `v: -.inf
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "v",
+					Origin:        "v",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.InfinityType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "-.inf",
+					Origin:        " -.inf\n",
+				},
+			},
+		},
+		{
+			YAML: `v: .nan
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "v",
+					Origin:        "v",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.NanType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         ".nan",
+					Origin:        " .nan\n",
+				},
+			},
+		},
+		{
+			YAML: `v: null
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "v",
+					Origin:        "v",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.NullType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "null",
+					Origin:        " null\n",
+				},
+			},
+		},
+		{
+			YAML: `v: ""
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "v",
+					Origin:        "v",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.DoubleQuoteType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.QuotedScalarIndicator,
+					Value:         "",
+					Origin:        " \"\"",
+				},
+			},
+		},
+		{
+			YAML: `v:
+		- A
+		- B
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "v",
+					Origin:        "v",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.SequenceEntryType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         "-",
+					Origin:        "\n\t\t-",
+				},
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "A",
+					Origin:        " A\n",
+				},
+				{
+					Type:          token.SequenceEntryType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         "-",
+					Origin:        "\t\t-",
+				},
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "B",
+					Origin:        " B\n",
+				},
+			},
+		},
+		{
+			YAML: `v:
+		- A
+		- |-
+		 B
+		 C
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "v",
+					Origin:        "v",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.SequenceEntryType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         "-",
+					Origin:        "\n\t\t-",
+				},
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "A",
+					Origin:        " A\n",
+				},
+				{
+					Type:          token.SequenceEntryType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         "-",
+					Origin:        "\t\t-",
+				},
+				{
+					Type:          token.LiteralType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockScalarIndicator,
+					Value:         "|-",
+					Origin:        " |-\n",
+				},
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "\t\t B",
+					Origin:        "\t\t B\n",
+				},
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "\t\t C",
+					Origin:        "\t\t C\n",
+				},
+			},
+		},
+		{
+			YAML: `v:
+		- A
+		- 1
+		- B:
+		 - 2
+		 - 3
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "v",
+					Origin:        "v",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.SequenceEntryType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         "-",
+					Origin:        "\n\t\t-",
+				},
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "A",
+					Origin:        " A\n",
+				},
+				{
+					Type:          token.SequenceEntryType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         "-",
+					Origin:        "\t\t-",
+				},
+				{
+					Type:          token.IntegerType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "1",
+					Origin:        " 1\n",
+				},
+				{
+					Type:          token.SequenceEntryType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         "-",
+					Origin:        "\t\t-",
+				},
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "B",
+					Origin:        " B",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.SequenceEntryType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         "-",
+					Origin:        "\n\t\t -",
+				},
+				{
+					Type:          token.IntegerType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "2",
+					Origin:        " 2\n",
+				},
+				{
+					Type:          token.SequenceEntryType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         "-",
+					Origin:        "\t\t -",
+				},
+				{
+					Type:          token.IntegerType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "3",
+					Origin:        " 3\n",
+				},
+			},
+		},
+		{
+			YAML: `a:
+		 b: c
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "a",
+					Origin:        "a",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "\t\t b",
+					Origin:        "\n\t\t b",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "c",
+					Origin:        " c\n",
+				},
+			},
+		},
+		{
+			YAML: `a: '-'
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "a",
+					Origin:        "a",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.SingleQuoteType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.QuotedScalarIndicator,
+					Value:         "-",
+					Origin:        " '-'",
+				},
+			},
+		},
+		{
+			YAML: `123
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.IntegerType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "123",
+					Origin:        "123\n",
+				},
+			},
+		},
+		{
+			YAML: `hello: world
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "hello",
+					Origin:        "hello",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "world",
+					Origin:        " world\n",
+				},
+			},
+		},
+		{
+			YAML: `a: null
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "a",
+					Origin:        "a",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.NullType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "null",
+					Origin:        " null\n",
+				},
+			},
+		},
+		{
+			YAML: `a: {x: 1}
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "a",
+					Origin:        "a",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.MappingStartType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.FlowCollectionIndicator,
+					Value:         "{",
+					Origin:        " {",
+				},
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "x",
+					Origin:        "x",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.IntegerType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "1",
+					Origin:        " 1",
+				},
+				{
+					Type:          token.MappingEndType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.FlowCollectionIndicator,
+					Value:         "}",
+					Origin:        "}",
+				},
+			},
+		},
+		{
+			YAML: `a: [1, 2]
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "a",
+					Origin:        "a",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.SequenceStartType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.FlowCollectionIndicator,
+					Value:         "[",
+					Origin:        " [",
+				},
+				{
+					Type:          token.IntegerType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "1",
+					Origin:        "1",
+				},
+				{
+					Type:          token.CollectEntryType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.FlowCollectionIndicator,
+					Value:         ",",
+					Origin:        ",",
+				},
+				{
+					Type:          token.IntegerType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "2",
+					Origin:        " 2",
+				},
+				{
+					Type:          token.SequenceEndType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.FlowCollectionIndicator,
+					Value:         "]",
+					Origin:        "]",
+				},
+			},
+		},
+		{
+			YAML: `t2: 2018-01-09T10:40:47Z
+		t4: 2098-01-09T10:40:47Z
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "t2",
+					Origin:        "t2",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "2018-01-09T10:40:47Z",
+					Origin:        " 2018-01-09T10:40:47Z\n",
+				},
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "\t\tt4",
+					Origin:        "\t\tt4",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "2098-01-09T10:40:47Z",
+					Origin:        " 2098-01-09T10:40:47Z\n",
+				},
+			},
+		},
+		{
+			YAML: `a: {b: c, d: e}
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "a",
+					Origin:        "a",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.MappingStartType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.FlowCollectionIndicator,
+					Value:         "{",
+					Origin:        " {",
+				},
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "b",
+					Origin:        "b",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "c",
+					Origin:        " c",
+				},
+				{
+					Type:          token.CollectEntryType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.FlowCollectionIndicator,
+					Value:         ",",
+					Origin:        ",",
+				},
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "d",
+					Origin:        " d",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "e",
+					Origin:        " e",
+				},
+				{
+					Type:          token.MappingEndType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.FlowCollectionIndicator,
+					Value:         "}",
+					Origin:        "}",
+				},
+			},
+		},
+		{
+			YAML: `a: 3s
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "a",
+					Origin:        "a",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "3s",
+					Origin:        " 3s\n",
+				},
+			},
+		},
+		{
+			YAML: `a: <foo>
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "a",
+					Origin:        "a",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "<foo>",
+					Origin:        " <foo>\n",
+				},
+			},
+		},
+		{
+			YAML: `a: "1:1"
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "a",
+					Origin:        "a",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.DoubleQuoteType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.QuotedScalarIndicator,
+					Value:         "1:1",
+					Origin:        " \"1:1\"",
+				},
+			},
+		},
+		{
+			YAML: `a: "\0"
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "a",
+					Origin:        "a",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.DoubleQuoteType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.QuotedScalarIndicator,
+					Value:         "\\0",
+					Origin:        " \"\\0\"",
+				},
+			},
+		},
+		{
+			YAML: `a: !!binary gIGC
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "a",
+					Origin:        "a",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.TagType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.NodePropertyIndicator,
+					Value:         "!!binary",
+					Origin:        " !!binary ",
+				},
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "gIGC",
+					Origin:        "gIGC\n",
+				},
+			},
+		},
+		{
+			YAML: `a: !!binary |
+		 kJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJ
+		 CQ
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "a",
+					Origin:        "a",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.TagType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.NodePropertyIndicator,
+					Value:         "!!binary",
+					Origin:        " !!binary ",
+				},
+				{
+					Type:          token.LiteralType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockScalarIndicator,
+					Value:         "|",
+					Origin:        "|\n",
+				},
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "\t\t kJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJ",
+					Origin:        "\t\t kJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJ\n",
+				},
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "\t\t CQ",
+					Origin:        "\t\t CQ\n",
+				},
+			},
+		},
+		{
+			YAML: `b: 2
+		a: 1
+		d: 4
+		c: 3
+		sub:
+		 e: 5
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "b",
+					Origin:        "b",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.IntegerType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "2",
+					Origin:        " 2\n",
+				},
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "\t\ta",
+					Origin:        "\t\ta",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.IntegerType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "1",
+					Origin:        " 1\n",
+				},
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "\t\td",
+					Origin:        "\t\td",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.IntegerType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "4",
+					Origin:        " 4\n",
+				},
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "\t\tc",
+					Origin:        "\t\tc",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.IntegerType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "3",
+					Origin:        " 3\n",
+				},
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "\t\tsub",
+					Origin:        "\t\tsub",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "\t\t e",
+					Origin:        "\n\t\t e",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.IntegerType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "5",
+					Origin:        " 5\n",
+				},
+			},
+		},
+		{
+			YAML: `a: 1.2.3.4
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "a",
+					Origin:        "a",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "1.2.3.4",
+					Origin:        " 1.2.3.4\n",
+				},
+			},
+		},
+		{
+			YAML: `a: "2015-02-24T18:19:39Z"
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "a",
+					Origin:        "a",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.DoubleQuoteType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.QuotedScalarIndicator,
+					Value:         "2015-02-24T18:19:39Z",
+					Origin:        " \"2015-02-24T18:19:39Z\"",
+				},
+			},
+		},
+		{
+			YAML: `a: 'b: c'
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "a",
+					Origin:        "a",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.SingleQuoteType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.QuotedScalarIndicator,
+					Value:         "b: c",
+					Origin:        " 'b: c'",
+				},
+			},
+		},
+		{
+			YAML: `a: 'Hello #comment'
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "a",
+					Origin:        "a",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.SingleQuoteType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.QuotedScalarIndicator,
+					Value:         "Hello #comment",
+					Origin:        " 'Hello #comment'",
+				},
+			},
+		},
+		{
+			YAML: `a: 100.5
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "a",
+					Origin:        "a",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.FloatType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "100.5",
+					Origin:        " 100.5\n",
+				},
+			},
+		},
+		{
+			YAML: `a: bogus
+		`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "a",
+					Origin:        "a",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "bogus",
+					Origin:        " bogus\n",
+				},
+			},
+		},
+		{
+			YAML: `"a": double quoted map key`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.DoubleQuoteType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.QuotedScalarIndicator,
+					Value:         "a",
+					Origin:        "\"a\"",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "double quoted map key",
+					Origin:        " double quoted map key",
+				},
+			},
+		},
+		{
+			YAML: `'a': single quoted map key`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.SingleQuoteType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.QuotedScalarIndicator,
+					Value:         "a",
+					Origin:        "'a'",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "single quoted map key",
+					Origin:        " single quoted map key",
+				},
+			},
+		},
+		{
+			YAML: `a: "double quoted"
+		b: "value map"`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "a",
+					Origin:        "a",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.DoubleQuoteType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.QuotedScalarIndicator,
+					Value:         "double quoted",
+					Origin:        " \"double quoted\"",
+				},
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "\t\tb",
+					Origin:        "\n\t\tb",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.DoubleQuoteType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.QuotedScalarIndicator,
+					Value:         "value map",
+					Origin:        " \"value map\"",
+				},
+			},
+		},
+		{
+			YAML: `a: 'single quoted'
+		b: 'value map'`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "a",
+					Origin:        "a",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.SingleQuoteType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.QuotedScalarIndicator,
+					Value:         "single quoted",
+					Origin:        " 'single quoted'",
+				},
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "\t\tb",
+					Origin:        "\n\t\tb",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.SingleQuoteType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.QuotedScalarIndicator,
+					Value:         "value map",
+					Origin:        " 'value map'",
+				},
+			},
+		},
+		{
+			YAML: `json: '\"expression\": \"thi:\"'`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "json",
+					Origin:        "json",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.SingleQuoteType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.QuotedScalarIndicator,
+					Value:         "\\\"expression\\\": \\\"thi:\\\"",
+					Origin:        " '\\\"expression\\\": \\\"thi:\\\"'",
+				},
+			},
+		},
+		{
+			YAML: `json: "\"expression\": \"thi:\""`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "json",
+					Origin:        "json",
+				},
+				{
+					Type:          token.MappingValueType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.BlockStructureIndicator,
+					Value:         ":",
+					Origin:        ":",
+				},
+				{
+					Type:          token.DoubleQuoteType,
+					CharacterType: token.CharacterTypeIndicator,
+					Indicator:     token.QuotedScalarIndicator,
+					Value:         "\"expression\": \"thi:\"",
+					Origin:        " \"\\\"expression\\\": \\\"thi:\\\"\"",
+				},
+			},
+		},
 	}
-	for _, src := range sources {
-		lexer.Tokenize(src).Dump()
+	for _, test := range tests {
+		t.Run(test.YAML, func(t *testing.T) {
+			tokens := lexer.Tokenize(test.YAML)
+			if len(tokens) != len(test.Tokens) {
+				t.Fatalf("Tokenize(%q) token count mismatch, expected: %d got: %d", test.YAML, len(test.Tokens), len(tokens))
+			}
+			for i := range test.Tokens {
+				if tokens[i].Type != test.Tokens[i].Type {
+					t.Errorf("Tokenize(%q)[%d] token.Type mismatch, expected: %s got: %s", test.YAML, i, test.Tokens[i].Type, tokens[i].Type)
+				}
+				if tokens[i].CharacterType != test.Tokens[i].CharacterType {
+					t.Errorf("Tokenize(%q)[%d] token.CharacterType mismatch, expected: %s got: %s", test.YAML, i, test.Tokens[i].CharacterType, tokens[i].CharacterType)
+				}
+				if tokens[i].Indicator != test.Tokens[i].Indicator {
+					t.Errorf("Tokenize(%q)[%d] token.Indicator mismatch, expected: %s got: %s", test.YAML, i, test.Tokens[i].Indicator, tokens[i].Indicator)
+				}
+				if tokens[i].Value != test.Tokens[i].Value {
+					t.Errorf("Tokenize(%q)[%d] token.Value mismatch, expected: %q got: %q", test.YAML, i, test.Tokens[i].Value, tokens[i].Value)
+				}
+				if tokens[i].Origin != test.Tokens[i].Origin {
+					t.Errorf("Tokenize(%q)[%d] token.Origin mismatch, expected: %q got: %q", test.YAML, i, test.Tokens[i].Origin, tokens[i].Origin)
+				}
+			}
+		})
 	}
 }
 
diff --git a/parser/parser_test.go b/parser/parser_test.go
index 8d697e8..3dc9440 100644
--- a/parser/parser_test.go
+++ b/parser/parser_test.go
@@ -83,6 +83,9 @@ func TestParser(t *testing.T) {
 }`,
 		"\"a\": a\n\"b\": b",
 		"'a': a\n'b': b",
+		"a: \r\n  b: 1\r\n",
+		"a_ok: \r  bc: 2\r",
+		"a_mk: \n  bd: 3\n",
 	}
 	for _, src := range sources {
 		if _, err := parser.Parse(lexer.Tokenize(src), 0); err != nil {
diff --git a/path_test.go b/path_test.go
index c0073ce..8a4d6b5 100644
--- a/path_test.go
+++ b/path_test.go
@@ -104,6 +104,31 @@ store:
 			path:     builder().Root().Child("store").Child(`bicycle*unicycle`).Child("price").Build(),
 			expected: float64(20.25),
 		},
+		{
+			name: "$",
+			path: builder().Root().Build(),
+			expected: map[string]interface{}{
+				"store": map[string]interface{}{
+					"book": []interface{}{
+						map[string]interface{}{
+							"author": "john",
+							"price":  uint64(10),
+						},
+						map[string]interface{}{
+							"author": "ken",
+							"price":  uint64(12),
+						},
+					},
+					"bicycle": map[string]interface{}{
+						"color": "red",
+						"price": 19.95,
+					},
+					"bicycle*unicycle": map[string]interface{}{
+						"price": 20.25,
+					},
+				},
+			},
+		},
 	}
 	t.Run("PathString", func(t *testing.T) {
 		for _, test := range tests {