Repository: goccy/go-yaml (go)

Issue:
error on parsing block string
environment: go 1.13.4

(日本語でいいでしょうか・・・)

次の様な複数行文字列をパースしている際に文字列中でエラーが発生しました。

``` yaml
foo:
  value: |
     {
      "versions": [
            {
              "status": "CURRENT",
              "updated": "2011-01-21T11:33:21Z",
              "id": "v2.0",
              "links": [
                  {
                      "href": "http://127.0.0.1:8774/v2/",
                      "rel": "self"
                  }
              ]
          },
          {
              "status": "EXPERIMENTAL",
              "updated": "2013-07-23T11:33:21Z",
              "id": "v3.0",
              "links": [
                  {
                      "href": "http://127.0.0.1:8774/v3/",
                      "rel": "self"
                  }
              ]
          }
      ]
     }
```

`.foo.value`以下はすべて文字列となる、という想定で、gopkg.in/yaml.v2では問題なく動作していたものです。
エラーメッセージは次の様なものです。

```
--- FAIL: TestAPIWithExample (0.00s)
    unmarshalyaml_test.go:25: [66:51] failed to parse flow mapping value node
              62 | }
              63 |                             ]
              64 |                         },
              65 |                         {
            > 66 |                             "status": "EXPERIMENTAL",
              67 |                             "updated": "2013-07-23T11:33:21Z",
                                                                    ^
              68 |                             "id": "v3.0",
              69 |                             "links": [
              70 |                                 {
FAIL
```

当該部分は文字列なので、(インデントが足りないとかを除いて)途中でエラーが発生するというのは挙動として正しくないように思えます。

再現用Playground: https://play.golang.org/p/yLh0AZ5lQ62

Pull request:
Fix parsing of literal token
resolve #30



Patch:
diff --git a/ast/ast.go b/ast/ast.go
index 8831b3b..cf0e5db 100644
--- a/ast/ast.go
+++ b/ast/ast.go
@@ -424,7 +424,8 @@ func (n *LiteralNode) GetValue() interface{} {
 
 // String literal to text
 func (n *LiteralNode) String() string {
-	return n.Value.String()
+	origin := n.Value.GetToken().Origin
+	return fmt.Sprintf("|\n%s", strings.TrimRight(strings.TrimRight(origin, " "), "\n"))
 }
 
 // MergeKeyNode type of merge key node
diff --git a/scanner/context.go b/scanner/context.go
index 15a94e9..2292f9b 100644
--- a/scanner/context.go
+++ b/scanner/context.go
@@ -62,6 +62,10 @@ func (c *Context) addOriginBuf(r rune) {
 	c.obuf = append(c.obuf, r)
 }
 
+func (c *Context) isDocument() bool {
+	return c.isLiteral || c.isFolded || c.isRawFolded
+}
+
 func (c *Context) isEOS() bool {
 	return len(c.src)-1 <= c.idx
 }
diff --git a/scanner/scanner.go b/scanner/scanner.go
index cfab6f6..da73fce 100644
--- a/scanner/scanner.go
+++ b/scanner/scanner.go
@@ -34,6 +34,7 @@ type Scanner struct {
 	prevIndentLevel        int
 	prevIndentNum          int
 	prevIndentColumn       int
+	docStartColumn         int
 	indentLevel            int
 	indentNum              int
 	isFirstCharAtLine      bool
@@ -87,7 +88,20 @@ func (s *Scanner) progressLine(ctx *Context) {
 	ctx.progress(1)
 }
 
-func (s *Scanner) updateIndent(c rune) {
+func (s *Scanner) isNeededKeepPreviousIndentNum(ctx *Context, c rune) bool {
+	if !s.isChangedToIndentStateUp() {
+		return false
+	}
+	if ctx.isDocument() {
+		return true
+	}
+	if c == '-' && ctx.bufferedSrc() != "" {
+		return true
+	}
+	return false
+}
+
+func (s *Scanner) updateIndent(ctx *Context, c rune) {
 	if s.isFirstCharAtLine && c == ' ' {
 		s.indentNum++
 		return
@@ -119,10 +133,13 @@ func (s *Scanner) updateIndent(c rune) {
 			s.indentState = IndentStateDown
 		}
 	}
+	s.isFirstCharAtLine = false
+	if s.isNeededKeepPreviousIndentNum(ctx, c) {
+		return
+	}
 	s.prevIndentNum = s.indentNum
 	s.prevIndentColumn = 0
 	s.prevIndentLevel = s.indentLevel
-	s.isFirstCharAtLine = false
 }
 
 func (s *Scanner) isChangedToIndentStateDown() bool {
@@ -142,6 +159,7 @@ func (s *Scanner) addBufferedTokenIfExists(ctx *Context) {
 }
 
 func (s *Scanner) breakLiteral(ctx *Context) {
+	s.docStartColumn = 0
 	ctx.breakLiteral()
 }
 
@@ -212,6 +230,7 @@ func (s *Scanner) scanLiteral(ctx *Context, c rune) {
 	if ctx.isEOS() {
 		value := ctx.bufferedSrc()
 		ctx.addToken(token.New(value, string(ctx.obuf), s.pos()))
+		ctx.resetBuffer()
 	}
 	if c == '\n' {
 		if ctx.isLiteral {
@@ -221,8 +240,14 @@ func (s *Scanner) scanLiteral(ctx *Context, c rune) {
 		}
 		s.progressLine(ctx)
 	} else if s.isFirstCharAtLine && c == ' ' {
+		if 0 < s.docStartColumn && s.docStartColumn <= s.column {
+			ctx.addBuf(c)
+		}
 		s.progressColumn(ctx, 1)
 	} else {
+		if s.docStartColumn == 0 {
+			s.docStartColumn = s.column
+		}
 		ctx.addBuf(c)
 		s.progressColumn(ctx, 1)
 	}
@@ -280,13 +305,18 @@ func (s *Scanner) scan(ctx *Context) (pos int) {
 	for ctx.next() {
 		pos = ctx.nextPos()
 		c := ctx.currentChar()
-		s.updateIndent(c)
-		if s.isChangedToIndentStateDown() {
+		s.updateIndent(ctx, c)
+		if ctx.isDocument() {
+			if s.isChangedToIndentStateEqual() ||
+				s.isChangedToIndentStateDown() {
+				s.addBufferedTokenIfExists(ctx)
+				s.breakLiteral(ctx)
+			} else {
+				s.scanLiteral(ctx, c)
+				continue
+			}
+		} else if s.isChangedToIndentStateDown() {
 			s.addBufferedTokenIfExists(ctx)
-			s.breakLiteral(ctx)
-		} else if ctx.isLiteral || ctx.isFolded || ctx.isRawFolded {
-			s.scanLiteral(ctx, c)
-			continue
 		} else if s.isChangedToIndentStateEqual() {
 			// if first character is \n, buffer expect to raw folded literal
 			if len(ctx.obuf) > 0 && ctx.obuf[0] != '\n' {


Test patch:
diff --git a/parser/parser_test.go b/parser/parser_test.go
index f7026f4..49c09ac 100644
--- a/parser/parser_test.go
+++ b/parser/parser_test.go
@@ -375,6 +375,24 @@ b: 2
 c: 3
 d: 4
 ...
+`,
+		},
+		{
+			`
+a:
+  b: |
+    {
+      [ 1, 2 ]
+    }
+  c: d
+`,
+			`
+a:
+  b: |
+    {
+      [ 1, 2 ]
+    }
+  c: d
 `,
 		},
 	}