Repository: mikefarah/yq
Pull request #1001 (2021-11-13 23:52:53)

Fix header processing


Patch:
diff --git a/examples/data1.yaml b/examples/data1.yaml
index e940b64d..fa32f314 100644
--- a/examples/data1.yaml
+++ b/examples/data1.yaml
@@ -1,4 +1,8 @@
-- name: bob
-  age: 23
-- name: tim
-  age: 17
\ No newline at end of file
+# welcome! # cat
+# bob
+---
+# bob
+
+a: cat # meow
+
+# have a great day
\ No newline at end of file
diff --git a/examples/data2.yaml b/examples/data2.yaml
index 7eaac718..f4313254 100644
--- a/examples/data2.yaml
+++ b/examples/data2.yaml
@@ -1,3 +1,2 @@
-#b1 
-b: 2
-#b2
\ No newline at end of file
+c:
+  d: hamster
\ No newline at end of file
diff --git a/pkg/yqlib/all_at_once_evaluator.go b/pkg/yqlib/all_at_once_evaluator.go
index e691f242..9c79a035 100644
--- a/pkg/yqlib/all_at_once_evaluator.go
+++ b/pkg/yqlib/all_at_once_evaluator.go
@@ -71,14 +71,15 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string
 
 	if allDocuments.Len() == 0 {
 		candidateNode := &CandidateNode{
-			Document:  0,
-			Filename:  "",
-			Node:      &yaml.Node{Kind: yaml.DocumentNode, HeadComment: firstFileLeadingContent, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
-			FileIndex: 0,
+			Document:       0,
+			Filename:       "",
+			Node:           &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
+			FileIndex:      0,
+			LeadingContent: firstFileLeadingContent,
 		}
 		allDocuments.PushBack(candidateNode)
 	} else {
-		allDocuments.Front().Value.(*CandidateNode).Node.HeadComment = firstFileLeadingContent
+		allDocuments.Front().Value.(*CandidateNode).LeadingContent = firstFileLeadingContent
 	}
 
 	matches, err := e.EvaluateCandidateNodes(expression, allDocuments)
diff --git a/pkg/yqlib/candidate_node.go b/pkg/yqlib/candidate_node.go
index f1e092bb..f685f679 100644
--- a/pkg/yqlib/candidate_node.go
+++ b/pkg/yqlib/candidate_node.go
@@ -9,10 +9,13 @@ import (
 )
 
 type CandidateNode struct {
-	Node      *yaml.Node     // the actual node
-	Parent    *CandidateNode // parent node
-	Path      []interface{}  /// the path we took to get to this node
-	Document  uint           // the document index of this node
+	Node   *yaml.Node     // the actual node
+	Parent *CandidateNode // parent node
+
+	LeadingContent string
+
+	Path      []interface{} /// the path we took to get to this node
+	Document  uint          // the document index of this node
 	Filename  string
 	FileIndex int
 	// when performing op against all nodes given, this will treat all the nodes as one
diff --git a/pkg/yqlib/doc/comment-operators.md b/pkg/yqlib/doc/comment-operators.md
index 507a6a7f..6086b20a 100644
--- a/pkg/yqlib/doc/comment-operators.md
+++ b/pkg/yqlib/doc/comment-operators.md
@@ -139,6 +139,26 @@ will output
 welcome!
 ```
 
+## 
+Given a sample.yml file of:
+```yaml
+# welcome!
+---
+# bob
+a: cat # meow
+
+# have a great day
+```
+then
+```bash
+yq eval 'headComment' sample.yml
+```
+will output
+```yaml
+welcome!
+bob
+```
+
 ## Get foot comment
 Given a sample.yml file of:
 ```yaml
@@ -147,6 +167,7 @@ Given a sample.yml file of:
 a: cat # meow
 
 # have a great day
+# no really
 ```
 then
 ```bash
@@ -155,5 +176,6 @@ yq eval '. | footComment' sample.yml
 will output
 ```yaml
 have a great day
+no really
 ```
 
diff --git a/pkg/yqlib/operator_comments.go b/pkg/yqlib/operator_comments.go
index 6ece8f29..c58a84bb 100644
--- a/pkg/yqlib/operator_comments.go
+++ b/pkg/yqlib/operator_comments.go
@@ -1,8 +1,12 @@
 package yqlib
 
 import (
+	// "bufio"
+	// "bytes"
+	"bufio"
+	"bytes"
 	"container/list"
-	"strings"
+	"regexp"
 
 	yaml "gopkg.in/yaml.v3"
 )
@@ -57,6 +61,7 @@ func assignCommentsOperator(d *dataTreeNavigator, context Context, expressionNod
 		}
 		if preferences.HeadComment {
 			candidate.Node.HeadComment = comment
+			candidate.LeadingContent = "" // clobber the leading content, if there was any.
 		}
 		if preferences.FootComment {
 			candidate.Node.FootComment = comment
@@ -68,6 +73,9 @@ func assignCommentsOperator(d *dataTreeNavigator, context Context, expressionNod
 
 func getCommentsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	preferences := expressionNode.Operation.Preferences.(commentOpPreferences)
+	var startCommentCharaterRegExp = regexp.MustCompile(`^# `)
+	var subsequentCommentCharaterRegExp = regexp.MustCompile(`\n# `)
+
 	log.Debugf("GetComments operator!")
 	var results = list.New()
 
@@ -76,12 +84,25 @@ func getCommentsOperator(d *dataTreeNavigator, context Context, expressionNode *
 		comment := ""
 		if preferences.LineComment {
 			comment = candidate.Node.LineComment
+		} else if preferences.HeadComment && candidate.LeadingContent != "" {
+			var chompRegexp = regexp.MustCompile(`\n$`)
+			var output bytes.Buffer
+			var writer = bufio.NewWriter(&output)
+			if err := processLeadingContent(candidate, writer, false, YamlOutputFormat); err != nil {
+				return Context{}, err
+			}
+			if err := writer.Flush(); err != nil {
+				return Context{}, err
+			}
+			comment = output.String()
+			comment = chompRegexp.ReplaceAllString(comment, "")
 		} else if preferences.HeadComment {
 			comment = candidate.Node.HeadComment
 		} else if preferences.FootComment {
 			comment = candidate.Node.FootComment
 		}
-		comment = strings.Replace(comment, "# ", "", 1)
+		comment = startCommentCharaterRegExp.ReplaceAllString(comment, "")
+		comment = subsequentCommentCharaterRegExp.ReplaceAllString(comment, "\n")
 
 		node := &yaml.Node{Kind: yaml.ScalarNode, Value: comment, Tag: "!!str"}
 		result := candidate.CreateChild(nil, node)
diff --git a/pkg/yqlib/operator_multiply.go b/pkg/yqlib/operator_multiply.go
index 00265387..2746b568 100644
--- a/pkg/yqlib/operator_multiply.go
+++ b/pkg/yqlib/operator_multiply.go
@@ -21,29 +21,25 @@ func multiplyOperator(d *dataTreeNavigator, context Context, expressionNode *Exp
 	return crossFunction(d, context, expressionNode, multiply(expressionNode.Operation.Preferences.(multiplyPreferences)), false)
 }
 
-func getNewBlankNode(lhs *yaml.Node, rhs *yaml.Node) *yaml.Node {
-
-	blankNode := &yaml.Node{}
-
-	if lhs.HeadComment != "" {
-		blankNode.HeadComment = lhs.HeadComment
-	} else if rhs.HeadComment != "" {
-		blankNode.HeadComment = rhs.HeadComment
+func getComments(lhs *CandidateNode, rhs *CandidateNode) (leadingContent string, headComment string, footComment string) {
+	leadingContent = rhs.LeadingContent
+	headComment = rhs.Node.HeadComment
+	footComment = rhs.Node.FootComment
+	if lhs.Node.HeadComment != "" || lhs.LeadingContent != "" {
+		headComment = lhs.Node.HeadComment
+		leadingContent = lhs.LeadingContent
 	}
 
-	if lhs.FootComment != "" {
-		blankNode.FootComment = lhs.FootComment
-	} else if rhs.FootComment != "" {
-		blankNode.FootComment = rhs.FootComment
+	if lhs.Node.FootComment != "" {
+		footComment = lhs.Node.FootComment
 	}
-
-	return blankNode
+	return leadingContent, headComment, footComment
 }
 
 func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 	return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 		// need to do this before unWrapping the potential document node
-		newBlankNode := getNewBlankNode(lhs.Node, rhs.Node)
+		leadingContent, headComment, footComment := getComments(lhs, rhs)
 		lhs.Node = unwrapDoc(lhs.Node)
 		rhs.Node = unwrapDoc(rhs.Node)
 		log.Debugf("Multipling LHS: %v", lhs.Node.Tag)
@@ -56,15 +52,10 @@ func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, contex
 			if err != nil {
 				return nil, err
 			}
-			newBlank.Node.HeadComment = newBlankNode.HeadComment
-			newBlank.Node.FootComment = newBlankNode.FootComment
-
-			// var newBlank = lhs.CreateChild(nil, newBlankNode)
-			// log.Debugf("merge - merge lhs into blank")
-			// var newThing, err = mergeObjects(d, context.WritableClone(), newBlank, lhs, multiplyPreferences{})
-			// if err != nil {
-			// 	return nil, err
-			// }
+			newBlank.LeadingContent = leadingContent
+			newBlank.Node.HeadComment = headComment
+			newBlank.Node.FootComment = footComment
+
 			return mergeObjects(d, context.WritableClone(), &newBlank, rhs, preferences)
 		} else if lhs.Node.Tag == "!!int" && rhs.Node.Tag == "!!int" {
 			return multiplyIntegers(lhs, rhs)
diff --git a/pkg/yqlib/printer.go b/pkg/yqlib/printer.go
index 3ac4bcee..a264bd49 100644
--- a/pkg/yqlib/printer.go
+++ b/pkg/yqlib/printer.go
@@ -5,7 +5,7 @@ import (
 	"container/list"
 	"fmt"
 	"io"
-	"strings"
+	"regexp"
 
 	yaml "gopkg.in/yaml.v3"
 )
@@ -84,7 +84,7 @@ func (p *resultsPrinter) printNode(node *yaml.Node, writer io.Writer) error {
 
 	var encoder Encoder
 	if node.Kind == yaml.ScalarNode && p.unwrapScalar && p.outputFormat == YamlOutputFormat {
-		return p.writeString(writer, node.Value+"\n")
+		return writeString(writer, node.Value+"\n")
 	}
 
 	if p.outputFormat == JsonOutputFormat {
@@ -97,54 +97,6 @@ func (p *resultsPrinter) printNode(node *yaml.Node, writer io.Writer) error {
 	return encoder.Encode(node)
 }
 
-func (p *resultsPrinter) writeString(writer io.Writer, txt string) error {
-	_, errorWriting := writer.Write([]byte(txt))
-	return errorWriting
-}
-
-func (p *resultsPrinter) processLeadingContent(mappedDoc *CandidateNode, writer io.Writer) error {
-	if strings.Contains(mappedDoc.Node.HeadComment, "$yqLeadingContent$") {
-		log.Debug("headcommentwas %v", mappedDoc.Node.HeadComment)
-		log.Debug("finished headcomment")
-		reader := bufio.NewReader(strings.NewReader(mappedDoc.Node.HeadComment))
-		mappedDoc.Node.HeadComment = ""
-
-		for {
-
-			readline, errReading := reader.ReadString('\n')
-			if errReading != nil && errReading != io.EOF {
-				return errReading
-			}
-			if strings.Contains(readline, "$yqLeadingContent$") {
-				// skip this
-
-			} else if strings.Contains(readline, "$yqDocSeperator$") {
-				if p.printDocSeparators {
-					if err := p.writeString(writer, "---\n"); err != nil {
-						return err
-					}
-				}
-			} else if p.outputFormat == YamlOutputFormat {
-				if err := p.writeString(writer, readline); err != nil {
-					return err
-				}
-			}
-
-			if errReading == io.EOF {
-				if readline != "" {
-					// the last comment we read didn't have a new line, put one in
-					if err := p.writeString(writer, "\n"); err != nil {
-						return err
-					}
-				}
-				break
-			}
-		}
-
-	}
-	return nil
-}
-
 func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 	log.Debug("PrintResults for %v matches", matchingNodes.Len())
 
@@ -180,16 +132,17 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 			return errorWriting
 		}
 
-		commentStartsWithSeparator := strings.Contains(mappedDoc.Node.HeadComment, "$yqLeadingContent$\n$yqDocSeperator$")
+		commentsStartWithSepExp := regexp.MustCompile(`^\$yqDocSeperator\$`)
+		commentStartsWithSeparator := commentsStartWithSepExp.MatchString(mappedDoc.LeadingContent)
 
 		if (p.previousDocIndex != mappedDoc.Document || p.previousFileIndex != mappedDoc.FileIndex) && p.printDocSeparators && !commentStartsWithSeparator {
 			log.Debug("-- writing doc sep")
-			if err := p.writeString(writer, "---\n"); err != nil {
+			if err := writeString(writer, "---\n"); err != nil {
 				return err
 			}
 		}
 
-		if err := p.processLeadingContent(mappedDoc, writer); err != nil {
+		if err := processLeadingContent(mappedDoc, writer, p.printDocSeparators, p.outputFormat); err != nil {
 			return err
 		}
 
diff --git a/pkg/yqlib/stream_evaluator.go b/pkg/yqlib/stream_evaluator.go
index 84cebc2c..71380121 100644
--- a/pkg/yqlib/stream_evaluator.go
+++ b/pkg/yqlib/stream_evaluator.go
@@ -33,10 +33,11 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer, leadin
 		return err
 	}
 	candidateNode := &CandidateNode{
-		Document:  0,
-		Filename:  "",
-		Node:      &yaml.Node{Kind: yaml.DocumentNode, HeadComment: leadingContent, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
-		FileIndex: 0,
+		Document:       0,
+		Filename:       "",
+		Node:           &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
+		FileIndex:      0,
+		LeadingContent: leadingContent,
 	}
 	inputList := list.New()
 	inputList.PushBack(candidateNode)
@@ -100,15 +101,16 @@ func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *Expr
 		} else if errorReading != nil {
 			return currentIndex, errorReading
 		}
-		if currentIndex == 0 {
-			dataBucket.HeadComment = leadingContent
-		}
+
 		candidateNode := &CandidateNode{
 			Document:  currentIndex,
 			Filename:  filename,
 			Node:      &dataBucket,
 			FileIndex: s.fileIndex,
 		}
+		if currentIndex == 0 {
+			candidateNode.LeadingContent = leadingContent
+		}
 		inputList := list.New()
 		inputList.PushBack(candidateNode)
 
diff --git a/pkg/yqlib/utils.go b/pkg/yqlib/utils.go
index 4927e1f6..5d46a29e 100644
--- a/pkg/yqlib/utils.go
+++ b/pkg/yqlib/utils.go
@@ -31,10 +31,51 @@ func readStream(filename string, leadingContentPreProcessing bool) (io.Reader, s
 	return processReadStream(reader)
 }
 
+func writeString(writer io.Writer, txt string) error {
+	_, errorWriting := writer.Write([]byte(txt))
+	return errorWriting
+}
+
+func processLeadingContent(mappedDoc *CandidateNode, writer io.Writer, printDocSeparators bool, outputFormat PrinterOutputFormat) error {
+	log.Debug("headcommentwas %v", mappedDoc.LeadingContent)
+	log.Debug("finished headcomment")
+	reader := bufio.NewReader(strings.NewReader(mappedDoc.LeadingContent))
+
+	for {
+
+		readline, errReading := reader.ReadString('\n')
+		if errReading != nil && errReading != io.EOF {
+			return errReading
+		}
+		if strings.Contains(readline, "$yqDocSeperator$") {
+			if printDocSeparators {
+				if err := writeString(writer, "---\n"); err != nil {
+					return err
+				}
+			}
+		} else if outputFormat == YamlOutputFormat {
+			if err := writeString(writer, readline); err != nil {
+				return err
+			}
+		}
+
+		if errReading == io.EOF {
+			if readline != "" {
+				// the last comment we read didn't have a new line, put one in
+				if err := writeString(writer, "\n"); err != nil {
+					return err
+				}
+			}
+			break
+		}
+	}
+
+	return nil
+}
+
 func processReadStream(reader *bufio.Reader) (io.Reader, string, error) {
 	var commentLineRegEx = regexp.MustCompile(`^\s*#`)
 	var sb strings.Builder
-	sb.WriteString("$yqLeadingContent$\n")
 	for {
 		peekBytes, err := reader.Peek(3)
 		if err == io.EOF {


Test patch:
diff --git a/pkg/yqlib/all_at_once_evaluator_test.go b/pkg/yqlib/all_at_once_evaluator_test.go
index b75ab300..2e9abc43 100644
--- a/pkg/yqlib/all_at_once_evaluator_test.go
+++ b/pkg/yqlib/all_at_once_evaluator_test.go
@@ -35,6 +35,6 @@ func TestAllAtOnceEvaluateNodes(t *testing.T) {
 	for _, tt := range evaluateNodesScenario {
 		node := test.ParseData(tt.document)
 		list, _ := evaluator.EvaluateNodes(tt.expression, &node)
-		test.AssertResultComplex(t, tt.expected, resultsToString(list))
+		test.AssertResultComplex(t, tt.expected, resultsToString(t, list))
 	}
 }
diff --git a/pkg/yqlib/operator_comments_test.go b/pkg/yqlib/operator_comments_test.go
index 14b1d83e..bd724a8f 100644
--- a/pkg/yqlib/operator_comments_test.go
+++ b/pkg/yqlib/operator_comments_test.go
@@ -112,13 +112,22 @@ var commentOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!str)::welcome!\n",
 		},
 	},
+	{
+		skipDoc:               false,
+		dontFormatInputForDoc: true,
+		document:              "# welcome!\n---\n# bob\na: cat # meow\n\n# have a great day",
+		expression:            `headComment`,
+		expected: []string{
+			"D0, P[], (!!str)::welcome!\nbob\n",
+		},
+	},
 	{
 		description:           "Get foot comment",
 		dontFormatInputForDoc: true,
-		document:              "# welcome!\n\na: cat # meow\n\n# have a great day",
+		document:              "# welcome!\n\na: cat # meow\n\n# have a great day\n# no really",
 		expression:            `. | footComment`,
 		expected: []string{
-			"D0, P[], (!!str)::have a great day\n",
+			"D0, P[], (!!str)::have a great day\nno really\n",
 		},
 	},
 }
diff --git a/pkg/yqlib/operator_equals_test.go b/pkg/yqlib/operator_equals_test.go
index 360e7e99..1064ebd3 100644
--- a/pkg/yqlib/operator_equals_test.go
+++ b/pkg/yqlib/operator_equals_test.go
@@ -96,7 +96,7 @@ var equalsOperatorScenarios = []expressionScenario{
 		document:   "{a: { b: {things: \"\"}, f: [1], g: [] }}",
 		expression: ".. | select(. == \"\")",
 		expected: []string{
-			"D0, P[a b things], (!!str)::\"\"\n",
+			"D0, P[a b things], (!!str)::\n",
 		},
 	},
 	{
diff --git a/pkg/yqlib/operator_multiply_test.go b/pkg/yqlib/operator_multiply_test.go
index 39bb83a9..b1355075 100644
--- a/pkg/yqlib/operator_multiply_test.go
+++ b/pkg/yqlib/operator_multiply_test.go
@@ -35,33 +35,28 @@ We then need to update the first array. We will use the relative update (|=) bec
 We set the current element of the first array as $cur. Now we multiply (merge) $cur with the matching entry in $two, by passing $two through a select filter.
 `
 
-var docWithHeader = `
-# here
+var docWithHeader = `# here
 
 a: apple
 `
 
-var nodeWithHeader = `
-# here
-a: apple
+var nodeWithHeader = `node:
+  # here
+  a: apple
 `
 
-var docNoComments = `
-b: banana
+var docNoComments = `b: banana
 `
 
-var docWithFooter = `
-a: apple
+var docWithFooter = `a: apple
 
 # footer
 `
 
-var nodeWithFooter = `
-a: apple
+var nodeWithFooter = `a: apple
 # footer`
 
-var document = `
-a: &cat {name: cat}
+var document = `a: &cat {name: cat}
 b: {name: dog}
 c: 
   <<: *cat
@@ -89,9 +84,9 @@ var multiplyOperatorScenarios = []expressionScenario{
 		skipDoc:    true,
 		document:   nodeWithHeader,
 		document2:  docNoComments,
-		expression: `select(fi == 0) * select(fi == 1)`,
+		expression: `(select(fi == 0) | .node) * select(fi == 1)`,
 		expected: []string{
-			"D0, P[], (!!map)::# here\na: apple\nb: banana\n",
+			"D0, P[node], (!!map)::# here\na: apple\nb: banana\n",
 		},
 	},
 	{
@@ -107,7 +102,7 @@ var multiplyOperatorScenarios = []expressionScenario{
 		skipDoc:    true,
 		document:   docNoComments,
 		document2:  nodeWithHeader,
-		expression: `select(fi == 0) * select(fi == 1)`,
+		expression: `select(fi == 0) * (select(fi == 1) | .node)`,
 		expected: []string{
 			"D0, P[], (!!map)::b: banana\n# here\na: apple\n",
 		},
diff --git a/pkg/yqlib/operator_recursive_descent_test.go b/pkg/yqlib/operator_recursive_descent_test.go
index 61caab12..0893242b 100644
--- a/pkg/yqlib/operator_recursive_descent_test.go
+++ b/pkg/yqlib/operator_recursive_descent_test.go
@@ -219,7 +219,7 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
 			"D0, P[foobarList], (!!map)::b: foobarList_b\n!!merge <<: [*foo, *bar]\nc: foobarList_c\n",
 			"D0, P[foobarList b], (!!str)::b\n",
 			"D0, P[foobarList b], (!!str)::foobarList_b\n",
-			"D0, P[foobarList <<], (!!merge)::!!merge <<\n",
+			"D0, P[foobarList <<], (!!merge)::<<\n",
 			"D0, P[foobarList <<], (!!seq)::[*foo, *bar]\n",
 			"D0, P[foobarList << 0], (alias)::*foo\n",
 			"D0, P[foobarList << 1], (alias)::*bar\n",
diff --git a/pkg/yqlib/operator_style_test.go b/pkg/yqlib/operator_style_test.go
index e02fa3ad..5f2490fd 100644
--- a/pkg/yqlib/operator_style_test.go
+++ b/pkg/yqlib/operator_style_test.go
@@ -142,8 +142,8 @@ e: >-
 		document:   `a: cat`,
 		expression: `.. | style`,
 		expected: []string{
-			"D0, P[], (!!str)::\"\"\n",
-			"D0, P[a], (!!str)::\"\"\n",
+			"D0, P[], (!!str)::\n",
+			"D0, P[a], (!!str)::\n",
 		},
 	},
 }
diff --git a/pkg/yqlib/operator_tag_test.go b/pkg/yqlib/operator_tag_test.go
index fd90f734..08b22741 100644
--- a/pkg/yqlib/operator_tag_test.go
+++ b/pkg/yqlib/operator_tag_test.go
@@ -10,12 +10,12 @@ var tagOperatorScenarios = []expressionScenario{
 		document:    `{a: cat, b: 5, c: 3.2, e: true, f: []}`,
 		expression:  `.. | tag`,
 		expected: []string{
-			"D0, P[], (!!str)::'!!map'\n",
-			"D0, P[a], (!!str)::'!!str'\n",
-			"D0, P[b], (!!str)::'!!int'\n",
-			"D0, P[c], (!!str)::'!!float'\n",
-			"D0, P[e], (!!str)::'!!bool'\n",
-			"D0, P[f], (!!str)::'!!seq'\n",
+			"D0, P[], (!!str)::!!map\n",
+			"D0, P[a], (!!str)::!!str\n",
+			"D0, P[b], (!!str)::!!int\n",
+			"D0, P[c], (!!str)::!!float\n",
+			"D0, P[e], (!!str)::!!bool\n",
+			"D0, P[f], (!!str)::!!seq\n",
 		},
 	},
 	{
@@ -23,7 +23,7 @@ var tagOperatorScenarios = []expressionScenario{
 		document:   `{a: cat, b: 5, c: 3.2, e: true, f: []}`,
 		expression: `tag`,
 		expected: []string{
-			"D0, P[], (!!str)::'!!map'\n",
+			"D0, P[], (!!str)::!!map\n",
 		},
 	},
 	{
diff --git a/pkg/yqlib/operator_value_test.go b/pkg/yqlib/operator_value_test.go
index 4821c1e3..81534804 100644
--- a/pkg/yqlib/operator_value_test.go
+++ b/pkg/yqlib/operator_value_test.go
@@ -125,13 +125,13 @@ var valueOperatorScenarios = []expressionScenario{
 		document:   ``,
 		expression: `"1.3"`,
 		expected: []string{
-			"D0, P[], (!!str)::\"1.3\"\n",
+			"D0, P[], (!!str)::1.3\n",
 		},
 	}, {
 		document:   ``,
 		expression: `"true"`,
 		expected: []string{
-			"D0, P[], (!!str)::\"true\"\n",
+			"D0, P[], (!!str)::true\n",
 		},
 	}, {
 		document:   ``,
diff --git a/pkg/yqlib/operators_test.go b/pkg/yqlib/operators_test.go
index af7b1d7a..9352950a 100644
--- a/pkg/yqlib/operators_test.go
+++ b/pkg/yqlib/operators_test.go
@@ -26,6 +26,20 @@ type expressionScenario struct {
 	dontFormatInputForDoc bool // dont format input doc for documentation generation
 }
 
+func readDocumentWithLeadingContent(content string, fakefilename string, fakeFileIndex int) (*list.List, error) {
+	reader, firstFileLeadingContent, err := processReadStream(bufio.NewReader(strings.NewReader(content)))
+	if err != nil {
+		return nil, err
+	}
+
+	inputs, err := readDocuments(reader, fakefilename, fakeFileIndex)
+	if err != nil {
+		return nil, err
+	}
+	inputs.Front().Value.(*CandidateNode).LeadingContent = firstFileLeadingContent
+	return inputs, nil
+}
+
 func testScenario(t *testing.T, s *expressionScenario) {
 	var err error
 
@@ -37,15 +51,17 @@ func testScenario(t *testing.T, s *expressionScenario) {
 	inputs := list.New()
 
 	if s.document != "" {
-		inputs, err = readDocuments(strings.NewReader(s.document), "sample.yml", 0)
+		inputs, err = readDocumentWithLeadingContent(s.document, "sample.yml", 0)
+
 		if err != nil {
 			t.Error(err, s.document, s.expression)
 			return
 		}
+
 		if s.document2 != "" {
-			moreInputs, err := readDocuments(strings.NewReader(s.document2), "another.yml", 1)
+			moreInputs, err := readDocumentWithLeadingContent(s.document2, "another.yml", 1)
 			if err != nil {
-				t.Error(err, s.document, s.expression)
+				t.Error(err, s.document2, s.expression)
 				return
 			}
 			inputs.PushBackList(moreInputs)
@@ -71,14 +87,31 @@ func testScenario(t *testing.T, s *expressionScenario) {
 		t.Error(fmt.Errorf("%v: %v", err, s.expression))
 		return
 	}
-	test.AssertResultComplexWithContext(t, s.expected, resultsToString(context.MatchingNodes), fmt.Sprintf("desc: %v\nexp: %v\ndoc: %v", s.description, s.expression, s.document))
+	test.AssertResultComplexWithContext(t, s.expected, resultsToString(t, context.MatchingNodes), fmt.Sprintf("desc: %v\nexp: %v\ndoc: %v", s.description, s.expression, s.document))
 }
 
-func resultsToString(results *list.List) []string {
+func resultsToString(t *testing.T, results *list.List) []string {
 	var pretty []string = make([]string, 0)
+
 	for el := results.Front(); el != nil; el = el.Next() {
 		n := el.Value.(*CandidateNode)
-		pretty = append(pretty, NodeToString(n))
+		var valueBuffer bytes.Buffer
+		printer := NewPrinterWithSingleWriter(bufio.NewWriter(&valueBuffer), YamlOutputFormat, true, false, 4, true)
+
+		err := printer.PrintResults(n.AsList())
+		if err != nil {
+			t.Error(err)
+			return nil
+		}
+
+		tag := n.Node.Tag
+		if n.Node.Kind == yaml.DocumentNode {
+			tag = "doc"
+		} else if n.Node.Kind == yaml.AliasNode {
+			tag = "alias"
+		}
+		output := fmt.Sprintf(`D%v, P%v, (%v)::%v`, n.Document, n.Path, tag, valueBuffer.String())
+		pretty = append(pretty, output)
 	}
 	return pretty
 }
@@ -227,13 +260,14 @@ func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formatt
 	inputs := list.New()
 
 	if s.document != "" {
-		inputs, err = readDocuments(strings.NewReader(formattedDoc), "sample.yml", 0)
+
+		inputs, err = readDocumentWithLeadingContent(formattedDoc, "sample.yml", 0)
 		if err != nil {
 			t.Error(err, s.document, s.expression)
 			return
 		}
 		if s.document2 != "" {
-			moreInputs, err := readDocuments(strings.NewReader(formattedDoc2), "another.yml", 1)
+			moreInputs, err := readDocumentWithLeadingContent(formattedDoc2, "another.yml", 1)
 			if err != nil {
 				t.Error(err, s.document, s.expression)
 				return
diff --git a/pkg/yqlib/printer_test.go b/pkg/yqlib/printer_test.go
index 56acd5a8..8c74af87 100644
--- a/pkg/yqlib/printer_test.go
+++ b/pkg/yqlib/printer_test.go
@@ -82,15 +82,15 @@ func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {
 	}
 
 	el := inputs.Front()
-	el.Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n# go cats\n$yqDocSeperator$\n"
+	el.Value.(*CandidateNode).LeadingContent = "# go cats\n$yqDocSeperator$\n"
 	sample1 := nodeToList(el.Value.(*CandidateNode))
 
 	el = el.Next()
-	el.Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n"
+	el.Value.(*CandidateNode).LeadingContent = "$yqDocSeperator$\n"
 	sample2 := nodeToList(el.Value.(*CandidateNode))
 
 	el = el.Next()
-	el.Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n# cool\n"
+	el.Value.(*CandidateNode).LeadingContent = "$yqDocSeperator$\n# cool\n"
 	sample3 := nodeToList(el.Value.(*CandidateNode))
 
 	err = printer.PrintResults(sample1)
@@ -174,21 +174,21 @@ func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
 	elNode := el.Value.(*CandidateNode)
 	elNode.Document = 0
 	elNode.FileIndex = 0
-	elNode.Node.HeadComment = "$yqLeadingContent$\n# go cats\n$yqDocSeperator$\n"
+	elNode.LeadingContent = "# go cats\n$yqDocSeperator$\n"
 	sample1 := nodeToList(elNode)
 
 	el = el.Next()
 	elNode = el.Value.(*CandidateNode)
 	elNode.Document = 0
 	elNode.FileIndex = 1
-	elNode.Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n"
+	elNode.LeadingContent = "$yqDocSeperator$\n"
 	sample2 := nodeToList(elNode)
 
 	el = el.Next()
 	elNode = el.Value.(*CandidateNode)
 	elNode.Document = 0
 	elNode.FileIndex = 2
-	elNode.Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n# cool\n"
+	elNode.LeadingContent = "$yqDocSeperator$\n# cool\n"
 	sample3 := nodeToList(elNode)
 
 	err = printer.PrintResults(sample1)
@@ -239,7 +239,7 @@ func TestPrinterMultipleDocsInSinglePrintWithLeadingDoc(t *testing.T) {
 		panic(err)
 	}
 
-	inputs.Front().Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n# go cats\n$yqDocSeperator$\n"
+	inputs.Front().Value.(*CandidateNode).LeadingContent = "# go cats\n$yqDocSeperator$\n"
 
 	err = printer.PrintResults(inputs)
 	if err != nil {
@@ -267,7 +267,7 @@ func TestPrinterMultipleDocsInSinglePrintWithLeadingDocTrailing(t *testing.T) {
 	if err != nil {
 		panic(err)
 	}
-	inputs.Front().Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n"
+	inputs.Front().Value.(*CandidateNode).LeadingContent = "$yqDocSeperator$\n"
 	err = printer.PrintResults(inputs)
 	if err != nil {
 		panic(err)
@@ -321,7 +321,7 @@ func TestPrinterMultipleDocsJson(t *testing.T) {
 		panic(err)
 	}
 
-	inputs.Front().Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n# ignore this\n"
+	inputs.Front().Value.(*CandidateNode).LeadingContent = "# ignore this\n"
 
 	err = printer.PrintResults(inputs)
 	if err != nil {