Repository: mikefarah/yq (go)

Issue:
JSON conversion (still) does not preserve object order
**Describe the bug**

It seems from #274 having been fixed that conversion to JSON should now be preserving object order, but that's not what I'm seeing.

**Input Yaml**
data1.yml:
```yaml
foo: a
bar: b
```

**Command**
```
yq r -jP data1.yaml
```

**Actual behavior**
```json
{
  "bar": "b",
  "foo": "a"
}
```

**Expected behavior**
```json
{
  "foo": "a",
  "bar": "b"
}
```

**Additional context**
yq version 3.3.0 on macOS 10.15.3

Pull request:
keep order of keys when json marshalling
I've implemented a custom solution to #435. (fixes #435)

This implementation uses a custom type, `orderedMap`, with custom marshalers/unmarshalers for both JSON and YAML.

Tests have been changed accordingly to allow for the new behaviour.

Patch:
diff --git a/examples/sample.json b/examples/sample.json
index d322c6b0..5a35f972 100644
--- a/examples/sample.json
+++ b/examples/sample.json
@@ -1,1 +1,1 @@
-{"a":"Easy! as one two three","b":{"c":2,"d":[3,4],"e":[{"name":"fred","value":3},{"name":"sam","value":4}]}}
+{"a":"Easy! as one two three","b":{"c":2,"d":[3,4],"e":[{"name":"fred","value":3},{"name":"sam","value":4}]},"ab":"must appear last"}
diff --git a/pkg/yqlib/encoder.go b/pkg/yqlib/encoder.go
index 5694722c..e9ab9587 100644
--- a/pkg/yqlib/encoder.go
+++ b/pkg/yqlib/encoder.go
@@ -3,6 +3,7 @@ package yqlib
 import (
 	"bytes"
 	"encoding/json"
+	"fmt"
 	"io"
 
 	yaml "gopkg.in/yaml.v3"
@@ -87,7 +88,7 @@ func NewJsonEncoder(destination io.Writer, prettyPrint bool, indent int) Encoder
 }
 
 func (je *jsonEncoder) Encode(node *yaml.Node) error {
-	var dataBucket interface{}
+	var dataBucket orderedMap
 	// firstly, convert all map keys to strings
 	mapKeysToStrings(node)
 	errorDecoding := node.Decode(&dataBucket)
@@ -96,3 +97,153 @@ func (je *jsonEncoder) Encode(node *yaml.Node) error {
 	}
 	return je.encoder.Encode(dataBucket)
 }
+
+// orderedMap allows to marshal and unmarshal JSON and YAML values keeping the
+// order of keys and values in a map or an object.
+type orderedMap struct {
+	// if this is an object, kv != nil. If this is not an object, kv == nil.
+	kv     []orderedMapKV
+	altVal interface{}
+}
+
+type orderedMapKV struct {
+	K string
+	V orderedMap
+}
+
+func (o *orderedMap) UnmarshalJSON(data []byte) error {
+	switch data[0] {
+	case '{':
+		// initialise so that even if the object is empty it is not nil
+		o.kv = []orderedMapKV{}
+
+		// create decoder
+		dec := json.NewDecoder(bytes.NewReader(data))
+		_, err := dec.Token() // open object
+		if err != nil {
+			return err
+		}
+
+		// cycle through k/v
+		var tok json.Token
+		for tok, err = dec.Token(); err != io.EOF; tok, err = dec.Token() {
+			// we can expect two types: string or Delim. Delim automatically means
+			// that it is the closing bracket of the object, whereas string means
+			// that there is another key.
+			if _, ok := tok.(json.Delim); ok {
+				break
+			}
+			kv := orderedMapKV{
+				K: tok.(string),
+			}
+			if err := dec.Decode(&kv.V); err != nil {
+				return err
+			}
+			o.kv = append(o.kv, kv)
+		}
+		// unexpected error
+		if err != nil && err != io.EOF {
+			return err
+		}
+		return nil
+	case '[':
+		var arr []orderedMap
+		return json.Unmarshal(data, &arr)
+	}
+
+	return json.Unmarshal(data, &o.altVal)
+}
+
+func (o orderedMap) MarshalJSON() ([]byte, error) {
+	if o.kv == nil {
+		return json.Marshal(o.altVal)
+	}
+	buf := new(bytes.Buffer)
+	enc := json.NewEncoder(buf)
+	buf.WriteByte('{')
+	for idx, el := range o.kv {
+		if err := enc.Encode(el.K); err != nil {
+			return nil, err
+		}
+		buf.WriteByte(':')
+		if err := enc.Encode(el.V); err != nil {
+			return nil, err
+		}
+		if idx != len(o.kv)-1 {
+			buf.WriteByte(',')
+		}
+	}
+	buf.WriteByte('}')
+	return buf.Bytes(), nil
+}
+
+func (o *orderedMap) UnmarshalYAML(node *yaml.Node) error {
+	switch node.Kind {
+	case yaml.DocumentNode:
+		if len(node.Content) == 0 {
+			return nil
+		}
+		return o.UnmarshalYAML(node.Content[0])
+	case yaml.AliasNode:
+		return o.UnmarshalYAML(node.Alias)
+	case yaml.ScalarNode:
+		return node.Decode(&o.altVal)
+	case yaml.MappingNode:
+		// set kv to non-nil
+		o.kv = []orderedMapKV{}
+		for i := 0; i < len(node.Content); i += 2 {
+			var key string
+			var val orderedMap
+			if err := node.Content[i].Decode(&key); err != nil {
+				return err
+			}
+			if err := node.Content[i+1].Decode(&val); err != nil {
+				return err
+			}
+			o.kv = append(o.kv, orderedMapKV{
+				K: key,
+				V: val,
+			})
+		}
+		return nil
+	case yaml.SequenceNode:
+		var res []orderedMap
+		if err := node.Decode(&res); err != nil {
+			return err
+		}
+		o.altVal = res
+		o.kv = nil
+		return nil
+	case 0:
+		// null
+		o.kv = nil
+		o.altVal = nil
+		return nil
+	default:
+		return fmt.Errorf("orderedMap: invalid yaml node")
+	}
+}
+
+func (o *orderedMap) MarshalYAML() (interface{}, error) {
+	// fast path: kv is nil, use altVal
+	if o.kv == nil {
+		return o.altVal, nil
+	}
+	content := make([]*yaml.Node, 0, len(o.kv)*2)
+	for _, val := range o.kv {
+		n := new(yaml.Node)
+		if err := n.Encode(val.V); err != nil {
+			return nil, err
+		}
+		content = append(content, &yaml.Node{
+			Kind:  yaml.ScalarNode,
+			Tag:   "!!str",
+			Value: val.K,
+		}, n)
+	}
+	return &yaml.Node{
+		Kind:    yaml.MappingNode,
+		Tag:     "!!map",
+		Content: content,
+	}, nil
+}


Test patch:
diff --git a/cmd/read_test.go b/cmd/read_test.go
index d68547d8..544fdb89 100644
--- a/cmd/read_test.go
+++ b/cmd/read_test.go
@@ -98,8 +98,8 @@ func TestReadOutputJsonNonStringKeysCmd(t *testing.T) {
 
 	content := `
 true: true
-5: 
-  null: 
+5:
+  null:
     0.1: deeply
     false: things`
 	filename := test.WriteTempYamlFile(content)
@@ -111,7 +111,7 @@ true: true
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
-	expectedOutput := `{"5":{"null":{"0.1":"deeply","false":"things"}},"true":true}
+	expectedOutput := `{"true":true,"5":{"null":{"0.1":"deeply","false":"things"}}}
 `
 	test.AssertResult(t, expectedOutput, result.Output)
 }
@@ -214,7 +214,7 @@ func TestReadArrayLengthCmd(t *testing.T) {
 }
 
 func TestReadArrayLengthDeepCmd(t *testing.T) {
-	content := `holder: 
+	content := `holder:
 - things
 - whatever
 `
@@ -230,12 +230,12 @@ func TestReadArrayLengthDeepCmd(t *testing.T) {
 }
 
 func TestReadArrayLengthDeepMultipleCmd(t *testing.T) {
-	content := `holderA: 
+	content := `holderA:
 - things
 - whatever
 skipMe:
 - yep
-holderB: 
+holderB:
 - other things
 - cool
 `
@@ -294,10 +294,10 @@ func TestReadCollectArrayCmd(t *testing.T) {
 }
 
 func TestReadArrayLengthDeepMultipleWithPathCmd(t *testing.T) {
-	content := `holderA: 
+	content := `holderA:
 - things
 - whatever
-holderB: 
+holderB:
 - other things
 - cool
 `
@@ -328,7 +328,7 @@ dog: bark
 }
 
 func TestReadObjectLengthDeepCmd(t *testing.T) {
-	content := `holder: 
+	content := `holder:
   cat: meow
   dog: bark
 `
@@ -344,10 +344,10 @@ func TestReadObjectLengthDeepCmd(t *testing.T) {
 }
 
 func TestReadObjectLengthDeepMultipleCmd(t *testing.T) {
-	content := `holderA: 
+	content := `holderA:
   cat: meow
   dog: bark
-holderB: 
+holderB:
   elephant: meow
   zebra: bark
 `
@@ -363,10 +363,10 @@ holderB:
 }
 
 func TestReadObjectLengthDeepMultipleWithPathsCmd(t *testing.T) {
-	content := `holderA: 
+	content := `holderA:
   cat: meow
   dog: bark
-holderB: 
+holderB:
   elephant: meow
   zebra: bark
 `
@@ -422,7 +422,7 @@ func TestReadSingleQuotedStringCmd(t *testing.T) {
 
 func TestReadQuotedMultinlineStringCmd(t *testing.T) {
 	content := `test: |
-  abcdefg    
+  abcdefg
   hijklmno
 `
 	filename := test.WriteTempYamlFile(content)
@@ -433,7 +433,7 @@ func TestReadQuotedMultinlineStringCmd(t *testing.T) {
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
-	expectedOutput := `abcdefg    
+	expectedOutput := `abcdefg
 hijklmno
 
 `
@@ -442,7 +442,7 @@ hijklmno
 
 func TestReadQuotedMultinlineNoNewLineStringCmd(t *testing.T) {
 	content := `test: |-
-  abcdefg    
+  abcdefg
   hijklmno
 `
 	filename := test.WriteTempYamlFile(content)
@@ -453,7 +453,7 @@ func TestReadQuotedMultinlineNoNewLineStringCmd(t *testing.T) {
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
-	expectedOutput := `abcdefg    
+	expectedOutput := `abcdefg
 hijklmno
 `
 	test.AssertResult(t, expectedOutput, result.Output)
@@ -583,7 +583,7 @@ func TestReadMergeAnchorsExplodeJsonCmd(t *testing.T) {
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
-	expectedOutput := `{"bar":{"b":2,"c":"oldbar","thing":"coconut"},"foo":{"a":"original","thing":"coolasdf","thirsty":"yep"},"foobar":{"a":"original","c":3,"thing":"ice","thirsty":"yep","thirty":"well beyond"},"foobarList":{"a":"original","b":2,"c":"newbar","thing":"coconut","thirsty":"yep"}}
+	expectedOutput := `{"foo":{"a":"original","thing":"coolasdf","thirsty":"yep"},"bar":{"b":2,"thing":"coconut","c":"oldbar"},"foobarList":{"c":"newbar","b":2,"thing":"coconut","a":"original","thirsty":"yep"},"foobar":{"thirty":"well beyond","thing":"ice","c":3,"a":"original","thirsty":"yep"}}
 `
 	test.AssertResult(t, expectedOutput, result.Output)
 }
@@ -910,9 +910,9 @@ func TestReadEmptyContentCmd(t *testing.T) {
 }
 
 func TestReadEmptyNodesPrintPathCmd(t *testing.T) {
-	content := `map: 
+	content := `map:
   that: {}
-array: 
+array:
   great: []
 null:
   indeed: ~`
@@ -962,6 +962,7 @@ b:
       value: 3
     - name: sam
       value: 4
+ab: must appear last
 `
 	test.AssertResult(t, expectedOutput, result.Output)
 }
@@ -1001,6 +1002,7 @@ b:
           value: 3
         - name: sam
           value: 4
+ab: must appear last
 `
 	test.AssertResult(t, expectedOutput, result.Output)
 }
@@ -1269,7 +1271,7 @@ func TestReadBadDataCmd(t *testing.T) {
 
 func TestReadDeepFromRootCmd(t *testing.T) {
 	content := `state:
-  country: 
+  country:
     city: foo
 `
 	filename := test.WriteTempYamlFile(content)
@@ -1439,3 +1441,23 @@ func TestReadFindValueDeepObjectCmd(t *testing.T) {
 `
 	test.AssertResult(t, expectedOutput, result.Output)
 }
+
+func TestReadKeepsKeyOrderInJson(t *testing.T) {
+	const content = `{
+	"z": "One",
+	"a": 1,
+	"w": ["a", "r"],
+	"u": {"d": "o", "0": 11.5},
+}`
+	filename := test.WriteTempYamlFile(content)
+	defer test.RemoveTempYamlFile(filename)
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, fmt.Sprintf("r -j %s", filename))
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+
+	expectedOutput := `{"z":"One","a":1,"w":["a","r"],"u":{"d":"o","0":11.5}}
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}