Repository: TomWright/dasel
Pull request #48 (2020-11-18 14:24:04)

Add multiJson file support
Check comments in commits.

Sorry for the long wait. I started playing around with Go inheritance to collect JSON and YAML parser in a single abstract implementation... but that got hairy... check branch "collectYamlJson" (I'm very new to go, so thanks for giving me the possibility to work on this - that was fun... but impossible to fix without breaking the API (Go needs possiblity of editing default constructors to make it work))

MikkelHJuul: I will look into marshaling correctly
TomWright: Could you also resolve any issues on go report card please? https://goreportcard.com/report/github.com/MikkelHJuul/dasel

Thanks for your contribution!
TomWright: @MikkelHJuul I've made the change to pretty-print the JSON

Patch:
diff --git a/README.md b/README.md
index e73aae4..100a830 100644
--- a/README.md
+++ b/README.md
@@ -449,6 +449,12 @@ Dasel attempts to find the correct parser for the given file type, but if that f
 ```
 Using [golang.org/pkg/encoding/json](https://golang.org/pkg/encoding/json/).
 
+#### Multi-document files
+
+Multi-document files are decoded into an array, with `[0]` being the first document, `[1]` being the second and so on.
+
+Once decoded, you can access them using any of the standard selectors provided by Dasel.
+
 ### TOML
 ```bash
 -p toml
diff --git a/internal/storage/csv.go b/internal/storage/csv.go
index 507cb1f..4faeb13 100644
--- a/internal/storage/csv.go
+++ b/internal/storage/csv.go
@@ -23,7 +23,7 @@ func (d *CSVDocument) RealValue() interface{} {
 	return d.Value
 }
 
-// FromBytes returns some Data that is represented by the given bytes.
+// FromBytes returns some data that is represented by the given bytes.
 func (p *CSVParser) FromBytes(byteData []byte) (interface{}, error) {
 	if byteData == nil {
 		return nil, fmt.Errorf("could not read csv file: no data")
diff --git a/internal/storage/json.go b/internal/storage/json.go
index 5fa4f1f..a03bb58 100644
--- a/internal/storage/json.go
+++ b/internal/storage/json.go
@@ -1,28 +1,86 @@
 package storage
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
+	"io"
 )
 
 // JSONParser is a Parser implementation to handle yaml files.
 type JSONParser struct {
 }
 
-// FromBytes returns some Data that is represented by the given bytes.
+// JSONSingleDocument represents a decoded single-document YAML file.
+type JSONSingleDocument struct {
+	originalRequired
+	Value interface{}
+}
+
+// RealValue returns the real value that dasel should use when processing data.
+func (d *JSONSingleDocument) RealValue() interface{} {
+	return d.Value
+}
+
+// JSONMultiDocument represents a decoded multi-document YAML file.
+type JSONMultiDocument struct {
+	originalRequired
+	Values []interface{}
+}
+
+// RealValue returns the real value that dasel should use when processing data.
+func (d *JSONMultiDocument) RealValue() interface{} {
+	return d.Values
+}
+
+// FromBytes returns some data that is represented by the given bytes.
 func (p *JSONParser) FromBytes(byteData []byte) (interface{}, error) {
-	var data interface{}
-	if err := json.Unmarshal(byteData, &data); err != nil {
-		return data, fmt.Errorf("could not unmarshal data: %w", err)
+	res := make([]interface{}, 0)
+
+	decoder := json.NewDecoder(bytes.NewBuffer(byteData))
+
+docLoop:
+	for {
+		var docData interface{}
+		if err := decoder.Decode(&docData); err != nil {
+			if err == io.EOF {
+				break docLoop
+			}
+			return nil, fmt.Errorf("could not unmarshal data: %w", err)
+		}
+		res = append(res, docData)
+	}
+	switch len(res) {
+	case 0:
+		return nil, nil
+	case 1:
+		return &JSONSingleDocument{Value: res[0]}, nil
+	default:
+		return &JSONMultiDocument{Values: res}, nil
 	}
-	return data, nil
 }
 
 // ToBytes returns a slice of bytes that represents the given value.
 func (p *JSONParser) ToBytes(value interface{}) ([]byte, error) {
-	byteData, err := json.MarshalIndent(value, "", "  ")
-	if err == nil {
-		byteData = append(byteData, []byte("\n")...)
+	buffer := new(bytes.Buffer)
+	encoder := json.NewEncoder(buffer)
+	encoder.SetIndent("", "  ")
+
+	switch v := value.(type) {
+	case *JSONSingleDocument:
+		if err := encoder.Encode(v.Value); err != nil {
+			return nil, fmt.Errorf("could not encode single document: %w", err)
+		}
+	case *JSONMultiDocument:
+		for index, d := range v.Values {
+			if err := encoder.Encode(d); err != nil {
+				return nil, fmt.Errorf("could not encode multi document [%d]: %w", index, err)
+			}
+		}
+	default:
+		if err := encoder.Encode(v); err != nil {
+			return nil, fmt.Errorf("could not encode default document type: %w", err)
+		}
 	}
-	return byteData, err
+	return buffer.Bytes(), nil
 }
diff --git a/internal/storage/parser.go b/internal/storage/parser.go
index 97b1455..2a67e1c 100644
--- a/internal/storage/parser.go
+++ b/internal/storage/parser.go
@@ -21,7 +21,7 @@ func (e UnknownParserErr) Error() string {
 
 // Parser can be used to load and save files from/to disk.
 type Parser interface {
-	// FromBytes returns some Data that is represented by the given bytes.
+	// FromBytes returns some data that is represented by the given bytes.
 	FromBytes(byteData []byte) (interface{}, error)
 	// ToBytes returns a slice of bytes that represents the given value.
 	ToBytes(value interface{}) ([]byte, error)
@@ -49,7 +49,7 @@ func NewParserFromFilename(filename string) (Parser, error) {
 // NewParserFromString returns a Parser from the given parser name.
 func NewParserFromString(parser string) (Parser, error) {
 	switch parser {
-	case "yaml":
+	case "yaml", "yml":
 		return &YAMLParser{}, nil
 	case "json":
 		return &JSONParser{}, nil
diff --git a/internal/storage/toml.go b/internal/storage/toml.go
index 52aea0c..762d032 100644
--- a/internal/storage/toml.go
+++ b/internal/storage/toml.go
@@ -5,11 +5,11 @@ import (
 	"github.com/pelletier/go-toml"
 )
 
-// TOMLParser is a Parser implementation to handle yaml files.
+// TOMLParser is a Parser implementation to handle toml files.
 type TOMLParser struct {
 }
 
-// FromBytes returns some Data that is represented by the given bytes.
+// FromBytes returns some data that is represented by the given bytes.
 func (p *TOMLParser) FromBytes(byteData []byte) (interface{}, error) {
 	var data interface{}
 	if err := toml.Unmarshal(byteData, &data); err != nil {
diff --git a/internal/storage/xml.go b/internal/storage/xml.go
index 0bc01c2..956cf2c 100644
--- a/internal/storage/xml.go
+++ b/internal/storage/xml.go
@@ -9,7 +9,7 @@ import (
 type XMLParser struct {
 }
 
-// FromBytes returns some Data that is represented by the given bytes.
+// FromBytes returns some data that is represented by the given bytes.
 func (p *XMLParser) FromBytes(byteData []byte) (interface{}, error) {
 	data, err := mxj.NewMapXml(byteData)
 	if err != nil {
diff --git a/internal/storage/yaml.go b/internal/storage/yaml.go
index bde446a..c64cb7f 100644
--- a/internal/storage/yaml.go
+++ b/internal/storage/yaml.go
@@ -33,7 +33,7 @@ func (d *YAMLMultiDocument) RealValue() interface{} {
 	return d.Values
 }
 
-// FromBytes returns some Data that is represented by the given bytes.
+// FromBytes returns some data that is represented by the given bytes.
 func (p *YAMLParser) FromBytes(byteData []byte) (interface{}, error) {
 	res := make([]interface{}, 0)
 


Test patch:
diff --git a/internal/command/root_put_test.go b/internal/command/root_put_test.go
index 3716c93..bf37eb9 100644
--- a/internal/command/root_put_test.go
+++ b/internal/command/root_put_test.go
@@ -94,8 +94,7 @@ func putTest(in string, varType string, parser string, selector string, value st
 			return
 		}
 
-		out = strings.TrimSpace(out)
-		outputStr := strings.TrimSpace(string(output))
+		outputStr := string(output)
 		if out != outputStr {
 			t.Errorf("expected result %v, got %v", out, outputStr)
 		}
@@ -152,17 +151,20 @@ func TestRootCMD_Put_JSON(t *testing.T) {
   "id": "x"
 }`, "json", "id", "y", `{
   "id": "y"
-}`, nil))
+}
+`, nil))
 	t.Run("Int", putIntTest(`{
   "id": 123
 }`, "json", "id", "456", `{
   "id": 456
-}`, nil))
+}
+`, nil))
 	t.Run("Bool", putBoolTest(`{
   "id": true
 }`, "json", "id", "false", `{
   "id": false
-}`, nil))
+}
+`, nil))
 	t.Run("OverwriteObject", putObjectTest(`{
   "numbers": [
     {
@@ -346,7 +348,8 @@ func TestRootCMD_Put_JSON(t *testing.T) {
   {
     "value": "X"
   }
-]`, nil, "-m"))
+]
+`, nil, "-m"))
 
 	t.Run("KeySearch", putStringTest(`{
   "users": [
@@ -394,7 +397,8 @@ func TestRootCMD_Put_JSON(t *testing.T) {
       "primary": false
     }
   ]
-}`, nil, "-m"))
+}
+`, nil, "-m"))
 
 	t.Run("ValueSearch", putStringTest(`{
   "users": [
@@ -442,7 +446,8 @@ func TestRootCMD_Put_JSON(t *testing.T) {
       "primary": false
     }
   ]
-}`, nil, "-m"))
+}
+`, nil, "-m"))
 
 	t.Run("KeyValueSearch", putStringTest(`{
   "users": [
@@ -490,33 +495,64 @@ func TestRootCMD_Put_JSON(t *testing.T) {
       "primary": false
     }
   ]
-}`, nil, "-m"))
+}
+`, nil, "-m"))
+
+	t.Run("StringMultiObjectDocument", putStringTest(`{
+  "id": "x"
+}
+{
+  "id": "y"
+}`, "json", ".[0].id", "z", `{
+  "id": "z"
+}
+{
+  "id": "y"
+}
+`, nil))
+
+	t.Run("StringMultiArrayDocument", putStringTest(`[
+  "a",
+  "b",
+  "c"
+]
+[
+  "d",
+  "e",
+  "f"
+]`, "json", ".[1].[1]", "z", `[
+  "a",
+  "b",
+  "c"
+]
+[
+  "d",
+  "z",
+  "f"
+]
+`, nil))
 }
 
 func TestRootCMD_Put_YAML(t *testing.T) {
 	t.Run("String", putStringTest(`
 id: "x"
 name: "Tom"
-`, "yaml", "id", "y", `
-id: "y"
+`, "yaml", "id", "y", `id: "y"
 name: Tom
 `, nil))
 	t.Run("StringInFile", putFileTest(`
 id: "x"
 name: "Tom"
-`, "string", "yaml", "id", "y", `
-id: "y"
+`, "string", "yaml", "id", "y", `id: "y"
 name: Tom
 `, "TestRootCMD_Put_YAML_out.yaml", nil))
 	t.Run("Int", putIntTest(`
 id: 123
-`, "yaml", "id", "456", `
-id: 456
+`, "yaml", "id", "456", `id: 456
 `, nil))
 	t.Run("Bool", putBoolTest(`
 id: true
-`, "yaml", "id", "false", `
-id: false
+`, "yaml", "id", "false", `id: false
 `, nil))
 	t.Run("OverwriteObject", putObjectTest(`
 numbers:
@@ -541,8 +577,7 @@ id: "x"
 id: "y"
 ---
 id: "z"
-`, "yaml", "[1].id", "1", `
-id: x
+`, "yaml", "[1].id", "1", `id: x
 ---
 id: "1"
 ---
@@ -552,8 +587,7 @@ id: z
 	t.Run("StringWithDotInName", putStringTest(`
 id: "asd"
 my.name: "Tom"
-`, "yaml", `my\.name`, "Jim", `
-id: asd
+`, "yaml", `my\.name`, "Jim", `id: asd
 my.name: Jim
 `, nil))
 
@@ -561,8 +595,7 @@ my.name: Jim
 metadata:
   annotations:
     node.longhorn.io/default-disks-config: '[ { "name":"fast",  "path":"/mnt/data-fast1", "allowScheduling":true, "tags":["fast"]}, { "name":"slow",  "path":"/mnt/data-slow1", "allowScheduling":true, "tags":["slow"]} ]'
-`, "yaml", `metadata.labels.node\.longhorn\.io\/create-default-disk`, "config", `
-metadata:
+`, "yaml", `metadata.labels.node\.longhorn\.io\/create-default-disk`, "config", `metadata:
   annotations:
     node.longhorn.io/default-disks-config: '[ { "name":"fast",  "path":"/mnt/data-fast1",
       "allowScheduling":true, "tags":["fast"]}, { "name":"slow",  "path":"/mnt/data-slow1",
@@ -575,18 +608,15 @@ metadata:
 func TestRootCMD_Put_TOML(t *testing.T) {
 	t.Run("String", putStringTest(`
 id = "x"
-`, "toml", "id", "y", `
-id = "y"
+`, "toml", "id", "y", `id = "y"
 `, nil))
 	t.Run("Int", putIntTest(`
 id = 123
-`, "toml", "id", "456", `
-id = 456
+`, "toml", "id", "456", `id = 456
 `, nil))
 	t.Run("Bool", putBoolTest(`
 id = true
-`, "toml", "id", "false", `
-id = false
+`, "toml", "id", "false", `id = false
 `, nil))
 	t.Run("OverwriteObject", putObjectTest(`
 [[numbers]]
diff --git a/internal/storage/json_test.go b/internal/storage/json_test.go
index 827cebb..ee7d1b0 100644
--- a/internal/storage/json_test.go
+++ b/internal/storage/json_test.go
@@ -15,23 +15,57 @@ var jsonMap = map[string]interface{}{
 }
 
 func TestJSONParser_FromBytes(t *testing.T) {
-	got, err := (&storage.JSONParser{}).FromBytes(jsonBytes)
-	if err != nil {
-		t.Errorf("unexpected error: %s", err)
-		return
-	}
-	if !reflect.DeepEqual(jsonMap, got) {
-		t.Errorf("expected %v, got %v", jsonMap, got)
-	}
+	t.Run("Valid", func(t *testing.T) {
+		got, err := (&storage.JSONParser{}).FromBytes(jsonBytes)
+		if err != nil {
+			t.Errorf("unexpected error: %s", err)
+			return
+		}
+		exp := &storage.JSONSingleDocument{Value: jsonMap}
+		if !reflect.DeepEqual(exp, got) {
+			t.Errorf("expected %v, got %v", exp, got)
+		}
+	})
+	t.Run("ValidMultiDocument", func(t *testing.T) {
+		got, err := (&storage.JSONParser{}).FromBytes(jsonBytesMulti)
+		if err != nil {
+			t.Errorf("unexpected error: %s", err)
+			return
+		}
+		exp := &storage.JSONMultiDocument{
+			Values: jsonMapMulti,
+		}
+		if !reflect.DeepEqual(exp, got) {
+			t.Errorf("expected %v, got %v", jsonMap, got)
+		}
+	})
+	t.Run("ValidMultiDocumentMixed", func(t *testing.T) {
+		got, err := (&storage.JSONParser{}).FromBytes(jsonBytesMultiMixed)
+		if err != nil {
+			t.Errorf("unexpected error: %s", err)
+			return
+		}
+		exp := &storage.JSONMultiDocument{
+			Values: jsonMapMultiMixed,
+		}
+		if !reflect.DeepEqual(exp, got) {
+			t.Errorf("expected %v, got %v", jsonMap, got)
+		}
+	})
+	t.Run("Empty", func(t *testing.T) {
+		got, err := (&storage.JSONParser{}).FromBytes([]byte(``))
+		if err != nil {
+			t.Errorf("unexpected error: %s", err)
+			return
+		}
+		if !reflect.DeepEqual(nil, got) {
+			t.Errorf("expected %v, got %v", nil, got)
+		}
+	})
 }
 
 func TestJSONParser_FromBytes_Error(t *testing.T) {
-	_, err := (&storage.JSONParser{}).FromBytes(nil)
-	if err == nil {
-		t.Errorf("expected error but got none")
-		return
-	}
-	_, err = (&storage.JSONParser{}).FromBytes(yamlBytes)
+	_, err := (&storage.JSONParser{}).FromBytes(yamlBytes)
 	if err == nil {
 		t.Errorf("expected error but got none")
 		return
@@ -39,12 +73,74 @@ func TestJSONParser_FromBytes_Error(t *testing.T) {
 }
 
 func TestJSONParser_ToBytes(t *testing.T) {
-	got, err := (&storage.JSONParser{}).ToBytes(jsonMap)
-	if err != nil {
-		t.Errorf("unexpected error: %s", err)
-		return
-	}
-	if !reflect.DeepEqual(jsonBytes, got) {
-		t.Errorf("expected %v, got %v", string(jsonBytes), string(got))
-	}
+	t.Run("Valid", func(t *testing.T) {
+		got, err := (&storage.JSONParser{}).ToBytes(jsonMap)
+		if err != nil {
+			t.Errorf("unexpected error: %s", err)
+			return
+		}
+		if string(jsonBytes) != string(got) {
+			t.Errorf("expected %v, got %v", string(jsonBytes), string(got))
+		}
+	})
+
+	t.Run("ValidSingle", func(t *testing.T) {
+		got, err := (&storage.JSONParser{}).ToBytes(&storage.JSONSingleDocument{Value: jsonMap})
+		if err != nil {
+			t.Errorf("unexpected error: %s", err)
+			return
+		}
+		if string(jsonBytes) != string(got) {
+			t.Errorf("expected %v, got %v", string(jsonBytes), string(got))
+		}
+	})
+
+	t.Run("ValidMulti", func(t *testing.T) {
+		got, err := (&storage.JSONParser{}).ToBytes(&storage.JSONMultiDocument{Values: jsonMapMulti})
+		if err != nil {
+			t.Errorf("unexpected error: %s", err)
+			return
+		}
+		if string(jsonBytesMulti) != string(got) {
+			t.Errorf("expected %v, got %v", string(jsonBytesMulti), string(got))
+		}
+	})
+
+	t.Run("ValidMultiMixed", func(t *testing.T) {
+		got, err := (&storage.JSONParser{}).ToBytes(&storage.JSONMultiDocument{Values: jsonMapMultiMixed})
+		if err != nil {
+			t.Errorf("unexpected error: %s", err)
+			return
+		}
+		if string(jsonBytesMultiMixed) != string(got) {
+			t.Errorf("expected %v, got %v", string(jsonBytesMultiMixed), string(got))
+		}
+	})
+}
+
+var jsonBytesMulti = []byte(`{
+  "name": "Tom"
+}
+{
+  "name": "Ellis"
+}
+`)
+
+var jsonMapMulti = []interface{}{
+	map[string]interface{}{"name": "Tom"},
+	map[string]interface{}{"name": "Ellis"},
+}
+
+var jsonBytesMultiMixed = []byte(`{
+  "name": "Tom",
+  "other": true
+}
+{
+  "name": "Ellis"
+}
+`)
+
+var jsonMapMultiMixed = []interface{}{
+	map[string]interface{}{"name": "Tom", "other": true},
+	map[string]interface{}{"name": "Ellis"},
 }
diff --git a/internal/storage/parser_test.go b/internal/storage/parser_test.go
index 37688a5..e5d070f 100644
--- a/internal/storage/parser_test.go
+++ b/internal/storage/parser_test.go
@@ -23,6 +23,7 @@ func TestNewParserFromString(t *testing.T) {
 	}{
 		{In: "json", Out: &storage.JSONParser{}},
 		{In: "yaml", Out: &storage.YAMLParser{}},
+		{In: "yml", Out: &storage.YAMLParser{}},
 		{In: "toml", Out: &storage.TOMLParser{}},
 		{In: "bad", Out: nil, Err: &storage.UnknownParserErr{Parser: "bad"}},
 	}
@@ -117,8 +118,9 @@ func TestLoadFromFile(t *testing.T) {
 			t.Errorf("unexpected error: %s", err)
 			return
 		}
-		if !reflect.DeepEqual(jsonData, data) {
-			t.Errorf("data does not match")
+		exp := &storage.JSONSingleDocument{Value: jsonData}
+		if !reflect.DeepEqual(exp, data) {
+			t.Errorf("data does not match: exp %v, got %v", exp, data)
 		}
 	})
 	t.Run("BaseFilePath", func(t *testing.T) {
diff --git a/internal/storage/yaml_test.go b/internal/storage/yaml_test.go
index e2a6152..db43c1c 100644
--- a/internal/storage/yaml_test.go
+++ b/internal/storage/yaml_test.go
@@ -13,6 +13,19 @@ var yamlMap = map[interface{}]interface{}{
 	"name": "Tom",
 }
 
+var yamlBytesMulti = []byte(`name: Tom
+---
+name: Jim
+`)
+var yamlMapMulti = []interface{}{
+	map[interface{}]interface{}{
+		"name": "Tom",
+	},
+	map[interface{}]interface{}{
+		"name": "Jim",
+	},
+}
+
 func TestYAMLParser_FromBytes(t *testing.T) {
 	t.Run("Valid", func(t *testing.T) {
 		got, err := (&storage.YAMLParser{}).FromBytes(yamlBytes)
@@ -26,23 +39,12 @@ func TestYAMLParser_FromBytes(t *testing.T) {
 		}
 	})
 	t.Run("ValidMultiDocument", func(t *testing.T) {
-		got, err := (&storage.YAMLParser{}).FromBytes([]byte(`
-name: Tom
----
-name: Jim
-`))
+		got, err := (&storage.YAMLParser{}).FromBytes(yamlBytesMulti)
 		if err != nil {
 			t.Errorf("unexpected error: %s", err)
 			return
 		}
-		exp := &storage.YAMLMultiDocument{Values: []interface{}{
-			map[interface{}]interface{}{
-				"name": "Tom",
-			},
-			map[interface{}]interface{}{
-				"name": "Jim",
-			},
-		}}
+		exp := &storage.YAMLMultiDocument{Values: yamlMapMulti}
 
 		if !reflect.DeepEqual(exp, got) {
 			t.Errorf("expected %v, got %v", exp, got)
@@ -68,12 +70,34 @@ name: Jim
 }
 
 func TestYAMLParser_ToBytes(t *testing.T) {
-	got, err := (&storage.YAMLParser{}).ToBytes(yamlMap)
-	if err != nil {
-		t.Errorf("unexpected error: %s", err)
-		return
-	}
-	if string(yamlBytes) != string(got) {
-		t.Errorf("expected %s, got %s", yamlBytes, got)
-	}
+	t.Run("Valid", func(t *testing.T) {
+		got, err := (&storage.YAMLParser{}).ToBytes(yamlMap)
+		if err != nil {
+			t.Errorf("unexpected error: %s", err)
+			return
+		}
+		if string(yamlBytes) != string(got) {
+			t.Errorf("expected %s, got %s", yamlBytes, got)
+		}
+	})
+	t.Run("ValidSingle", func(t *testing.T) {
+		got, err := (&storage.YAMLParser{}).ToBytes(&storage.YAMLSingleDocument{Value: yamlMap})
+		if err != nil {
+			t.Errorf("unexpected error: %s", err)
+			return
+		}
+		if string(yamlBytes) != string(got) {
+			t.Errorf("expected %s, got %s", yamlBytes, got)
+		}
+	})
+	t.Run("ValidMulti", func(t *testing.T) {
+		got, err := (&storage.YAMLParser{}).ToBytes(&storage.YAMLMultiDocument{Values: yamlMapMulti})
+		if err != nil {
+			t.Errorf("unexpected error: %s", err)
+			return
+		}
+		if string(yamlBytesMulti) != string(got) {
+			t.Errorf("expected %s, got %s", yamlBytesMulti, got)
+		}
+	})
 }