Repository: kcl-lang/kcl-go
Pull request #145 (2023-08-29 07:58:16)

feat: generate kcl data from json
#### 1. Does this PR affect any open issues?(Y/N) and add issue references 

- [ ] N
- [x] Y 

part of https://github.com/kcl-lang/kcl/issues/526

#### 2. What is the scope of this PR (e.g. component or file name):

pkg/tools/gen

#### 3. Provide a description of the PR(e.g. more details, effects, motivations or doc link):

<!-- You can choose a brief description here -->
- [ ] Affects user behaviors
- [ ] Contains syntax changes
- [ ] Contains variable changes
- [ ] Contains experimental features
- [ ] Performance regression: Consumes more CPU
- [ ] Performance regression: Consumes more Memory
- [x] Other

Add the feature to generate kcl code from json. You can find an example in `testdata/json`. Additionally, I made the following changes: 
- Improve the auto mode selection to supports more modes and provides better accuracy.
- Change `kclSchema` struct to `kclFile`. It now contains all the imports, schemas and data in a kcl file. We can handle and render them using the same method `genKcl`
- Refactor `formatDoc` function to `indentLines` for better readability and reusability. Also add some template functions like `include` to help render the output.

#### 4. Are there any breaking changes?(Y/N) and describe the breaking changes(e.g. more details, motivations or doc link):

- [x] N
- [ ] Y 

#### 5. Are there test cases for these changes?(Y/N) select and add more details, references or doc links:

<!-- You can choose a brief description here -->
- [x] Unit test
- [ ] Integration test
- [ ] Benchmark (add benchmark stats below)
- [ ] Manual test (add detailed scripts or steps below)
- [ ] Other

<!-- You can add more details here.
e.g. 
The test case in XXXX is used to .....
test cases in /src/tests/XXXXX
test cases https://github.com/XXX/pull/44
benchmark stats: time XXX ms
-->


coveralls: ## Pull Request Test Coverage Report for [Build 6009495819](https://coveralls.io/builds/62290557)

* **69** of **81**   **(85.19%)**  changed or added relevant lines in **5** files are covered.
* No unchanged relevant lines lost coverage.
* Overall coverage increased (+**0.4%**) to **64.545%**

---

|  Changes Missing Coverage | Covered Lines | Changed/Added Lines | % |
| :-----|--------------|--------|---: |
| [pkg/tools/gen/template.go](https://coveralls.io/builds/62290557/source?filename=pkg%2Ftools%2Fgen%2Ftemplate.go#L49) | 25 | 27 | 92.59%
| [pkg/tools/gen/genkcl.go](https://coveralls.io/builds/62290557/source?filename=pkg%2Ftools%2Fgen%2Fgenkcl.go#L53) | 9 | 13 | 69.23%
| [pkg/tools/gen/genkcl_json.go](https://coveralls.io/builds/62290557/source?filename=pkg%2Ftools%2Fgen%2Fgenkcl_json.go#L12) | 31 | 37 | 83.78%
<!-- | **Total:** | **69** | **81** | **85.19%** | -->


|  Totals | [![Coverage Status](https://coveralls.io/builds/62290557/badge)](https://coveralls.io/builds/62290557) |
| :-- | --: |
| Change from base [Build 6000403335](https://coveralls.io/builds/62268843): |  0.4% |
| Covered Lines: | 2525 |
| Relevant Lines: | 3912 |

---
##### 💛  - [Coveralls](https://coveralls.io)


Patch:
diff --git a/pkg/tools/gen/genkcl.go b/pkg/tools/gen/genkcl.go
index c250755..d059b7d 100644
--- a/pkg/tools/gen/genkcl.go
+++ b/pkg/tools/gen/genkcl.go
@@ -1,6 +1,7 @@
 package gen
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -24,6 +25,7 @@ const (
 	ModeGoStruct
 	ModeJsonSchema
 	ModeTerraformSchema
+	ModeJson
 )
 
 type kclGenerator struct {
@@ -46,25 +48,26 @@ func newKclGenerator(opts *GenKclOptions) *kclGenerator {
 
 func (k *kclGenerator) GenSchema(w io.Writer, filename string, src interface{}) error {
 	if k.opts.Mode == ModeAuto {
+		code, err := readSource(filename, src)
+		if err != nil {
+			return err
+		}
+		codeStr := string(code)
+		var i interface{}
 		switch {
-		case strings.HasSuffix(filename, ".go"):
+		case strings.Contains(codeStr, "package "):
 			k.opts.Mode = ModeGoStruct
-		default:
-			code, err := readSource(filename, src)
-			if err != nil {
-				return err
-			}
-			codeStr := string(code)
+		case json.Unmarshal(code, &i) == nil:
 			switch {
-			case strings.Contains(codeStr, "package "):
-				k.opts.Mode = ModeGoStruct
 			case strings.Contains(codeStr, "$schema"):
 				k.opts.Mode = ModeJsonSchema
 			case strings.Contains(codeStr, "\"provider_schemas\""):
 				k.opts.Mode = ModeTerraformSchema
 			default:
-				return errors.New("failed to detect mode")
+				k.opts.Mode = ModeJson
 			}
+		default:
+			return errors.New("failed to detect mode")
 		}
 	}
 
@@ -75,6 +78,8 @@ func (k *kclGenerator) GenSchema(w io.Writer, filename string, src interface{})
 		return k.genSchemaFromJsonSchema(w, filename, src)
 	case ModeTerraformSchema:
 		return k.genSchemaFromTerraformSchema(w, filename, src)
+	case ModeJson:
+		return k.genKclFromJsonData(w, filename, src)
 	default:
 		return errors.New("unknown mode")
 	}
diff --git a/pkg/tools/gen/genkcl_json.go b/pkg/tools/gen/genkcl_json.go
new file mode 100644
index 0000000..ca5a310
--- /dev/null
+++ b/pkg/tools/gen/genkcl_json.go
@@ -0,0 +1,54 @@
+package gen
+
+import (
+	"encoding/json"
+	"io"
+	"sort"
+)
+
+func (k *kclGenerator) genKclFromJsonData(w io.Writer, filename string, src interface{}) error {
+	code, err := readSource(filename, src)
+	if err != nil {
+		return err
+	}
+	jsonData := &map[string]interface{}{}
+	if err = json.Unmarshal(code, jsonData); err != nil {
+		return err
+	}
+
+	// convert json data to kcl
+	result := convertKclFromJson(jsonData)
+
+	// generate kcl code
+	return k.genKcl(w, kclFile{Data: result})
+}
+
+func convertKclFromJson(jsonData *map[string]interface{}) []data {
+	var result []data
+	for key, value := range *jsonData {
+		switch value := value.(type) {
+		case map[string]interface{}:
+			result = append(result, data{
+				Key:   key,
+				Value: convertKclFromJson(&value),
+			})
+		case []interface{}:
+			var vals []interface{}
+			for _, v := range value {
+				switch v := v.(type) {
+				case map[string]interface{}:
+					vals = append(vals, convertKclFromJson(&v))
+				default:
+					vals = append(vals, v)
+				}
+			}
+			result = append(result, data{Key: key, Value: vals})
+		default:
+			result = append(result, data{Key: key, Value: value})
+		}
+	}
+	sort.Slice(result, func(i, j int) bool {
+		return result[i].Key < result[j].Key
+	})
+	return result
+}
diff --git a/pkg/tools/gen/genkcl_jsonschema.go b/pkg/tools/gen/genkcl_jsonschema.go
index d6704a5..be8e0e1 100644
--- a/pkg/tools/gen/genkcl_jsonschema.go
+++ b/pkg/tools/gen/genkcl_jsonschema.go
@@ -48,7 +48,7 @@ func (k *kclGenerator) genSchemaFromJsonSchema(w io.Writer, filename string, src
 	if !result.IsSchema {
 		panic("result is not schema")
 	}
-	kclSch := kclSchema{
+	kclSch := kclFile{
 		Imports: []string{},
 		Schemas: []schema{result.schema},
 	}
@@ -62,7 +62,7 @@ func (k *kclGenerator) genSchemaFromJsonSchema(w io.Writer, filename string, src
 	}
 
 	// generate kcl schema code
-	return k.genKclSchema(w, kclSch)
+	return k.genKcl(w, kclSch)
 }
 
 func convertSchemaFromJsonSchema(ctx convertContext, s *jsonschema.Schema, name string) convertResult {
diff --git a/pkg/tools/gen/genkcl_terraform.go b/pkg/tools/gen/genkcl_terraform.go
index 5f180d0..afd5085 100644
--- a/pkg/tools/gen/genkcl_terraform.go
+++ b/pkg/tools/gen/genkcl_terraform.go
@@ -86,11 +86,11 @@ func (k *kclGenerator) genSchemaFromTerraformSchema(w io.Writer, filename string
 	})
 
 	// generate kcl schema code
-	kclSch := kclSchema{
+	kclSch := kclFile{
 		Imports: []string{},
 		Schemas: result,
 	}
-	return k.genKclSchema(w, kclSch)
+	return k.genKcl(w, kclSch)
 }
 
 func tfTypeToKclType(t interface{}) typeInterface {
diff --git a/pkg/tools/gen/template.go b/pkg/tools/gen/template.go
index c58b34e..1b51e97 100644
--- a/pkg/tools/gen/template.go
+++ b/pkg/tools/gen/template.go
@@ -1,6 +1,7 @@
 package gen
 
 import (
+	"bytes"
 	_ "embed"
 	"fmt"
 	"io"
@@ -9,6 +10,8 @@ import (
 )
 
 var (
+	//go:embed templates/kcl/data.gotmpl
+	dataTmpl string
 	//go:embed templates/kcl/document.gotmpl
 	documentTmpl string
 	//go:embed templates/kcl/header.gotmpl
@@ -17,21 +20,43 @@ var (
 	validatorTmpl string
 	//go:embed templates/kcl/schema.gotmpl
 	schemaTmpl string
+	//go:embed templates/kcl/index.gotmpl
+	indexTmpl string
 )
 
 var funcs = template.FuncMap{
 	"formatType":  formatType,
 	"formatValue": formatValue,
 	"formatName":  formatName,
-	"formatDoc":   formatDoc,
+	"indentLines": indentLines,
+	"isKclData": func(v interface{}) bool {
+		_, ok := v.([]data)
+		return ok
+	},
+	"isArray": func(v interface{}) bool {
+		_, ok := v.([]interface{})
+		return ok
+	},
 }
 
-func (k *kclGenerator) genKclSchema(w io.Writer, s kclSchema) error {
+func (k *kclGenerator) genKcl(w io.Writer, s kclFile) error {
 	tmpl := &template.Template{}
+
+	// add "include" function. It works like "template" but can be used in pipeline.
+	funcs["include"] = func(name string, data interface{}) (string, error) {
+		buf := bytes.NewBuffer(nil)
+		if err := tmpl.ExecuteTemplate(buf, name, data); err != nil {
+			return "", err
+		}
+		return buf.String(), nil
+	}
+
+	tmpl = addTemplate(tmpl, "data", dataTmpl)
 	tmpl = addTemplate(tmpl, "document", documentTmpl)
 	tmpl = addTemplate(tmpl, "header", headerTmpl)
 	tmpl = addTemplate(tmpl, "validator", validatorTmpl)
 	tmpl = addTemplate(tmpl, "schema", schemaTmpl)
+	tmpl = addTemplate(tmpl, "index", indexTmpl)
 	return tmpl.Funcs(funcs).Execute(w, s)
 }
 
@@ -106,7 +131,12 @@ func formatName(name string) string {
 	return name
 }
 
-func formatDoc(doc, indent string) string {
-	doc = strings.Replace(doc, "\r\n", "\n", -1)
-	return indent + strings.Replace(doc, "\n", "\n"+indent, -1)
+func indentLines(s, indent string) string {
+	s = strings.Replace(s, "\r\n", "\n", -1)
+	n := strings.Count(s, "\n")
+	if s[len(s)-1] == '\n' {
+		// ignore last empty line
+		n--
+	}
+	return indent + strings.Replace(s, "\n", "\n"+indent, n)
 }
diff --git a/pkg/tools/gen/templates/kcl/data.gotmpl b/pkg/tools/gen/templates/kcl/data.gotmpl
new file mode 100644
index 0000000..914429c
--- /dev/null
+++ b/pkg/tools/gen/templates/kcl/data.gotmpl
@@ -0,0 +1,24 @@
+{{ formatName .Key }}{{ " = " }}
+    {{- if isKclData .Value }}
+        {{- "{\n" }}
+            {{- range .Value -}}
+            {{- indentLines (include "data" .) "    " }}
+            {{- end }}
+        {{- "}" }}
+    {{- else if isArray .Value }}
+        {{- "[\n" }}
+            {{- range .Value -}}
+                {{- if isKclData . }}
+                    {{- "    {\n" }}
+                        {{- range . -}}
+                        {{- indentLines (include "data" .) "        " }}
+                        {{- end }}
+                    {{- "    }\n" }}
+                {{- else }}
+                    {{- indentLines (formatValue .) "    " }}
+                {{- end }}
+            {{- end }}
+        {{- "]" }}
+    {{- else }}
+    {{- formatValue .Value }}
+    {{- end }}
diff --git a/pkg/tools/gen/templates/kcl/document.gotmpl b/pkg/tools/gen/templates/kcl/document.gotmpl
index 23441ee..f9b44b9 100644
--- a/pkg/tools/gen/templates/kcl/document.gotmpl
+++ b/pkg/tools/gen/templates/kcl/document.gotmpl
@@ -1,7 +1,7 @@
 {{- if .Description }}
-    {{- formatDoc .Description "    " }}
+    {{- indentLines .Description "    " }}
 {{- else }}
-    {{- formatDoc .Name "    " }}
+    {{- indentLines .Name "    " }}
 {{- end }}
 
 {{- if .Properties }}
@@ -11,7 +11,7 @@
   {{- range .Properties }}
     {{ formatName .Name }} : {{ formatType .Type }}, {{ if .Required }}required{{ else }}optional{{ end }}
     {{- if .HasDefault }}, default is {{ formatValue .DefaultValue }}{{ end }}
-    {{- if .Description }}{{ "\n" }}{{ formatDoc .Description "        " }}{{ end }}
+    {{- if .Description }}{{ "\n" }}{{ indentLines .Description "        " }}{{ end }}
   {{- end -}}
 
 {{- end -}}
diff --git a/pkg/tools/gen/templates/kcl/header.gotmpl b/pkg/tools/gen/templates/kcl/header.gotmpl
index ed9334b..74824ee 100644
--- a/pkg/tools/gen/templates/kcl/header.gotmpl
+++ b/pkg/tools/gen/templates/kcl/header.gotmpl
@@ -7,4 +7,4 @@ Editing this file might prove futile when you re-run the KCL auto-gen generate c
 {{- range .Imports }}
 import {{ . }}
 {{- end }}
-{{- end }}
+{{- end -}}
diff --git a/pkg/tools/gen/templates/kcl/index.gotmpl b/pkg/tools/gen/templates/kcl/index.gotmpl
new file mode 100644
index 0000000..bb31e1d
--- /dev/null
+++ b/pkg/tools/gen/templates/kcl/index.gotmpl
@@ -0,0 +1,9 @@
+{{ template "header" . }}
+
+{{ range .Schemas -}}
+{{ template "schema" . }}
+{{ end -}}
+
+{{ range .Data -}}
+{{ template "data" . -}}
+{{ end -}}
diff --git a/pkg/tools/gen/templates/kcl/schema.gotmpl b/pkg/tools/gen/templates/kcl/schema.gotmpl
index b0e2795..bd92cde 100644
--- a/pkg/tools/gen/templates/kcl/schema.gotmpl
+++ b/pkg/tools/gen/templates/kcl/schema.gotmpl
@@ -1,5 +1,3 @@
-{{ template "header" . }}
-{{ range .Schemas -}}
 schema {{ formatName .Name }}:
     """{{- "\n" }}
     {{- template "document" . }}
@@ -15,5 +13,4 @@ schema {{ formatName .Name }}:
 {{ if .Validations }}
     check:
         {{- template "validator" .Validations }}
-{{- end }}
-{{ end -}}
+{{- end -}}
diff --git a/pkg/tools/gen/types.go b/pkg/tools/gen/types.go
index 93545b0..cc68e7e 100644
--- a/pkg/tools/gen/types.go
+++ b/pkg/tools/gen/types.go
@@ -121,11 +121,12 @@ func getSortedFieldNames(fields map[string]*pb.KclType) []string {
 	return ss
 }
 
-// kclSchema is the top-level structure for a kcl schema file.
-// It contains all the imports and schemas in this file.
-type kclSchema struct {
+// kclFile is the top-level structure for a kcl file.
+// It contains all the imports, schemas and data in this file.
+type kclFile struct {
 	Imports []string
 	Schemas []schema
+	Data    []data
 }
 
 // schema is a kcl schema definition.
@@ -169,6 +170,12 @@ type indexSignature struct {
 	Type  typeInterface
 }
 
+// data is a kcl data definition.
+type data struct {
+	Key   string
+	Value interface{}
+}
+
 type typeInterface interface {
 	Format() string
 }


Test patch:
diff --git a/pkg/tools/gen/genkcl_test.go b/pkg/tools/gen/genkcl_test.go
index 2c875ae..73a44f5 100644
--- a/pkg/tools/gen/genkcl_test.go
+++ b/pkg/tools/gen/genkcl_test.go
@@ -115,7 +115,7 @@ schema Company:
 
 }
 
-func TestGenKclFromJson(t *testing.T) {
+func TestGenKclFromJsonSchema(t *testing.T) {
 	type testCase struct {
 		name   string
 		input  string
@@ -166,6 +166,20 @@ func TestGenKclFromTerraform(t *testing.T) {
 	assert2.Equal(t, expect, string(bytes.ReplaceAll(result, []byte("\r\n"), []byte("\n"))))
 }
 
+func TestGenKclFromJson(t *testing.T) {
+	input := filepath.Join("testdata", "json", "input.json")
+	expectFilepath := filepath.Join("testdata", "json", "expect.k")
+	expect := readFileString(t, expectFilepath)
+
+	var buf bytes.Buffer
+	err := GenKcl(&buf, input, nil, &GenKclOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	result := buf.Bytes()
+	assert2.Equal(t, expect, string(bytes.ReplaceAll(result, []byte("\r\n"), []byte("\n"))))
+}
+
 func readFileString(t testing.TB, p string) (content string) {
 	data, err := os.ReadFile(p)
 	if err != nil {
diff --git a/pkg/tools/gen/testdata/json/expect.k b/pkg/tools/gen/testdata/json/expect.k
new file mode 100644
index 0000000..090b16b
--- /dev/null
+++ b/pkg/tools/gen/testdata/json/expect.k
@@ -0,0 +1,41 @@
+"""
+This file was generated by the KCL auto-gen tool. DO NOT EDIT.
+Editing this file might prove futile when you re-run the KCL auto-gen generate command.
+"""
+
+apiVersion = "apps/v1"
+kind = "Deployment"
+metadata = {
+    labels = {
+        app = "nginx"
+    }
+    name = "nginx-deployment"
+}
+spec = {
+    replicas = 3
+    selector = {
+        matchLabels = {
+            app = "nginx"
+        }
+    }
+    template = {
+        metadata = {
+            labels = {
+                app = "nginx"
+            }
+        }
+        spec = {
+            containers = [
+                {
+                    image = "nginx:latest"
+                    name = "nginx-container"
+                    ports = [
+                        {
+                            containerPort = 80
+                        }
+                    ]
+                }
+            ]
+        }
+    }
+}
diff --git a/pkg/tools/gen/testdata/json/input.json b/pkg/tools/gen/testdata/json/input.json
new file mode 100644
index 0000000..7875427
--- /dev/null
+++ b/pkg/tools/gen/testdata/json/input.json
@@ -0,0 +1,38 @@
+{
+	"apiVersion": "apps/v1",
+	"kind": "Deployment",
+	"metadata": {
+		"name": "nginx-deployment",
+		"labels": {
+			"app": "nginx"
+		}
+	},
+	"spec": {
+		"replicas": 3,
+		"selector": {
+			"matchLabels": {
+				"app": "nginx"
+			}
+		},
+		"template": {
+			"metadata": {
+				"labels": {
+					"app": "nginx"
+				}
+			},
+			"spec": {
+				"containers": [
+					{
+						"name": "nginx-container",
+						"image": "nginx:latest",
+						"ports": [
+							{
+								"containerPort": 80
+							}
+						]
+					}
+				]
+			}
+		}
+	}
+}