Repository: kcl-lang/kcl-go
Pull request #131 (2023-08-08 06:35:35)

feat: support more keywords when generating from json schema
#### 1. Does this PR affect any open issues?(Y/N) and add issue references 
- [ ] N
- [x] Y 

part of https://github.com/kcl-lang/kcl/issues/526

#### 2. What is the scope of this PR (e.g. component or file name):

pkg/tools/gen

#### 3. Provide a description of the PR(e.g. more details, effects, motivations or doc link):

<!-- You can choose a brief description here -->
- [ ] Affects user behaviors
- [ ] Contains syntax changes
- [ ] Contains variable changes
- [ ] Contains experimental features
- [ ] Performance regression: Consumes more CPU
- [ ] Performance regression: Consumes more Memory
- [x] Other

Support convert  `const`, `$ref`, `$defs` and `additionalProperties` keywords from json schema to kcl schema.
Add `getSortedKeys` function for more ordered and stable results.

#### 4. Are there any breaking changes?(Y/N) and describe the breaking changes(e.g. more details, motivations or doc link):

- [x] N
- [ ] Y 

<!-- You can add more details here.
    e.g. 
    Calling method "XXXX" will cause the "XXXX", "XXXX" modules to be affected.
    More details: https://XXXX.com/doc......
-->

#### 5. Are there test cases for these changes?(Y/N) select and add more details, references or doc links:

<!-- You can choose a brief description here -->
- [x] Unit test
- [ ] Integration test
- [ ] Benchmark (add benchmark stats below)
- [ ] Manual test (add detailed scripts or steps below)
- [ ] Other

<!-- You can add more details here.
e.g. 
The test case in XXXX is used to .....
test cases in /src/tests/XXXXX
test cases https://github.com/XXX/pull/44
benchmark stats: time XXX ms
-->


coveralls: ## Pull Request Test Coverage Report for [Build 5793598221](https://coveralls.io/builds/61842214)

* **58** of **62**   **(93.55%)**  changed or added relevant lines in **3** files are covered.
* No unchanged relevant lines lost coverage.
* Overall coverage increased (+**0.7%**) to **59.834%**

---

|  Changes Missing Coverage | Covered Lines | Changed/Added Lines | % |
| :-----|--------------|--------|---: |
| [pkg/tools/gen/genkcl_jsonschema.go](https://coveralls.io/builds/61842214/source?filename=pkg%2Ftools%2Fgen%2Fgenkcl_jsonschema.go#L167) | 43 | 47 | 91.49%
<!-- | **Total:** | **58** | **62** | **93.55%** | -->


|  Totals | [![Coverage Status](https://coveralls.io/builds/61842214/badge)](https://coveralls.io/builds/61842214) |
| :-- | --: |
| Change from base [Build 5781327883](https://coveralls.io/builds/61813532): |  0.7% |
| Covered Lines: | 2023 |
| Relevant Lines: | 3381 |

---
##### 💛  - [Coveralls](https://coveralls.io)


Patch:
diff --git a/pkg/tools/gen/genkcl_jsonschema.go b/pkg/tools/gen/genkcl_jsonschema.go
index e4981be..362d011 100644
--- a/pkg/tools/gen/genkcl_jsonschema.go
+++ b/pkg/tools/gen/genkcl_jsonschema.go
@@ -48,12 +48,12 @@ func (k *kclGenerator) genSchemaFromJsonSchema(w io.Writer, filename string, src
 		Imports: []string{},
 		Schemas: []schema{result.schema},
 	}
-	for imp := range ctx.imports {
+	for _, imp := range getSortedKeys(ctx.imports) {
 		kclSch.Imports = append(kclSch.Imports, imp)
 	}
-	for _, res := range ctx.resultMap {
-		if res.IsSchema {
-			kclSch.Schemas = append(kclSch.Schemas, res.schema)
+	for _, key := range getSortedKeys(ctx.resultMap) {
+		if ctx.resultMap[key].IsSchema {
+			kclSch.Schemas = append(kclSch.Schemas, ctx.resultMap[key].schema)
 		}
 	}
 
@@ -152,6 +152,41 @@ func convertSchemaFromJsonSchema(ctx convertContext, s *jsonschema.Schema, name
 					Value: unmarshalledVal,
 				})
 			}
+		case *jsonschema.Const:
+			unmarshalledVal := interface{}(nil)
+			err := json.Unmarshal(*v, &unmarshalledVal)
+			if err != nil {
+				logger.GetLogger().Warningf("failed to unmarshal const value: %s", err)
+				continue
+			}
+			typeList.Items = append(typeList.Items, typeValue{
+				Value: unmarshalledVal,
+			})
+			result.HasDefault = true
+			result.DefaultValue = unmarshalledVal
+		case *jsonschema.Ref:
+			typeName := strcase.ToCamel(v.Reference[strings.LastIndex(v.Reference, "/")+1:])
+			typeList.Items = append(typeList.Items, typeCustom{Name: typeName})
+		case *jsonschema.Defs:
+			for key, val := range *v {
+				ctx.resultMap[key] = convertSchemaFromJsonSchema(ctx, val, key)
+			}
+		case *jsonschema.AdditionalProperties:
+			switch v.SchemaType {
+			case jsonschema.SchemaTypeObject:
+				sch := convertSchemaFromJsonSchema(ctx, (*jsonschema.Schema)(v), "additionalProperties")
+				result.HasIndexSignature = true
+				result.IndexSignature = indexSignature{
+					Type: sch.Type,
+				}
+			case jsonschema.SchemaTypeTrue:
+				result.HasIndexSignature = true
+				result.IndexSignature = indexSignature{
+					Type: typePrimitive(typAny),
+				}
+			case jsonschema.SchemaTypeFalse:
+				result.HasIndexSignature = false
+			}
 		case *jsonschema.Minimum:
 			result.Validations = append(result.Validations, validation{
 				Minimum:          (*float64)(v),
diff --git a/pkg/tools/gen/template.go b/pkg/tools/gen/template.go
index 3f38b03..951cfdb 100644
--- a/pkg/tools/gen/template.go
+++ b/pkg/tools/gen/template.go
@@ -56,6 +56,15 @@ func formatValue(v interface{}) string {
 			return "True"
 		}
 		return "False"
+	case map[string]interface{}:
+		var s strings.Builder
+		for _, key := range getSortedKeys(value) {
+			if s.Len() != 0 {
+				s.WriteString(", ")
+			}
+			s.WriteString(fmt.Sprintf("%s: %s", formatValue(key), formatValue(value[key])))
+		}
+		return "{" + s.String() + "}"
 	default:
 		return fmt.Sprintf("%v", value)
 	}
diff --git a/pkg/tools/gen/templates/schema.gotmpl b/pkg/tools/gen/templates/schema.gotmpl
index bf6f343..b0e2795 100644
--- a/pkg/tools/gen/templates/schema.gotmpl
+++ b/pkg/tools/gen/templates/schema.gotmpl
@@ -8,6 +8,10 @@ schema {{ formatName .Name }}:
 	{{- range .Properties }}
     {{ formatName .Name }}{{ if not .Required }}?{{ end }}: {{ formatType .Type }}{{ if .HasDefault }} = {{ formatValue .DefaultValue }}{{ end }}
     {{- end }}
+    {{- if .HasIndexSignature }}{{- "\n    [" }}
+    	{{- if .IndexSignature.Alias }}{{ formatName .IndexSignature.Alias }}:{{ end }}
+    	{{- "...str]: " }}{{ formatType .IndexSignature.Type }}
+    {{- end }}
 {{ if .Validations }}
     check:
         {{- template "validator" .Validations }}
diff --git a/pkg/tools/gen/types.go b/pkg/tools/gen/types.go
index d4bbffd..537bad1 100644
--- a/pkg/tools/gen/types.go
+++ b/pkg/tools/gen/types.go
@@ -127,10 +127,12 @@ type kclSchema struct {
 
 // schema is a kcl schema definition.
 type schema struct {
-	Name        string
-	Description string
-	Properties  []property
-	Validations []validation
+	Name              string
+	Description       string
+	Properties        []property
+	Validations       []validation
+	HasIndexSignature bool
+	IndexSignature    indexSignature
 }
 
 // property is a kcl schema property definition.
@@ -154,6 +156,13 @@ type validation struct {
 	MaxLength        *int
 }
 
+// indexSignature is a kcl schema index signature definition.
+// It can be used to construct a dict with type.
+type indexSignature struct {
+	Alias string
+	Type  typeInterface
+}
+
 type typeInterface interface {
 	Format() string
 }
diff --git a/pkg/tools/gen/utils.go b/pkg/tools/gen/utils.go
index 0f5b135..0432d10 100644
--- a/pkg/tools/gen/utils.go
+++ b/pkg/tools/gen/utils.go
@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"sort"
 )
 
 var _ = assert
@@ -55,3 +56,13 @@ func readSource(filename string, src interface{}) (data []byte, err error) {
 		return nil, fmt.Errorf("unsupported src type: %T", src)
 	}
 }
+
+// getSortedKeys returns the keys sorted in alphabetical order of a map.
+func getSortedKeys[V any](m map[string]V) []string {
+	keys := make([]string, 0, len(m))
+	for k := range m {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	return keys
+}


Test patch:
diff --git a/pkg/tools/gen/testdata/jsonschema/additional/expect.k b/pkg/tools/gen/testdata/jsonschema/additional/expect.k
new file mode 100644
index 0000000..466b6dd
--- /dev/null
+++ b/pkg/tools/gen/testdata/jsonschema/additional/expect.k
@@ -0,0 +1,25 @@
+"""
+This file was generated by the KCL auto-gen tool. DO NOT EDIT.
+Editing this file might prove futile when you re-run the KCL auto-gen generate command.
+"""
+
+schema Ethernet:
+    """
+    Ethernet
+
+    Attributes
+    ----------
+    name : str, optional
+    socket : Socket, optional, default is {"HTTP": 80, "HTTPS": 443}
+    """
+
+    name?: str
+    socket?: Socket = {"HTTP": 80, "HTTPS": 443}
+
+schema Socket:
+    """
+    Socket
+    """
+
+    [...str]: int
+
diff --git a/pkg/tools/gen/testdata/jsonschema/additional/input.json b/pkg/tools/gen/testdata/jsonschema/additional/input.json
new file mode 100644
index 0000000..2465a0c
--- /dev/null
+++ b/pkg/tools/gen/testdata/jsonschema/additional/input.json
@@ -0,0 +1,20 @@
+{
+	"$schema": "http://json-schema.org/draft-07/schema#",
+	"$id": "https://example.com/schemas/ethernet.json",
+	"type": "object",
+	"properties": {
+		"name": {
+			"type": "string"
+		},
+		"socket": {
+			"type": "object",
+			"additionalProperties": {
+				"type": "integer"
+			},
+			"default": {
+				"HTTP": 80,
+				"HTTPS": 443
+			}
+		}
+	}
+}
diff --git a/pkg/tools/gen/testdata/jsonschema/const/expect.k b/pkg/tools/gen/testdata/jsonschema/const/expect.k
new file mode 100644
index 0000000..c534148
--- /dev/null
+++ b/pkg/tools/gen/testdata/jsonschema/const/expect.k
@@ -0,0 +1,20 @@
+"""
+This file was generated by the KCL auto-gen tool. DO NOT EDIT.
+Editing this file might prove futile when you re-run the KCL auto-gen generate command.
+"""
+
+schema Book:
+    """
+    Book
+
+    Attributes
+    ----------
+    title : str, required
+    authors : [str], optional
+    status : "available", optional, default is "available"
+    """
+
+    title: str
+    authors?: [str]
+    status?: "available" = "available"
+
diff --git a/pkg/tools/gen/testdata/jsonschema/const/input.json b/pkg/tools/gen/testdata/jsonschema/const/input.json
new file mode 100644
index 0000000..ecf5f7b
--- /dev/null
+++ b/pkg/tools/gen/testdata/jsonschema/const/input.json
@@ -0,0 +1,22 @@
+{
+	"$schema": "http://json-schema.org/draft-07/schema#",
+	"$id": "https://example.com/schemas/book.json",
+	"type": "object",
+	"properties": {
+		"title": {
+			"type": "string"
+		},
+		"authors": {
+			"type": "array",
+			"items": {
+				"type": "string"
+			}
+		},
+		"status": {
+			"const": "available"
+		}
+	},
+	"required": [
+		"title"
+	]
+}
diff --git a/pkg/tools/gen/testdata/jsonschema/ref/expect.k b/pkg/tools/gen/testdata/jsonschema/ref/expect.k
new file mode 100644
index 0000000..2361879
--- /dev/null
+++ b/pkg/tools/gen/testdata/jsonschema/ref/expect.k
@@ -0,0 +1,42 @@
+"""
+This file was generated by the KCL auto-gen tool. DO NOT EDIT.
+Editing this file might prove futile when you re-run the KCL auto-gen generate command.
+"""
+
+schema Customer:
+    """
+    Customer
+
+    Attributes
+    ----------
+    name : str, optional
+    address : Address, optional
+    """
+
+    name?: str
+    address?: Address
+
+schema Address:
+    """
+    Address
+
+    Attributes
+    ----------
+    city : str, optional
+    state : State, optional
+    """
+
+    city?: str
+    state?: State
+
+schema State:
+    """
+    State
+
+    Attributes
+    ----------
+    name : str, optional
+    """
+
+    name?: str
+
diff --git a/pkg/tools/gen/testdata/jsonschema/ref/input.json b/pkg/tools/gen/testdata/jsonschema/ref/input.json
new file mode 100644
index 0000000..7aaee36
--- /dev/null
+++ b/pkg/tools/gen/testdata/jsonschema/ref/input.json
@@ -0,0 +1,34 @@
+{
+	"$schema": "http://json-schema.org/draft-07/schema#",
+	"$id": "https://example.com/schemas/customer.json",
+	"type": "object",
+	"$defs": {
+		"address": {
+			"type": "object",
+			"properties": {
+				"city": {
+					"type": "string"
+				},
+				"state": {
+					"$ref": "#/$defs/state"
+				}
+			}
+		},
+		"state": {
+			"type": "object",
+			"properties": {
+				"name": {
+					"type": "string"
+				}
+			}
+		}
+	},
+	"properties": {
+		"name": {
+			"type": "string"
+		},
+		"address": {
+			"$ref": "#/$defs/address"
+		}
+	}
+}