Repository: getkin/kin-openapi
Pull request #443 (2021-11-04 03:59:11)

Internalize references
I needed this functionality for some tooling I'm building and thought I would make a pull request in-case anyone would find it useful. If so I can create a few tests too.

All this does is either inline or add to the components section all external references, so that when you write the file back out it is self contained and portable. This makes it easier for other tools to process and removes all dependencies.

If there's a better way of doing this let me know and I will update my code!

fenollp: Please add tests, specifically one with recursive references.
jhwz: Should be all there now. I spent some time and made the implementation complete.

I have tests on an array of files found in the testdata directory.

Cheers!
fenollp: Ah you'll need your tests to compile for Go 1.14 please (that `os.WriteFile` call)
Also please have these tests define `err` as in
```go
err := os.WriteFile("__internalized.openapi.json", data, 0644)
require.NoError(t, err, "writing internalized spec to new context")
```
instead of:
```go
require.NoError(t, os.WriteFile("__internalized.openapi.json", data, 0644), "writing internalized spec to new context")
```

And finally, no need to write to file, just use `LoadFromData` or `LoadFromDataWithPath`
https://pkg.go.dev/github.com/getkin/kin-openapi@v0.80.0/openapi3#Loader.LoadFromData
jhwz: Ah right my mistake.

All good now, there is one issue with the recursive references though:
```
              "components": {
                "schemas": {
                  "Bar": {
                    "$ref": "#/components/schemas/Bar"
                  },
                  "Foo": {
                    "$ref": "#/components/schemas/Foo"
                  },
                  "Foo2": {
                    "$ref": "#/components/schemas/Foo2"
                  }
                }
              },
              "info": {
                "title": "Recursive refs example",
                "version": "1.0"
              },
              "openapi": "3.0.3",
              "paths": {
                "/foo": {
                  "get": {
                    "responses": {
                      "200": {
                        "content": {
                          "application/json": {
                            "schema": {
                              "properties": {
                                "foo2": {
                                  "$ref": "#/components/schemas/Foo2"
                                }
                              },
                              "type": "object"
                            }
                          }
                        },
                        "description": "OK"
                      }
                    }
                  }
                }
              }
            }
```

Which is produced by the function on the recursiveRefs schema. I'm not sure you would consider this schema valid, but maybe. The issue is it is not caught by [this](https://github.com/getkin/kin-openapi/blob/e9b36dac1a2890474ae44cfcf6446c00393527b5/openapi3/loader.go#L688-L696) because the value is always nil.

Not sure how you want to deal with that..
fenollp: Hm this is not how that document should be "internalized". Bar's schema should be 
```yaml
type: string
example: bar
```
I think you're right in pointing out the "visited" clause is ineffectual but I don't see how `component.Value` is `nil`. 
Can you help fix this?

Patch:
diff --git a/openapi3/internalize_refs.go b/openapi3/internalize_refs.go
new file mode 100644
index 0000000..3a993bf
--- /dev/null
+++ b/openapi3/internalize_refs.go
@@ -0,0 +1,369 @@
+package openapi3
+
+import (
+	"context"
+	"path/filepath"
+	"strings"
+)
+
+type RefNameResolver func(string) string
+
+// DefaultRefResolver is a default implementation of refNameResolver for the
+// InternalizeRefs function.
+//
+// If a reference points to an element inside a document, it returns the last
+// element in the reference using filepath.Base. Otherwise if the reference points
+// to a file, it returns the file name trimmed of all extensions.
+func DefaultRefNameResolver(ref string) string {
+	if ref == "" {
+		return ""
+	}
+	split := strings.SplitN(ref, "#", 2)
+	if len(split) == 2 {
+		return filepath.Base(split[1])
+	}
+	ref = split[0]
+	for ext := filepath.Ext(ref); len(ext) > 0; ext = filepath.Ext(ref) {
+		ref = strings.TrimSuffix(ref, ext)
+	}
+	return filepath.Base(ref)
+}
+
+func schemaNames(s Schemas) []string {
+	out := make([]string, 0, len(s))
+	for i := range s {
+		out = append(out, i)
+	}
+	return out
+}
+
+func parametersMapNames(s ParametersMap) []string {
+	out := make([]string, 0, len(s))
+	for i := range s {
+		out = append(out, i)
+	}
+	return out
+}
+
+func isExternalRef(ref string) bool {
+	return ref != "" && !strings.HasPrefix(ref, "#/components/")
+}
+
+func (doc *T) addSchemaToSpec(s *SchemaRef, refNameResolver RefNameResolver) {
+	if s == nil || !isExternalRef(s.Ref) {
+		return
+	}
+
+	name := refNameResolver(s.Ref)
+	if _, ok := doc.Components.Schemas[name]; ok {
+		s.Ref = "#/components/schemas/" + name
+		return
+	}
+
+	if doc.Components.Schemas == nil {
+		doc.Components.Schemas = make(Schemas)
+	}
+	doc.Components.Schemas[name] = s.Value.NewRef()
+	s.Ref = "#/components/schemas/" + name
+}
+
+func (doc *T) addParameterToSpec(p *ParameterRef, refNameResolver RefNameResolver) {
+	if p == nil || !isExternalRef(p.Ref) {
+		return
+	}
+	name := refNameResolver(p.Ref)
+	if _, ok := doc.Components.Parameters[name]; ok {
+		p.Ref = "#/components/parameters/" + name
+		return
+	}
+
+	if doc.Components.Parameters == nil {
+		doc.Components.Parameters = make(ParametersMap)
+	}
+	doc.Components.Parameters[name] = &ParameterRef{Value: p.Value}
+	p.Ref = "#/components/parameters/" + name
+}
+
+func (doc *T) addHeaderToSpec(h *HeaderRef, refNameResolver RefNameResolver) {
+	if h == nil || !isExternalRef(h.Ref) {
+		return
+	}
+	name := refNameResolver(h.Ref)
+	if _, ok := doc.Components.Headers[name]; ok {
+		h.Ref = "#/components/headers/" + name
+		return
+	}
+	if doc.Components.Headers == nil {
+		doc.Components.Headers = make(Headers)
+	}
+	doc.Components.Headers[name] = &HeaderRef{Value: h.Value}
+	h.Ref = "#/components/headers/" + name
+}
+
+func (doc *T) addRequestBodyToSpec(r *RequestBodyRef, refNameResolver RefNameResolver) {
+	if r == nil || !isExternalRef(r.Ref) {
+		return
+	}
+	name := refNameResolver(r.Ref)
+	if _, ok := doc.Components.RequestBodies[name]; ok {
+		r.Ref = "#/components/requestBodies/" + name
+		return
+	}
+	if doc.Components.RequestBodies == nil {
+		doc.Components.RequestBodies = make(RequestBodies)
+	}
+	doc.Components.RequestBodies[name] = &RequestBodyRef{Value: r.Value}
+	r.Ref = "#/components/requestBodies/" + name
+}
+
+func (doc *T) addResponseToSpec(r *ResponseRef, refNameResolver RefNameResolver) {
+	if r == nil || !isExternalRef(r.Ref) {
+		return
+	}
+	name := refNameResolver(r.Ref)
+	if _, ok := doc.Components.Responses[name]; ok {
+		r.Ref = "#/components/responses/" + name
+		return
+	}
+	if doc.Components.Responses == nil {
+		doc.Components.Responses = make(Responses)
+	}
+	doc.Components.Responses[name] = &ResponseRef{Value: r.Value}
+	r.Ref = "#/components/responses/" + name
+
+}
+
+func (doc *T) addSecuritySchemeToSpec(ss *SecuritySchemeRef, refNameResolver RefNameResolver) {
+	if ss == nil || !isExternalRef(ss.Ref) {
+		return
+	}
+	name := refNameResolver(ss.Ref)
+	if _, ok := doc.Components.SecuritySchemes[name]; ok {
+		ss.Ref = "#/components/securitySchemes/" + name
+		return
+	}
+	if doc.Components.SecuritySchemes == nil {
+		doc.Components.SecuritySchemes = make(SecuritySchemes)
+	}
+	doc.Components.SecuritySchemes[name] = &SecuritySchemeRef{Value: ss.Value}
+	ss.Ref = "#/components/securitySchemes/" + name
+
+}
+
+func (doc *T) addExampleToSpec(e *ExampleRef, refNameResolver RefNameResolver) {
+	if e == nil || !isExternalRef(e.Ref) {
+		return
+	}
+	name := refNameResolver(e.Ref)
+	if _, ok := doc.Components.Examples[name]; ok {
+		e.Ref = "#/components/examples/" + name
+		return
+	}
+	if doc.Components.Examples == nil {
+		doc.Components.Examples = make(Examples)
+	}
+	doc.Components.Examples[name] = &ExampleRef{Value: e.Value}
+	e.Ref = "#/components/examples/" + name
+
+}
+
+func (doc *T) addLinkToSpec(l *LinkRef, refNameResolver RefNameResolver) {
+	if l == nil || !isExternalRef(l.Ref) {
+		return
+	}
+	name := refNameResolver(l.Ref)
+	if _, ok := doc.Components.Links[name]; ok {
+		l.Ref = "#/components/links/" + name
+		return
+	}
+	if doc.Components.Links == nil {
+		doc.Components.Links = make(Links)
+	}
+	doc.Components.Links[name] = &LinkRef{Value: l.Value}
+	l.Ref = "#/components/links/" + name
+
+}
+
+func (doc *T) addCallbackToSpec(c *CallbackRef, refNameResolver RefNameResolver) {
+	if c == nil || !isExternalRef(c.Ref) {
+		return
+	}
+	name := refNameResolver(c.Ref)
+	if _, ok := doc.Components.Callbacks[name]; ok {
+		c.Ref = "#/components/callbacks/" + name
+	}
+	if doc.Components.Callbacks == nil {
+		doc.Components.Callbacks = make(Callbacks)
+	}
+	doc.Components.Callbacks[name] = &CallbackRef{Value: c.Value}
+	c.Ref = "#/components/callbacks/" + name
+}
+
+func (doc *T) derefSchema(s *Schema, refNameResolver RefNameResolver) {
+	if s == nil {
+		return
+	}
+
+	for _, list := range []SchemaRefs{s.AllOf, s.AnyOf, s.OneOf} {
+		for _, s2 := range list {
+			doc.addSchemaToSpec(s2, refNameResolver)
+			if s2 != nil {
+				doc.derefSchema(s2.Value, refNameResolver)
+			}
+		}
+	}
+	for _, s2 := range s.Properties {
+		doc.addSchemaToSpec(s2, refNameResolver)
+		if s2 != nil {
+			doc.derefSchema(s2.Value, refNameResolver)
+		}
+	}
+	for _, ref := range []*SchemaRef{s.Not, s.AdditionalProperties, s.Items} {
+		doc.addSchemaToSpec(ref, refNameResolver)
+		if ref != nil {
+			doc.derefSchema(ref.Value, refNameResolver)
+		}
+	}
+}
+
+func (doc *T) derefHeaders(hs Headers, refNameResolver RefNameResolver) {
+	for _, h := range hs {
+		doc.addHeaderToSpec(h, refNameResolver)
+		doc.derefParameter(h.Value.Parameter, refNameResolver)
+	}
+}
+
+func (doc *T) derefExamples(es Examples, refNameResolver RefNameResolver) {
+	for _, e := range es {
+		doc.addExampleToSpec(e, refNameResolver)
+	}
+}
+
+func (doc *T) derefContent(c Content, refNameResolver RefNameResolver) {
+	for _, mediatype := range c {
+		doc.addSchemaToSpec(mediatype.Schema, refNameResolver)
+		if mediatype.Schema != nil {
+			doc.derefSchema(mediatype.Schema.Value, refNameResolver)
+		}
+		doc.derefExamples(mediatype.Examples, refNameResolver)
+		for _, e := range mediatype.Encoding {
+			doc.derefHeaders(e.Headers, refNameResolver)
+		}
+	}
+}
+
+func (doc *T) derefLinks(ls Links, refNameResolver RefNameResolver) {
+	for _, l := range ls {
+		doc.addLinkToSpec(l, refNameResolver)
+	}
+}
+
+func (doc *T) derefResponses(es Responses, refNameResolver RefNameResolver) {
+	for _, e := range es {
+		doc.addResponseToSpec(e, refNameResolver)
+		if e.Value != nil {
+			doc.derefHeaders(e.Value.Headers, refNameResolver)
+			doc.derefContent(e.Value.Content, refNameResolver)
+			doc.derefLinks(e.Value.Links, refNameResolver)
+		}
+	}
+}
+
+func (doc *T) derefParameter(p Parameter, refNameResolver RefNameResolver) {
+	doc.addSchemaToSpec(p.Schema, refNameResolver)
+	doc.derefContent(p.Content, refNameResolver)
+	if p.Schema != nil {
+		doc.derefSchema(p.Schema.Value, refNameResolver)
+	}
+}
+
+func (doc *T) derefRequestBody(r RequestBody, refNameResolver RefNameResolver) {
+	doc.derefContent(r.Content, refNameResolver)
+}
+
+func (doc *T) derefPaths(paths map[string]*PathItem, refNameResolver RefNameResolver) {
+	for _, ops := range paths {
+		// inline full operations
+		ops.Ref = ""
+
+		for _, op := range ops.Operations() {
+			doc.addRequestBodyToSpec(op.RequestBody, refNameResolver)
+			if op.RequestBody != nil && op.RequestBody.Value != nil {
+				doc.derefRequestBody(*op.RequestBody.Value, refNameResolver)
+			}
+			for _, cb := range op.Callbacks {
+				doc.addCallbackToSpec(cb, refNameResolver)
+				if cb.Value != nil {
+					doc.derefPaths(*cb.Value, refNameResolver)
+				}
+			}
+			doc.derefResponses(op.Responses, refNameResolver)
+			for _, param := range op.Parameters {
+				doc.addParameterToSpec(param, refNameResolver)
+				if param.Value != nil {
+					doc.derefParameter(*param.Value, refNameResolver)
+				}
+			}
+		}
+	}
+}
+
+// InternalizeRefs removes all references to external files from the spec and moves them
+// to the components section.
+//
+// refNameResolver takes in references to returns a name to store the reference under locally.
+// It MUST return a unique name for each reference type.
+// A default implementation is provided that will suffice for most use cases. See the function
+// documention for more details.
+//
+// Example:
+//
+//   doc.InternalizeRefs(context.Background(), nil)
+func (doc *T) InternalizeRefs(ctx context.Context, refNameResolver func(ref string) string) {
+	if refNameResolver == nil {
+		refNameResolver = DefaultRefNameResolver
+	}
+
+	// Handle components section
+	names := schemaNames(doc.Components.Schemas)
+	for _, name := range names {
+		schema := doc.Components.Schemas[name]
+		doc.addSchemaToSpec(schema, refNameResolver)
+		if schema != nil {
+			schema.Ref = "" // always dereference the top level
+			doc.derefSchema(schema.Value, refNameResolver)
+		}
+	}
+	names = parametersMapNames(doc.Components.Parameters)
+	for _, name := range names {
+		p := doc.Components.Parameters[name]
+		doc.addParameterToSpec(p, refNameResolver)
+		if p != nil && p.Value != nil {
+			p.Ref = "" // always dereference the top level
+			doc.derefParameter(*p.Value, refNameResolver)
+		}
+	}
+	doc.derefHeaders(doc.Components.Headers, refNameResolver)
+	for _, req := range doc.Components.RequestBodies {
+		doc.addRequestBodyToSpec(req, refNameResolver)
+		if req != nil && req.Value != nil {
+			req.Ref = "" // always dereference the top level
+			doc.derefRequestBody(*req.Value, refNameResolver)
+		}
+	}
+	doc.derefResponses(doc.Components.Responses, refNameResolver)
+	for _, ss := range doc.Components.SecuritySchemes {
+		doc.addSecuritySchemeToSpec(ss, refNameResolver)
+	}
+	doc.derefExamples(doc.Components.Examples, refNameResolver)
+	doc.derefLinks(doc.Components.Links, refNameResolver)
+	for _, cb := range doc.Components.Callbacks {
+		doc.addCallbackToSpec(cb, refNameResolver)
+		if cb != nil && cb.Value != nil {
+			cb.Ref = "" // always dereference the top level
+			doc.derefPaths(*cb.Value, refNameResolver)
+		}
+	}
+
+	doc.derefPaths(doc.Paths, refNameResolver)
+}
diff --git a/openapi3/schema.go b/openapi3/schema.go
index 443f719..a079698 100644
--- a/openapi3/schema.go
+++ b/openapi3/schema.go
@@ -800,13 +800,22 @@ func (schema *Schema) visitJSON(settings *schemaValidationSettings, value interf
 		return schema.visitJSONArray(settings, value)
 	case map[string]interface{}:
 		return schema.visitJSONObject(settings, value)
-	default:
-		return &SchemaError{
-			Value:       value,
-			Schema:      schema,
-			SchemaField: "type",
-			Reason:      fmt.Sprintf("unhandled value of type %T", value),
+	case map[interface{}]interface{}: // for YAML cf. issue #444
+		values := make(map[string]interface{}, len(value))
+		for key, v := range value {
+			if k, ok := key.(string); ok {
+				values[k] = v
+			}
 		}
+		if len(value) == len(values) {
+			return schema.visitJSONObject(settings, values)
+		}
+	}
+	return &SchemaError{
+		Value:       value,
+		Schema:      schema,
+		SchemaField: "type",
+		Reason:      fmt.Sprintf("unhandled value of type %T", value),
 	}
 }
 


Test patch:
diff --git a/openapi3/internalize_refs_test.go b/openapi3/internalize_refs_test.go
new file mode 100644
index 0000000..d6264d4
--- /dev/null
+++ b/openapi3/internalize_refs_test.go
@@ -0,0 +1,55 @@
+package openapi3
+
+import (
+	"context"
+	"regexp"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestInternalizeRefs(t *testing.T) {
+	var regexpRef = regexp.MustCompile(`"\$ref":`)
+	var regexpRefInternal = regexp.MustCompile(`"\$ref":"\#`)
+
+	tests := []struct {
+		filename string
+	}{
+		{"testdata/testref.openapi.yml"},
+		{"testdata/recursiveRef/openapi.yml"},
+		{"testdata/spec.yaml"},
+		{"testdata/callbacks.yml"},
+	}
+
+	for _, test := range tests {
+		t.Run(test.filename, func(t *testing.T) {
+			// Load in the reference spec from the testdata
+			sl := NewLoader()
+			sl.IsExternalRefsAllowed = true
+			doc, err := sl.LoadFromFile(test.filename)
+			require.NoError(t, err, "loading test file")
+
+			// Internalize the references
+			doc.InternalizeRefs(context.Background(), DefaultRefNameResolver)
+
+			// Validate the internalized spec
+			err = doc.Validate(context.Background())
+			require.Nil(t, err, "validating internalized spec")
+
+			data, err := doc.MarshalJSON()
+			require.NoError(t, err, "marshalling internalized spec")
+
+			// run a static check over the file, making sure each occurence of a
+			// reference is followed by a #
+			numRefs := len(regexpRef.FindAll(data, -1))
+			numInternalRefs := len(regexpRefInternal.FindAll(data, -1))
+			require.Equal(t, numRefs, numInternalRefs, "checking all references are internal")
+
+			// load from data, but with the path set to the current directory
+			doc2, err := sl.LoadFromData(data)
+			require.NoError(t, err, "reloading spec")
+			err = doc2.Validate(context.Background())
+			require.Nil(t, err, "validating reloaded spec")
+		})
+	}
+}
diff --git a/routers/legacy/issue444_test.go b/routers/legacy/issue444_test.go
new file mode 100644
index 0000000..222ecbb
--- /dev/null
+++ b/routers/legacy/issue444_test.go
@@ -0,0 +1,58 @@
+package legacy_test
+
+import (
+	"bytes"
+	"context"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/getkin/kin-openapi/openapi3"
+	"github.com/getkin/kin-openapi/openapi3filter"
+	legacyrouter "github.com/getkin/kin-openapi/routers/legacy"
+	"github.com/stretchr/testify/require"
+)
+
+func TestIssue444(t *testing.T) {
+	loader := openapi3.NewLoader()
+	oas, err := loader.LoadFromData([]byte(`
+openapi: '3.0.0'
+info:
+  title: API
+  version: 1.0.0
+paths:
+  '/path':
+    post:
+      requestBody:
+        required: true
+        content:
+          application/x-yaml:
+            schema:
+              type: object
+      responses:
+        '200':
+          description: x
+          content:
+            application/json:
+              schema:
+                type: string
+`))
+	require.NoError(t, err)
+	router, err := legacyrouter.NewRouter(oas)
+	require.NoError(t, err)
+
+	r := httptest.NewRequest("POST", "/path", bytes.NewReader([]byte(`
+foo: bar
+`)))
+	r.Header.Set("Content-Type", "application/x-yaml")
+
+	openapi3.SchemaErrorDetailsDisabled = true
+	route, pathParams, err := router.FindRoute(r)
+	require.NoError(t, err)
+	reqValidationInput := &openapi3filter.RequestValidationInput{
+		Request:    r,
+		PathParams: pathParams,
+		Route:      route,
+	}
+	err = openapi3filter.ValidateRequest(context.Background(), reqValidationInput)
+	require.NoError(t, err)
+}