Change:
What happened: Created a CRD with a group/version/kind that defines a CustomResourceDefinition type overlapping with the in-tree kubernetes type (WARNING: do not create this CRD on a cluster you care about, see ) Note that two definitions for the group/version/kind appear in openapi: What you expected to happen: published openapi only includes the in-tree type, which has precedence /cc /area custom-resources /priority important-soon /sig api-machinery
/cc
haven't looked into the code yet. I'd expect the naming controller / apiextensions server to prevent conflicting CR GVK, but they are unaware of builtin API groups
CRD handling is isolated to the apiextensions server... it does not have information about all other built-in and aggregated APIs
mentioned something he was working on to make the aggregation layer honor APIService priorities (so that later definitions would not be included if something had already defined a GVK) edit: possibly
a naive idea is to reserve k8s groups in a hard code way that sounds good the CRD in defines a but the kubectl results are about . Did I miss something?
oops... fixed. I started with an API type under but the issue only surfaces for specific APIs (due to the generated definition name colliding for some types)
that wouldn't address duplicate GVKs from aggregated APIs
Three kinds of openapi paths/definitions that we want to prevent CRDs from overwriting: CRD API the spec aggregation in apiextensions-apiserver allows overwriting. I will send a fix native APIs well protected. The spec aggregation in kube-aggregator: 1) ignores duplicated paths and renames duplicated definitions; 2) treats kube-apiserver with than apiextensions-apiserver APIs the spec aggregation in kube-aggregator allows overwriting. The spec aggregation in kube-aggregator treats aggregated APIs with than apiextensions-apiserver. I'm thinking about giving apiextensions-apiserver the lowest priority, but then the problem is aggregated APIs can potentially overwrite the CRD API Note that none of the above prevents CRDs from adding new subpaths to existing APIs. E.g. a CRD can add a path to the CRD API, or to a aggregated API if it didn't have one

Discussion:
require ( github.com/coreos/etcd v3.3.15+incompatible github.com/emicklei/go-restful v2.9.5+incompatible <ins> github.com/ghodss/yaml v0.0.0-20180820084758-c7ce16629ff4 // indirect </ins> github.com/go-openapi/errors v0.19.2 github.com/go-openapi/spec v0.19.2 github.com/go-openapi/strfmt v0.19.0

github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= <ins> github.com/ghodss/yaml v0.0.0-20180820084758-c7ce16629ff4 h1:bRzFpEzvausOAt4va+I/22BZ1vXDtERngp0BNYDKej0= github.com/ghodss/yaml v0.0.0-20180820084758-c7ce16629ff4/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= </ins> github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 h1:DujepqpGd1hyOd7aW59XpK7Qymp8iy83xq74fLr21is= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=

"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//vendor/github.com/emicklei/go-restful:go_default_library", "//vendor/github.com/go-openapi/spec:go_default_library", <ins> "//vendor/k8s.io/kube-openapi/pkg/aggregator:go_default_library", </ins> "//vendor/k8s.io/kube-openapi/pkg/builder:go_default_library", "//vendor/k8s.io/kube-openapi/pkg/common:go_default_library", "//vendor/k8s.io/kube-openapi/pkg/util:go_default_library",

import ( "github.com/go-openapi/spec" <ins> "k8s.io/kube-openapi/pkg/aggregator" </ins> ) // MergeSpecs aggregates all OpenAPI specs, reusing the metadata of the first, static spec as the basis. <del> // Later paths and definitions override earlier ones. None of the input is mutated, but input // and output share data structures. func MergeSpecs(staticSpec *spec.Swagger, crdSpecs ...*spec.Swagger) *spec.Swagger { </del> <ins> // The static spec has the highest priority, and its paths and definitions won't get overlapped by // user-defined CRDs. None of the input is mutated, but input and output share data structures. func MergeSpecs(staticSpec *spec.Swagger, crdSpecs ...*spec.Swagger) (*spec.Swagger, error) { </ins> // create shallow copy of staticSpec, but replace paths and definitions because we modify them. specToReturn := *staticSpec if staticSpec.Definitions != nil {

} } <ins> crdSpec := &spec.Swagger{} </ins> for _, s := range crdSpecs { <del> mergeSpec(&specToReturn, s) </del> <ins> // merge specs without checking conflicts, since the naming controller prevents // conflicts between user-defined CRDs mergeSpec(crdSpec, s) </ins> } <del> return &specToReturn </del> <ins> // The static spec has the highest priority. Resolve conflicts to prevent user-defined // CRDs potentially overlapping the built-in apiextensions API if err := aggregator.MergeSpecsIgnorePathConflict(&specToReturn, crdSpec); err != nil { return nil, err } return &specToReturn, nil </ins> } // mergeSpec copies paths and definitions from source to dest, mutating dest, but not source.

crdSpecs = append(crdSpecs, s) } } <del> return c.openAPIService.UpdateSpec(builder.MergeSpecs(c.staticSpec, crdSpecs...)) </del> <ins> mergedSpec, err := builder.MergeSpecs(c.staticSpec, crdSpecs...) if err != nil { return fmt.Errorf("failed to merge specs: %v", err) } return c.openAPIService.UpdateSpec(mergedSpec) </ins> } func (c *Controller) addCustomResourceDefinition(obj interface{}) {

package master import ( <ins> "bytes" </ins> "encoding/json" "fmt" "net/http"

"testing" "time" <ins> "github.com/go-openapi/spec" </ins> appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" <ins> apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" </ins> metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/registry/generic/registry" "k8s.io/client-go/kubernetes" "k8s.io/kube-aggregator/pkg/apis/apiregistration" kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" <ins> "k8s.io/kubernetes/test/integration/etcd" </ins> "k8s.io/kubernetes/test/integration/framework" ) <ins> const ( // testApiextensionsOverlapProbeString is a probe string which identifies whether // a CRD change triggers an OpenAPI spec change testApiextensionsOverlapProbeString = "testApiextensionsOverlapProbeField" ) </ins> func TestRun(t *testing.T) { server := kubeapiservertesting.StartTestServerOrDie(t, nil, nil, framework.SharedEtcd()) defer server.TearDownFn()

} } <ins> func TestOpenAPIApiextensionsOverlapProtection(t *testing.T) { server := kubeapiservertesting.StartTestServerOrDie(t, nil, nil, framework.SharedEtcd()) defer server.TearDownFn() apiextensionsclient, err := apiextensionsclientset.NewForConfig(server.ClientConfig) if err != nil { t.Fatalf("unexpected error: %v", err) } crdPath, exist, err := getOpenAPIPath(apiextensionsclient, `/apis/apiextensions.k8s.io/v1beta1/customresourcedefinitions/{name}`) if err != nil { t.Fatalf("unexpected error getting CRD OpenAPI path: %v", err) } if !exist { t.Fatalf("unexpected error: apiextensions OpenAPI path doesn't exist") } // Create a CRD that overlaps OpenAPI path with the CRD API crd := &apiextensionsv1beta1.CustomResourceDefinition{ ObjectMeta: metav1.ObjectMeta{ Name:        "customresourcedefinitions.apiextensions.k8s.io", Annotations: map[string]string{"api-approved.kubernetes.io": "unapproved, test-only"}, }, Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ Group:   "apiextensions.k8s.io", Version: "v1beta1", Scope:   apiextensionsv1beta1.ClusterScoped, Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ Plural:   "customresourcedefinitions", Singular: "customresourcedefinition", Kind:     "CustomResourceDefinition", ListKind: "CustomResourceDefinitionList", }, Validation: &apiextensionsv1beta1.CustomResourceValidation{ OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{ Type: "object", Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ testApiextensionsOverlapProbeString: {Type: "boolean"}, }, }, }, }, } etcd.CreateTestCRDs(t, apiextensionsclient, false, crd) // Create a probe CRD foo that triggers an OpenAPI spec change if err := triggerSpecUpdateWithProbeCRD(t, apiextensionsclient, "foo"); err != nil { t.Fatalf("unexpected error: %v", err) } // Expect the CRD path to not change path, _, err := getOpenAPIPath(apiextensionsclient, `/apis/apiextensions.k8s.io/v1beta1/customresourcedefinitions/{name}`) if err != nil { t.Fatalf("unexpected error: %v", err) } pathBytes, err := json.Marshal(path) if err != nil { t.Fatalf("unexpected error: %v", err) } crdPathBytes, err := json.Marshal(crdPath) if err != nil { t.Fatalf("unexpected error: %v", err) } if !bytes.Equal(pathBytes, crdPathBytes) { t.Fatalf("expected CRD OpenAPI path to not change, but got different results: want %q, got %q", string(crdPathBytes), string(pathBytes)) } // Expect the orphan definition to be pruned from the spec exist, err = specHasProbe(apiextensionsclient, testApiextensionsOverlapProbeString) if err != nil { t.Fatalf("unexpected error: %v", err) } if exist { t.Fatalf("unexpected error: orphan definition isn't pruned") } // Create a CRD that overlaps OpenAPI definition with the CRD API crd = &apiextensionsv1beta1.CustomResourceDefinition{ ObjectMeta: metav1.ObjectMeta{ Name:        "customresourcedefinitions.apiextensions.apis.pkg.apiextensions-apiserver.k8s.io", Annotations: map[string]string{"api-approved.kubernetes.io": "unapproved, test-only"}, }, Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ Group:   "apiextensions.apis.pkg.apiextensions-apiserver.k8s.io", Version: "v1beta1", Scope:   apiextensionsv1beta1.ClusterScoped, Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ Plural:   "customresourcedefinitions", Singular: "customresourcedefinition", Kind:     "CustomResourceDefinition", ListKind: "CustomResourceDefinitionList", }, Validation: &apiextensionsv1beta1.CustomResourceValidation{ OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{ Type: "object", Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ testApiextensionsOverlapProbeString: {Type: "boolean"}, }, }, }, }, } etcd.CreateTestCRDs(t, apiextensionsclient, false, crd) // Create a probe CRD bar that triggers an OpenAPI spec change if err := triggerSpecUpdateWithProbeCRD(t, apiextensionsclient, "bar"); err != nil { t.Fatalf("unexpected error: %v", err) } // Expect the apiextensions definition to not change, since the overlapping definition will get renamed. apiextensionsDefinition, exist, err := getOpenAPIDefinition(apiextensionsclient, `io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1beta1.CustomResourceDefinition`) if err != nil { t.Fatalf("unexpected error: %v", err) } if !exist { t.Fatalf("unexpected error: apiextensions definition doesn't exist") } bytes, err := json.Marshal(apiextensionsDefinition) if exist := strings.Contains(string(bytes), testApiextensionsOverlapProbeString); exist { t.Fatalf("unexpected error: apiextensions definition gets overlapped") } } // triggerSpecUpdateWithProbeCRD creates a probe CRD with suffix in name, and waits until // the path and definition for the probe CRD show up in the OpenAPI spec func triggerSpecUpdateWithProbeCRD(t *testing.T, apiextensionsclient *apiextensionsclientset.Clientset, suffix string) error { // Create a probe CRD that triggers OpenAPI spec change name := fmt.Sprintf("integration-test-%s-crd", suffix) kind := fmt.Sprintf("Integration-test-%s-crd", suffix) group := "probe.test.com" crd := &apiextensionsv1beta1.CustomResourceDefinition{ ObjectMeta: metav1.ObjectMeta{Name: name + "s." + group}, Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ Group:   group, Version: "v1", Scope:   apiextensionsv1beta1.ClusterScoped, Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ Plural:   name + "s", Singular: name, Kind:     kind, ListKind: kind + "List", }, }, } etcd.CreateTestCRDs(t, apiextensionsclient, false, crd) // Expect the probe CRD path to show up in the OpenAPI spec // TODO(roycaihw): expose response header in rest client and utilize etag here if err := wait.Poll(1*time.Second, wait.ForeverTestTimeout, func() (bool, error) { _, exist, err := getOpenAPIPath(apiextensionsclient, fmt.Sprintf(`/apis/%s/v1/%ss/{name}`, group, name)) if err != nil { return false, err } return exist, nil }); err != nil { return fmt.Errorf("failed to observe probe CRD path in the spec: %v", err) } return nil } func specHasProbe(clientset *apiextensionsclientset.Clientset, probe string) (bool, error) { bs, err := clientset.RESTClient().Get().AbsPath("openapi", "v2").DoRaw() if err != nil { return false, err } return strings.Contains(string(bs), probe), nil } func getOpenAPIPath(clientset *apiextensionsclientset.Clientset, path string) (spec.PathItem, bool, error) { bs, err := clientset.RESTClient().Get().AbsPath("openapi", "v2").DoRaw() if err != nil { return spec.PathItem{}, false, err } s := spec.Swagger{} if err := json.Unmarshal(bs, &s); err != nil { return spec.PathItem{}, false, err } if s.SwaggerProps.Paths == nil { return spec.PathItem{}, false, fmt.Errorf("unexpected empty path") } value, ok := s.SwaggerProps.Paths.Paths[path] return value, ok, nil } func getOpenAPIDefinition(clientset *apiextensionsclientset.Clientset, definition string) (spec.Schema, bool, error) { bs, err := clientset.RESTClient().Get().AbsPath("openapi", "v2").DoRaw() if err != nil { return spec.Schema{}, false, err } s := spec.Swagger{} if err := json.Unmarshal(bs, &s); err != nil { return spec.Schema{}, false, err } if s.SwaggerProps.Definitions == nil { return spec.Schema{}, false, fmt.Errorf("unexpected empty path") } value, ok := s.SwaggerProps.Definitions[definition] return value, ok, nil } </ins> // return the unique endpoint IPs func getEndpointIPs(endpoints *corev1.Endpoints) []string { endpointMap := make(map[string]bool)