Change:
What happened: While upgrading from v1.15 to try out kubernetes v1.16.0-beta.2, kube-apiserver is printing error messages: This also brakes the API server since the user is unable to even delete the "faulty" CR What you expected to happen: Not to print any error message and accept a CR that was already valid in 1.15 How to reproduce it (as minimally and precisely as possible): Install k8s 1.15.x (optional) Create the following CR Upgrade to k8s v1.16.0-beta.2check kube-apiserver for errors:Try delete the faulty CR (Won't be possible) Environment: Kubernetes version (use ): Anything else we need to know?: A quick search sent me to this code path that is being used to compile k8s It seems the problem it known but not yet fixed since this is in the latest commit of : /kind bug /king regression cc
/sig api-machinery
/assign /priority critical-urgent /milestone v1.16
This is caused by compiling the openapi spec for use by server-side apply here: SSA is using the method, which has significant gaps in handling valid openapi specs (see all the TODO comments in ) and until now was only used for client-side kubectl validation (as far as I'm aware). Using it in the server-side path is concerning. /assign
running with , this works as expected and is compatible with 1.15
/assign
fix and covering test is in
this is resolved in , available now

Discussion:
structuralSchemas[v.Name] = s } <del> var openAPIModels proto.Models if utilfeature.DefaultFeatureGate.Enabled(features.ServerSideApply) && r.staticOpenAPISpec != nil { specs := []*spec.Swagger{} for _, v := range crd.Spec.Versions { s, err := builder.BuildSwagger(crd, v.Name, builder.Options{V2: false, StripDefaults: true, StripValueValidation: true}) if err != nil { utilruntime.HandleError(err) return nil, fmt.Errorf("the server could not properly serve the CR schema") } specs = append(specs, s) } mergedOpenAPI, err := builder.MergeSpecs(r.staticOpenAPISpec, specs...) if err != nil { utilruntime.HandleError(err) return nil, fmt.Errorf("the server could not properly merge the CR schema") } openAPIModels, err = utilopenapi.ToProtoModels(mergedOpenAPI) if err != nil { utilruntime.HandleError(err) return nil, fmt.Errorf("the server could not properly serve the CR schema") } </del> <ins> openAPIModels, err := buildOpenAPIModelsForApply(r.staticOpenAPISpec, crd) if err != nil { utilruntime.HandleError(fmt.Errorf("error building openapi models for %s: %v", crd.Name, err)) openAPIModels = nil </ins> } for _, v := range crd.Spec.Versions {

} return err } <ins> // buildOpenAPIModelsForApply constructs openapi models from any validation schemas specified in the custom resource, // and merges it with the models defined in the static OpenAPI spec. // Returns nil models if the ServerSideApply feature is disabled, or the static spec is nil, or an error is encountered. func buildOpenAPIModelsForApply(staticOpenAPISpec *spec.Swagger, crd *apiextensions.CustomResourceDefinition) (proto.Models, error) { if !utilfeature.DefaultFeatureGate.Enabled(features.ServerSideApply) { return nil, nil } if staticOpenAPISpec == nil { return nil, nil } specs := []*spec.Swagger{} for _, v := range crd.Spec.Versions { s, err := builder.BuildSwagger(crd, v.Name, builder.Options{V2: false, StripDefaults: true, StripValueValidation: true}) if err != nil { return nil, err } specs = append(specs, s) } mergedOpenAPI, err := builder.MergeSpecs(staticOpenAPISpec, specs...) if err != nil { return nil, err } models, err := utilopenapi.ToProtoModels(mergedOpenAPI) if err != nil { return nil, err } return models, nil } </ins>

t.Fatalf("expected %v ports but got %v:n%v", expected, actual, string(b)) } } <ins> // TestApplyCRDUnhandledSchema tests that when a CRD has a schema that kube-openapi ToProtoModels cannot handle correctly, // apply falls back to non-schema behavior func TestApplyCRDUnhandledSchema(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.ServerSideApply, true)() server, err := apiservertesting.StartTestServer(t, apiservertesting.NewDefaultTestServerOptions(), nil, framework.SharedEtcd()) if err != nil { t.Fatal(err) } defer server.TearDownFn() config := server.ClientConfig apiExtensionClient, err := clientset.NewForConfig(config) if err != nil { t.Fatal(err) } dynamicClient, err := dynamic.NewForConfig(config) if err != nil { t.Fatal(err) } noxuDefinition := fixtures.NewNoxuCustomResourceDefinition(apiextensionsv1beta1.ClusterScoped) // This is a schema that kube-openapi ToProtoModels does not handle correctly. // https://github.com/kubernetes/kubernetes/blob/38752f7f99869ed65fb44378360a517649dc2f83/vendor/k8s.io/kube-openapi/pkg/util/proto/document.go#L184 var c apiextensionsv1beta1.CustomResourceValidation err = json.Unmarshal([]byte(`{ "openAPIV3Schema": { "properties": { "TypeFooBar": { "type": "array" } } } }`), &c) if err != nil { t.Fatal(err) } noxuDefinition.Spec.Validation = &c noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient) if err != nil { t.Fatal(err) } kind := noxuDefinition.Spec.Names.Kind apiVersion := noxuDefinition.Spec.Group + "/" + noxuDefinition.Spec.Version name := "mytest" rest := apiExtensionClient.Discovery().RESTClient() yamlBody := []byte(fmt.Sprintf(` apiVersion: %s kind: %s metadata: name: %s spec: replicas: 1`, apiVersion, kind, name)) result, err := rest.Patch(types.ApplyPatchType). AbsPath("/apis", noxuDefinition.Spec.Group, noxuDefinition.Spec.Version, noxuDefinition.Spec.Names.Plural). Name(name). Param("fieldManager", "apply_test"). Body(yamlBody). DoRaw() if err != nil { t.Fatalf("failed to create custom resource with apply: %v:n%v", err, string(result)) } verifyReplicas(t, result, 1) // Patch object to change the number of replicas result, err = rest.Patch(types.MergePatchType). AbsPath("/apis", noxuDefinition.Spec.Group, noxuDefinition.Spec.Version, noxuDefinition.Spec.Names.Plural). Name(name). Body([]byte(`{"spec":{"replicas": 5}}`)). DoRaw() if err != nil { t.Fatalf("failed to update number of replicas with merge patch: %v:n%v", err, string(result)) } verifyReplicas(t, result, 5) // Re-apply, we should get conflicts now, since the number of replicas was changed. result, err = rest.Patch(types.ApplyPatchType). AbsPath("/apis", noxuDefinition.Spec.Group, noxuDefinition.Spec.Version, noxuDefinition.Spec.Names.Plural). Name(name). Param("fieldManager", "apply_test"). Body(yamlBody). DoRaw() if err == nil { t.Fatalf("Expecting to get conflicts when applying object after updating replicas, got no error: %s", result) } status, ok := err.(*errors.StatusError) if !ok { t.Fatalf("Expecting to get conflicts as API error") } if len(status.Status().Details.Causes) != 1 { t.Fatalf("Expecting to get one conflict when applying object after updating replicas, got: %v", status.Status().Details.Causes) } // Re-apply with force, should work fine. result, err = rest.Patch(types.ApplyPatchType). AbsPath("/apis", noxuDefinition.Spec.Group, noxuDefinition.Spec.Version, noxuDefinition.Spec.Names.Plural). Name(name). Param("force", "true"). Param("fieldManager", "apply_test"). Body(yamlBody). DoRaw() if err != nil { t.Fatalf("failed to apply object with force after updating replicas: %v:n%v", err, string(result)) } verifyReplicas(t, result, 1) } </ins>